diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 94e78e0..3385b16 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -62,6 +62,8 @@ jobs: sudo apt-get update -qq sudo apt-get install -y llvm python3 tools/check-alloc-symbols.py --arch ${{ matrix.arch }} + - name: Link libkore.so and check its exported ABI + run: python3 tools/check-program.py --arch ${{ matrix.arch }} miri: runs-on: ubuntu-latest timeout-minutes: 30 diff --git a/Cargo.lock b/Cargo.lock index df51275..7e4ec99 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -802,6 +802,23 @@ dependencies = [ "kore-window", ] +[[package]] +name = "kore-c" +version = "0.1.0-alpha.1" +dependencies = [ + "kore-c-shim", + "kore-ffi", + "kore-ld", + "kore-std", +] + +[[package]] +name = "kore-c-shim" +version = "0.1.0" +dependencies = [ + "kore-rt", +] + [[package]] name = "kore-collections" version = "0.1.0" @@ -1314,9 +1331,9 @@ checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d" [[package]] name = "rustix" -version = "1.1.5" +version = "1.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "891efababe418670775f199f0d233d84843c227a0949a883ce15b37c78d6629d" +checksum = "e5e76742e5294153d4dde1207dbec5bdd81eef6fabdc0d2118babadabfd858aa" dependencies = [ "bitflags", "errno", diff --git a/Cargo.toml b/Cargo.toml index b69f68b..e6ec140 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,7 +8,7 @@ repository = "https://github.com/MicroPerceptron/libkore" [workspace] resolver = "3" -members = ["tests/consumer", "crates/ffi", "crates/entry", "crates/wasi", "crates/lua", "crates/host"] +members = ["tests/consumer", "crates/ffi", "crates/c", "crates/entry", "crates/wasi", "crates/lua", "crates/host"] exclude = ["native"] [features] @@ -35,4 +35,4 @@ kore-driver = { path = "native/crates/driver", optional = true } # root: the offline volume procedure (and the filesystem service closure it # drives) that host programs such as vKore consume; it never enters a Kore # executable's graph. -owner-paths = { kore-start = "crates/start", kore-abi = "crates/abi", kore-app = "crates/app", kore-rt = "crates/rt", kore-dagfs = "crates/dagfs", kore-driver = "crates/driver", kore-late = "crates/late", kore-vol = "tools/kore-vol" } +owner-paths = { kore-start = "crates/start", kore-abi = "crates/abi", kore-app = "crates/app", kore-rt = "crates/rt", kore-c-shim = "crates/c-shim", kore-dagfs = "crates/dagfs", kore-driver = "crates/driver", kore-late = "crates/late", kore-ld = "crates/ld", kore-vol = "tools/kore-vol" } diff --git a/Makefile b/Makefile index ff12454..99d83ac 100644 --- a/Makefile +++ b/Makefile @@ -17,7 +17,7 @@ check: python3 tools/check-native.py fmt: - cargo fmt -p kore-std -p libkore-consumer -p kore-ffi -p kore-entry -p kore-wasi -p kore-lua -p kore-host --check + cargo fmt -p kore-std -p libkore-consumer -p kore-ffi -p kore-c -p kore-entry -p kore-wasi -p kore-lua -p kore-host --check $(ZIG) fmt --check build.zig build.zig.zon crates/ffi/tests/consumer.zig extern/zig/src/root.zig extern/zig/src/allocator.zig docs: @@ -30,7 +30,9 @@ owners: python3 tools/check-owners.py < target/owners.json # Actual Kore user targets, including its hardware-float x86 ABI. A stock -# x86_64-unknown-none check alone cannot certify this interface. `kore-lua` +# x86_64-unknown-none check alone cannot certify this interface. `kore-c` is +# linked rather than checked (`tools/check-program.py`): `libkore.so`'s +# exported ABI, soname and TLS model are properties of the linked image. `kore-lua` # is here because its engine exists only on these targets — its host build # carries the logic and none of the C — so a Kore-only regression in the # amalgamation would otherwise reach a keytest before it reached a @@ -46,6 +48,7 @@ cross: -Z build-std-features=compiler-builtins-mem; \ done python3 tools/check-alloc-symbols.py + python3 tools/check-program.py python3 tools/check-entry.py # kore-host drives real image files under POSIX record locks, which Miri does diff --git a/README.md b/README.md index caae6f0..4476d13 100644 --- a/README.md +++ b/README.md @@ -14,12 +14,13 @@ machine's volume through the same exported owners a booted Kore runs. | --- | --- | | `kore-std` | Rust facade: filesystem, prelude, time, random, collections, process and optional driver APIs | | `kore-entry` | Executable-only adapter for a returning Rust `fn main()` on Kore | -| `kore-ffi` | C headers and Zig module over a Rust-founded host, including filesystem, time, random, allocation and worker APIs; no standalone C/Zig CRT | +| `kore-ffi` | C headers and Zig module over a Rust-founded host, including filesystem, time, random, allocation and worker APIs | +| `kore-c` | The runtime of `libkore.so`: `kore.h` as one shared object, and the host of an independent C program (`main` on a worker, its filesystem session, its end) | | `kore-lua` | Embeddable Lua 5.5 engine and bridge; interactive and opt-in restricted worker policies | | `kore-wasi` | WASI 0.2 component host composed by an embedder with its Wasmtime engine and native owners | | `kore-host` | Host-side (Linux/macOS) tooling: a Kore machine's DagFS volume, offline, through the filesystem row's own engine | -All six package manifests set `publish = false`; consume a reviewed Git +All seven package manifests set `publish = false`; consume a reviewed Git revision or a deliberately composed local checkout, not a registry version. The WASI host supports the `wasi:cli/command` world and native-backed I/O, filesystem, clocks and random imports; unsupported imports such as sockets diff --git a/crates/c/Cargo.toml b/crates/c/Cargo.toml new file mode 100644 index 0000000..2188bbc --- /dev/null +++ b/crates/c/Cargo.toml @@ -0,0 +1,33 @@ +[package] +name = "kore-c" +version = "0.1.0-alpha.1" +edition = "2024" +publish = false +description = "The runtime of libkore.so: the C ABI and the host of an independent C program on Kore" +# No native library is linked: the key carries the headers this runtime +# implements to dependents' build scripts as `DEP_KORE_INCLUDE` (`build.rs`). +links = "kore" +build = "build.rs" + +# The runtime `libkore.so` is built from: a distribution's one-line `cdylib` +# crate names this one and links it under its own `[lib] name = "kore"`, so +# the image is `libkore.so` (Kore's is `user/runtime/libkore`). Link it into +# that shared object alone: it installs the process heap and the panic handler +# a C program's runtime needs, which a Rust executable brings itself. +[lib] +name = "kore_c" +path = "src/lib.rs" + +[dependencies] +kore-std = { package = "kore-std", path = "../..", features = ["heap", "fs"] } +# The C ABI itself, with the standard C allocation and thread names a C +# program calls (`malloc`, `pthread_create`, ...) exported as real symbols. +kore-ffi = { path = "../ffi", features = ["c-stdlib"] } +# Kore's C library: the `printf` family and the standard streams, `` +# and ``'s parsers, `errno`. Its process half without its own +# `malloc` and `abort`: kore-ffi's allocator is this image's `malloc`, and the +# host ends a whole program. +kore-c-shim = { path = "../../native/crates/c-shim", default-features = false, features = ["runtime"] } +# The link window's placer: one cursor every `mmap` of the process takes its +# address from, as a Wasmtime engine's guest mappings do. +kore-ld = { path = "../../native/crates/ld", default-features = false } diff --git a/crates/c/build.rs b/crates/c/build.rs new file mode 100644 index 0000000..6928f56 --- /dev/null +++ b/crates/c/build.rs @@ -0,0 +1,29 @@ +//! Publish the C headers `libkore.so` implements to whoever links this crate +//! (`links = "kore"`): a dependent's build script reads `DEP_KORE_INCLUDE`, the +//! directory holding `kore.h` and `kore/`, from this exact source revision, so +//! the headers a distribution ships cannot drift from the image it builds. +use std::path::PathBuf; + +fn main() { + let manifest = + PathBuf::from(std::env::var_os("CARGO_MANIFEST_DIR").expect("CARGO_MANIFEST_DIR")); + let include = manifest.join("../../extern/c/include"); + let include = include + .canonicalize() + .unwrap_or_else(|error| panic!("{}: {error}", include.display())); + for header in [ + "kore.h", + "kore/stdlib.h", + "kore/pthread.h", + "kore/program.h", + ] { + let path = include.join(header); + assert!( + path.is_file(), + "missing generated header {}", + path.display() + ); + println!("cargo:rerun-if-changed={}", path.display()); + } + println!("cargo:include={}", include.display()); +} diff --git a/crates/c/cbindgen.toml b/crates/c/cbindgen.toml new file mode 100644 index 0000000..0c770e4 --- /dev/null +++ b/crates/c/cbindgen.toml @@ -0,0 +1,23 @@ +language = "C" +include_guard = "KORE_PROGRAM_H" +pragma_once = true +cpp_compat = true +usize_is_size_t = true +style = "both" +no_includes = true +includes = ["../kore.h"] +autogen_warning = "/* Generated by cbindgen 0.29.4 from kore-c. Do not edit. */" +header = "/* libkore.so: what an independent C program on Kore adds to kore.h. Link with -lkore. */" +[parse] +parse_deps = false +[export] +# The C library's half of the program lifecycle (its start and exit call +# these), not an application API: the C library declares them itself, as +# its declares `abort`. `KoreFsStep` is kore.h's typedef, which +# this header includes. +# `open`, `close`, `mmap` and `munmap` are POSIX's, declared by , +# and . +exclude = ["__kore_program_start", "__kore_program_exit", "__kore_program_arguments", "abort", "open", "close", "mmap", "munmap", "KoreFsStep"] +[enum] +prefix_with_name = true +rename_variants = "ScreamingSnakeCase" diff --git a/crates/c/src/api.rs b/crates/c/src/api.rs new file mode 100644 index 0000000..aa16bb3 --- /dev/null +++ b/crates/c/src/api.rs @@ -0,0 +1,220 @@ +//! The C entry points `kore/program.h` declares: what an independent C program +//! on Kore adds to `kore.h` by linking `libkore.so`. +//! +//! Compiled on every target so the header has one source of declarations; +//! off Kore there is no program to host, and each entry refuses. + +use core::ffi::c_void; + +use kore_ffi::{KoreFsError, KoreStatus}; + +/// `kore.h`'s `KoreFsStep` as a C caller may pass it: possibly null. +pub type KoreFsStep = Option; + +/// The stack `main` runs on: a guarded strip of the task's stack window, +/// committed whole when the program starts. A frame deeper than this faults +/// on the guard page below it, as a stack overflow does. +/// +/// The size of the strip `tcc`, a large C program whose depth its input +/// decides, measured its need under Kore's runtime at and runs in; ISO C's +/// own guarantee of 127 nested blocks (C11 §5.2.4.1) needs a small part of +/// it. Larger strips come out of the 16 MiB window the program's SPMD gangs +/// share, and would be committed at every launch. +pub const KORE_MAIN_STACK_BYTES: usize = 1 << 20; + +/// Run `step` as this program's filesystem session, through the filesystem +/// face its launch carried, rooted where it was launched (its working +/// directory). The callbacks and their pointers obey `KoreFsStep`'s contract: +/// each callback runs on the program's host thread, as on any native +/// executor, while the caller waits, so a callback must not depend on the +/// caller's thread-local state. +/// +/// Returns `KORE_STATUS_OK` once the callback program ended and every view it +/// left open closed. `KORE_STATUS_QUARANTINED`: a reply was lost or +/// uncertain, and no session can run again in this program. +/// `KORE_STATUS_REFUSED`: a close was refused (`*error` names why, and the +/// view stays open), the launch carried no filesystem face +/// (`KORE_FS_ERROR_UNSUPPORTED`), or the caller is not the thread `main` runs +/// on, which alone waits for the host. `KORE_STATUS_CAPACITY`: the native +/// owner could not be formed. `KORE_STATUS_INVALID_ARGUMENT`: `step` is null. +/// `*error` is written whenever `error` is not null. +/// +/// # Safety +/// `step` obeys `KoreFsStep`'s pointer, serialization and no-unwind contract; +/// `user` remains valid, and the callbacks' alone, until this returns. `error` +/// is null or valid for one aligned write. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn kore_fs_run( + step: KoreFsStep, + user: *mut c_void, + error: *mut KoreFsError, +) -> KoreStatus { + let (status, cause) = match step { + None => (KoreStatus::InvalidArgument, KoreFsError::None), + // SAFETY: the caller's contract, forwarded whole. + #[cfg(target_os = "none")] + Some(step) => unsafe { crate::program::session(step, user) }, + // No program to host off Kore. + #[cfg(not(target_os = "none"))] + Some(_) => { + let _ = user; + (KoreStatus::Refused, KoreFsError::None) + } + }; + if !error.is_null() { + // SAFETY: the caller's contract: null or valid for one aligned write. + unsafe { error.write(cause) }; + } + status +} + +/// [`KoreFdInfo::kind`]: the memory supply the program's allocator grows +/// on. `mmap` answers a fresh, zeroed region of it. +pub const KORE_FD_MEMORY: u32 = 1; +/// [`KoreFdInfo::access`]: the capability may be read. +pub const KORE_FD_READ: u32 = 1 << 0; +/// [`KoreFdInfo::access`]: the capability may be written. +pub const KORE_FD_WRITE: u32 = 1 << 1; + +/// What a descriptor names, as the calling thread holds it +/// ([`kore_fd_describe`]). +#[repr(C)] +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct KoreFdInfo { + /// What kind of capability it is: `KORE_FD_MEMORY`. + pub kind: u32, + /// What the capability's own rights permit this thread, as + /// `KORE_FD_READ | KORE_FD_WRITE` bits: the rights it was founded with, + /// which the access `open` asked for can only narrow. + pub access: u32, + /// For `KORE_FD_MEMORY`, the supply's whole length in bytes: what the + /// allocator and every mapping share, not what is still free. + pub length: u64, +} + +/// Describe the capability behind `fd`, a descriptor `open` answered +/// (``): `open` finds a capability this program was founded holding +/// by its name, and this reads what was found. Nothing is granted or changed. +/// +/// Capabilities are held per thread, so the answer is the calling thread's: +/// the same descriptor may describe a capability on the thread that opened +/// it and be refused on one founded without it. +/// +/// Returns `KORE_STATUS_OK` with `*out` written. +/// `KORE_STATUS_INVALID_ARGUMENT`: `out` is null. `KORE_STATUS_STALE_HANDLE`: +/// `fd` is not an open descriptor. `KORE_STATUS_REFUSED`: the calling thread +/// holds no such capability, or this is not a Kore program. A refusal leaves +/// `*out` unchanged. +/// +/// # Safety +/// `out` is null or valid for one aligned write. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn kore_fd_describe( + fd: core::ffi::c_int, + out: *mut KoreFdInfo, +) -> KoreStatus { + if out.is_null() { + return KoreStatus::InvalidArgument; + } + let Ok((row, _)) = crate::fd::TABLE.get(fd) else { + return KoreStatus::StaleHandle; + }; + let Some(info) = describe(&crate::fd::NAMES[row]) else { + return KoreStatus::Refused; + }; + // SAFETY: the caller's contract: valid for one aligned write. + unsafe { out.write(info) }; + KoreStatus::Ok +} + +/// What the calling thread holds at `name`. +#[cfg(target_os = "none")] +fn describe(name: &crate::fd::Name) -> Option { + use crate::fd::Kind; + let held = crate::fd::holding(name)?; + let (kind, length) = match name.kind { + Kind::Memory => ( + KORE_FD_MEMORY, + kore_std::runtime::cap::verbs::lease_geometry(held.slot)?.1, + ), + }; + Some(KoreFdInfo { + kind, + access: if held.readable { KORE_FD_READ } else { 0 } + | if held.writable { KORE_FD_WRITE } else { 0 }, + length, + }) +} + +/// Off Kore no thread holds a capability. +#[cfg(not(target_os = "none"))] +fn describe(_name: &crate::fd::Name) -> Option { + None +} + +#[cfg(test)] +mod tests { + use super::*; + + unsafe extern "C" fn never( + _: *mut kore_ffi::KoreFsContext, + _: *const kore_ffi::KoreFsCompletion, + _: *mut c_void, + ) { + panic!("a session ran with no program to host"); + } + + #[test] + fn a_null_step_is_an_invalid_argument_and_the_error_is_written() { + let mut error = KoreFsError::Busy; + // SAFETY: a null step names no callback; `error` is valid. + let status = unsafe { kore_fs_run(None, core::ptr::null_mut(), &mut error) }; + assert_eq!(status, KoreStatus::InvalidArgument); + assert_eq!(error, KoreFsError::None); + } + + #[test] + fn with_no_program_to_host_a_session_is_refused_and_never_runs() { + let mut error = KoreFsError::Busy; + // SAFETY: `never` is never called; `error` is valid. + let status = unsafe { kore_fs_run(Some(never), core::ptr::null_mut(), &mut error) }; + assert_eq!(status, KoreStatus::Refused); + assert_eq!(error, KoreFsError::None); + // A null error pointer is permitted. + // SAFETY: as above. + let status = + unsafe { kore_fs_run(Some(never), core::ptr::null_mut(), core::ptr::null_mut()) }; + assert_eq!(status, KoreStatus::Refused); + } + + #[test] + fn describing_refuses_by_cause_and_leaves_the_answer_alone() { + let untouched = KoreFdInfo { + kind: 7, + access: 7, + length: 7, + }; + let mut info = untouched; + // SAFETY: null is permitted; `info` is valid. + unsafe { + assert_eq!( + kore_fd_describe(3, core::ptr::null_mut()), + KoreStatus::InvalidArgument + ); + // The streams' numbers and an unopened one are not descriptors. + for fd in [0, 1, 2, -1, 63, 1 << 20] { + assert_eq!(kore_fd_describe(fd, &mut info), KoreStatus::StaleHandle); + } + } + // An open descriptor, off Kore: no thread holds a capability. + let access = crate::fd::Access::of(crate::fd::O_RDWR).unwrap(); + let fd = crate::fd::TABLE.claim(0, access).unwrap(); + // SAFETY: `info` is valid. + assert_eq!( + unsafe { kore_fd_describe(fd, &mut info) }, + KoreStatus::Refused + ); + crate::fd::TABLE.release(fd).unwrap(); + assert_eq!(info, untouched); + } +} diff --git a/crates/c/src/args.rs b/crates/c/src/args.rs new file mode 100644 index 0000000..693bf2e --- /dev/null +++ b/crates/c/src/args.rs @@ -0,0 +1,247 @@ +//! **A C program's arguments**: `main`'s `argc` and `argv`. +//! +//! A launcher hands a program its command line on the process page: the name +//! it resolved the program by ([`ProcessIo::command`]) and the opaque tail +//! after it ([`ProcessIo::args`]). C wants a vector. The host reads the page +//! once, on thread 0 before the main thread exists, splits the tail the way +//! every kore program that wants words does ([`words`]: a POSIX shell's +//! quoting, without expansion), and keeps the vector for the life of the +//! program. The C library's start asks for it through +//! [`__kore_program_arguments`] and passes it to `main`. +//! +//! `argv[0]` is the command name, or the empty string when the launcher gave +//! none (C11 §5.1.2.2.1p2). A launch with no page at all has no arguments, +//! not even a name: `argc` is zero, which the same clause permits. The +//! strings are the program's to change, as that clause requires, and +//! `argv[argc]` is null. +//! +//! A tail that does not split, an unclosed quote or a lone trailing +//! backslash, is the launcher's line being malformed, not something `main` +//! could be handed: the host says so on `stderr` and ends the program before +//! it runs, as `wasm` and `tcc` answer the same line. +//! +//! [`ProcessIo::command`]: kore_std::abi::process::ProcessIo::command +//! [`ProcessIo::args`]: kore_std::abi::process::ProcessIo::args +//! [`words`]: kore_std::process::words + +extern crate alloc; + +use alloc::vec::Vec; + +use kore_std::process::words::{self, WordsError}; + +/// An argument vector's storage: every word NUL-terminated, back to back, +/// and where each begins. +#[derive(Debug, PartialEq, Eq)] +pub(crate) struct Vector { + bytes: Vec, + starts: Vec, +} + +impl Vector { + /// `command`, then the words of `tail`. + /// + /// A NUL inside a word would end the C string early and hand `main` a + /// different argument than the line spelled, so such a line is no + /// vector either. + pub(crate) fn of(command: &[u8], tail: &[u8]) -> Result { + let mut vector = Self { + bytes: Vec::new(), + starts: Vec::new(), + }; + vector.push(command.iter().copied())?; + for word in words::words(tail) { + vector.push(word.map_err(Malformed::Words)?.bytes())?; + } + Ok(vector) + } + + fn push(&mut self, word: impl Iterator) -> Result<(), Malformed> { + self.starts.push(self.bytes.len()); + for byte in word { + if byte == 0 { + return Err(Malformed::Nul { + word: self.starts.len() - 1, + }); + } + self.bytes.push(byte); + } + self.bytes.push(0); + Ok(()) + } + + /// Words held: `argc`. + pub(crate) fn len(&self) -> usize { + self.starts.len() + } + + /// Word `index`, without its terminator. + #[cfg(test)] + fn word(&self, index: usize) -> &[u8] { + let start = self.starts[index]; + let end = self.bytes[start..] + .iter() + .position(|byte| *byte == 0) + .unwrap(); + &self.bytes[start..start + end] + } +} + +/// Why a command line is no argument vector. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum Malformed { + /// The tail does not split into words. + Words(WordsError), + /// Word `word` (0 is the command name) holds a NUL byte. + Nul { word: usize }, +} + +impl core::fmt::Display for Malformed { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + match self { + Self::Words(error) => error.fmt(f), + Self::Nul { word } => write!(f, "argument {word} holds a NUL byte"), + } + } +} + +#[cfg(target_os = "none")] +pub(crate) use metal::capture; + +#[cfg(target_os = "none")] +mod metal { + use super::alloc::boxed::Box; + use super::alloc::vec::Vec; + use super::{Malformed, Vector}; + use core::ffi::{c_char, c_int}; + use core::sync::atomic::{AtomicI32, AtomicPtr, Ordering}; + use kore_std::runtime::entry::Rt; + + /// `argc`, and the vector `argv` points at: `argc` pointers and a null. + /// Published once by [`capture`] on thread 0 before the main thread is + /// founded, whose founding orders these stores before `main`'s loads. + static ARGC: AtomicI32 = AtomicI32::new(0); + static ARGV: AtomicPtr<*mut c_char> = AtomicPtr::new(core::ptr::null_mut()); + + /// Read the launcher's page and keep the program's argument vector. + /// Thread 0, once, before any other thread exists. A launch that carried + /// no page leaves the program without arguments. + pub(crate) fn capture(rt: &Rt) -> Result<(), Malformed> { + let Some(page) = kore_std::process::map(rt) else { + return Ok(()); + }; + // A page that does not validate is no command line: no arguments, + // as for a launch without one. + let (Ok(command), Ok(tail)) = (page.command(), page.args()) else { + return Ok(()); + }; + let Vector { bytes, starts } = Vector::of(command, tail)?; + // The program's for its whole life, and its to change: never freed. + let bytes: &'static mut [u8] = Box::leak(bytes.into_boxed_slice()); + let base = bytes.as_mut_ptr(); + let mut pointers: Vec<*mut c_char> = starts + .iter() + // SAFETY: each start is an offset inside `bytes`. + .map(|start| unsafe { base.add(*start) }.cast::()) + .collect(); + pointers.push(core::ptr::null_mut()); + let argc = c_int::try_from(starts.len()).unwrap_or(c_int::MAX); + let pointers: &'static mut [*mut c_char] = Box::leak(pointers.into_boxed_slice()); + ARGV.store(pointers.as_mut_ptr(), Ordering::Release); + ARGC.store(argc, Ordering::Release); + Ok(()) + } + + /// The program's argument vector: answers `argc` and stores `argv` + /// through `argv`, `argv[argc]` null. The C library's start calls it on + /// the main thread before `main`. A program whose launch carried no + /// command line gets zero and a vector holding only the null. + /// + /// # Safety + /// `argv` is null or names one writable `char **`. + #[unsafe(no_mangle)] + pub unsafe extern "C" fn __kore_program_arguments(argv: *mut *mut *mut c_char) -> c_int { + /// The vector of a program without arguments: its null. + struct Empty(core::cell::UnsafeCell<[*mut c_char; 1]>); + // SAFETY: never written: C may not store through `argv[argc]`'s + // slot, and there is no other. + unsafe impl Sync for Empty {} + static EMPTY: Empty = Empty(core::cell::UnsafeCell::new([core::ptr::null_mut()])); + + let held = ARGV.load(Ordering::Acquire); + let (count, vector) = if held.is_null() { + (0, EMPTY.0.get().cast::<*mut c_char>()) + } else { + (ARGC.load(Ordering::Acquire), held) + }; + if !argv.is_null() { + // SAFETY: the caller's contract. + unsafe { argv.write(vector) }; + } + count + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn vector(command: &[u8], tail: &[u8]) -> Vec> { + let vector = Vector::of(command, tail).unwrap(); + (0..vector.len()) + .map(|index| vector.word(index).to_vec()) + .collect() + } + + #[test] + fn the_command_is_the_first_word_and_the_tail_follows_as_a_shell_splits_it() { + assert_eq!(vector(b"answer", b""), [b"answer".to_vec()]); + assert_eq!( + vector(b"copy", b" a.txt b.txt "), + [b"copy".to_vec(), b"a.txt".to_vec(), b"b.txt".to_vec()] + ); + assert_eq!( + vector(b"say", br#"-c 'two words' "a \"quote\"" back\ slash """#), + [ + b"say".to_vec(), + b"-c".to_vec(), + b"two words".to_vec(), + br#"a "quote""#.to_vec(), + b"back slash".to_vec(), + b"".to_vec(), + ] + ); + // A launcher that names nothing still gives `argv[0]`, empty. + assert_eq!(vector(b"", b"x"), [b"".to_vec(), b"x".to_vec()]); + } + + #[test] + fn every_word_is_its_own_terminated_string() { + let vector = Vector::of(b"p", b"ab c").unwrap(); + assert_eq!(vector.bytes, b"p\0ab\0c\0"); + assert_eq!(vector.starts, [0, 2, 5]); + assert_eq!(vector.len(), 3); + } + + #[test] + fn a_line_that_does_not_split_is_no_vector() { + assert_eq!( + Vector::of(b"p", b"'open"), + Err(Malformed::Words(WordsError::Unterminated { + quote: b'\'', + at: 0 + })) + ); + assert_eq!( + Vector::of(b"p", b"a \\"), + Err(Malformed::Words(WordsError::TrailingBackslash)) + ); + // A NUL would end the C string early: `a\0b` is not the word `a`. + assert_eq!(Vector::of(b"p", b"x a\0b"), Err(Malformed::Nul { word: 2 })); + assert_eq!(Vector::of(b"p\0q", b""), Err(Malformed::Nul { word: 0 })); + let mut text = alloc::string::String::new(); + core::fmt::Write::write_fmt(&mut text, format_args!("{}", Malformed::Nul { word: 2 })) + .unwrap(); + assert_eq!(text, "argument 2 holds a NUL byte"); + } +} diff --git a/crates/c/src/exit.rs b/crates/c/src/exit.rs new file mode 100644 index 0000000..027788b --- /dev/null +++ b/crates/c/src/exit.rs @@ -0,0 +1,114 @@ +//! The program's end request: one word any of the program's threads posts and +//! the host reads. The first request wins; later ones only end their thread. + +use core::ffi::c_int; +use core::sync::atomic::{AtomicU64, Ordering}; + +/// One end request. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) struct Exit { + /// The C status: the wire's `i32`, committed as given. + pub(crate) status: c_int, + /// `exit` rather than `_Exit` or a panic: finalizers may run. + pub(crate) finalize: bool, + /// The main thread asked, and is ending itself; the host waits for it + /// before the barrier, so a program that ran on it alone stops cleanly. + pub(crate) from_main: bool, +} + +const POSTED: u64 = 1 << 63; +const FINALIZE: u64 = 1 << 33; +const FROM_MAIN: u64 = 1 << 32; + +impl Exit { + fn word(self) -> u64 { + POSTED + | (u64::from(self.finalize) * FINALIZE) + | (u64::from(self.from_main) * FROM_MAIN) + | u64::from(self.status as u32) + } + + fn from_word(word: u64) -> Option { + (word & POSTED != 0).then_some(Self { + status: word as u32 as c_int, + finalize: word & FINALIZE != 0, + from_main: word & FROM_MAIN != 0, + }) + } +} + +/// Zero until the first request, then that request, for the program's life. +pub(crate) struct ExitWord(AtomicU64); + +impl ExitWord { + pub(crate) const fn new() -> Self { + Self(AtomicU64::new(0)) + } + + /// The request, once one was posted. + pub(crate) fn posted(&self) -> Option { + Exit::from_word(self.0.load(Ordering::Acquire)) + } + + /// Ask for `exit`. `false` when another request was first; the word keeps + /// that one. + pub(crate) fn post(&self, exit: Exit) -> bool { + self.0 + .compare_exchange(0, exit.word(), Ordering::AcqRel, Ordering::Acquire) + .is_ok() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_request_round_trips_every_status_and_flag() { + for status in [0, 1, 42, -1, c_int::MIN, c_int::MAX] { + for finalize in [false, true] { + for from_main in [false, true] { + let exit = Exit { + status, + finalize, + from_main, + }; + let word = ExitWord::new(); + assert_eq!(word.posted(), None); + assert!(word.post(exit)); + assert_eq!(word.posted(), Some(exit)); + } + } + } + } + + #[test] + fn the_first_request_wins() { + let word = ExitWord::new(); + let first = Exit { + status: 3, + finalize: true, + from_main: false, + }; + let second = Exit { + status: 7, + finalize: false, + from_main: true, + }; + assert!(word.post(first)); + assert!(!word.post(second)); + assert_eq!(word.posted(), Some(first)); + } + + #[test] + fn a_status_of_zero_is_still_a_request() { + let word = ExitWord::new(); + let exit = Exit { + status: 0, + finalize: false, + from_main: false, + }; + assert!(word.post(exit)); + assert_eq!(word.posted(), Some(exit)); + } +} diff --git a/crates/c/src/fd.rs b/crates/c/src/fd.rs new file mode 100644 index 0000000..91d271f --- /dev/null +++ b/crates/c/src/fd.rs @@ -0,0 +1,405 @@ +//! Descriptors: C's spelling for "a capability I hold". +//! +//! A program's authority arrives as capabilities at fixed places: its boot +//! block names each by a role and an index, and the capability itself sits in +//! the thread's arena. `open` is the readable way to say which one is meant. +//! It grants nothing and reaches no filesystem: +//! +//! - **A name is a row of one static table** ([`NAMES`]): a path-shaped +//! string, the bootstrap coordinate it stands for, and what kind of +//! capability lives there. A path that is no row is `ENOENT`. +//! - **A descriptor stores the row, never the capability.** Capabilities are +//! held per thread on Kore, and a thread is a task: `main` runs on one, a +//! `pthread` on another, and each has its own boot block. So every use of a +//! descriptor resolves the row's coordinate in the *calling* thread's boot +//! block. Descriptor numbers are the process's, as C expects, and no +//! authority crosses a thread through one: a thread that was not founded +//! holding the capability gets `ENXIO`, whoever opened the descriptor. +//! - **`close` forgets the number.** The capability was the program's before +//! `open` and is after `close`; nothing is revoked. +//! +//! Descriptors 0, 1 and 2 are the standard streams' numbers and are never +//! handed out; the streams themselves are ``'s. +//! +//! The table is plain data over atomics, so the host suite exercises the code +//! the target runs. Finding a row's capability in the boot block is the metal +//! half ([`holding`]). + +use core::ffi::c_int; +use core::sync::atomic::{AtomicU32, Ordering}; + +use kore_std::abi::task::role; + +/// `errno` values this module and [`crate::mmap`] report: musl's numbers, +/// which kore-c-shim's `errno` and the hosted `` share. +pub(crate) mod errno { + use core::ffi::c_int; + pub(crate) const ENOENT: c_int = 2; + pub(crate) const ENXIO: c_int = 6; + pub(crate) const EBADF: c_int = 9; + pub(crate) const EAGAIN: c_int = 11; + pub(crate) const ENOMEM: c_int = 12; + pub(crate) const EACCES: c_int = 13; + pub(crate) const EFAULT: c_int = 14; + pub(crate) const EINVAL: c_int = 22; + pub(crate) const EMFILE: c_int = 24; + pub(crate) const ENOTSUP: c_int = 95; +} + +/// What kind of capability a name stands for: which operations its +/// descriptor has. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum Kind { + /// The memory supply the program's allocator grows on. `mmap` takes a + /// region off it, through the runtime's own reservation so the two never + /// meet ([`crate::mmap`]). + Memory, +} + +/// One row: a name and the bootstrap coordinate it stands for. +#[derive(Debug)] +pub(crate) struct Name { + /// The path `open` matches, byte for byte. + pub(crate) path: &'static [u8], + /// The boot block role of the capability. + pub(crate) role: u16, + /// The instance within that role. + pub(crate) index: u16, + /// What lives there. + pub(crate) kind: Kind, +} + +/// **The names.** Each is a coordinate every thread that holds the +/// capability finds it under, so a row is true on whichever thread asks. +/// +/// `/kore/memory` is `role::HEAP@0`: the launch's memory supply, which +/// kore-rt endows every thread that may allocate with a share of +/// (`WorkerCap::heap`). +pub(crate) static NAMES: [Name; 1] = [Name { + path: b"/kore/memory", + role: role::HEAP, + index: 0, + kind: Kind::Memory, +}]; + +/// The row `path` names. +pub(crate) fn resolve(path: &[u8]) -> Option { + NAMES.iter().position(|name| name.path == path) +} + +/// What an `open` asked to do with the capability. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) struct Access { + pub(crate) read: bool, + pub(crate) write: bool, +} + +/// ``'s access modes, musl's values. They are the only flags `open` +/// takes: every other flag describes a file this namespace does not have. +pub(crate) const O_RDONLY: c_int = 0; +pub(crate) const O_WRONLY: c_int = 1; +pub(crate) const O_RDWR: c_int = 2; + +impl Access { + /// The access `flags` ask for, or `EINVAL` for any flag but an access mode. + pub(crate) fn of(flags: c_int) -> Result { + match flags { + O_RDONLY => Ok(Self { + read: true, + write: false, + }), + O_WRONLY => Ok(Self { + read: false, + write: true, + }), + O_RDWR => Ok(Self { + read: true, + write: true, + }), + _ => Err(errno::EINVAL), + } + } + + /// Whether a capability that can be read (`ACCESS`) and, if `writable`, + /// written (`MODIFY`) permits this access. + pub(crate) fn within(self, readable: bool, writable: bool) -> bool { + (!self.read || readable) && (!self.write || writable) + } + + const fn bits(self) -> u32 { + self.read as u32 | (self.write as u32) << 1 + } + + const fn from_bits(bits: u32) -> Self { + Self { + read: bits & 1 != 0, + write: bits & 2 != 0, + } + } +} + +/// The first number `open` hands out: 0, 1 and 2 are the standard streams'. +pub(crate) const FIRST: c_int = 3; + +/// How many descriptors a process may hold open. A descriptor is one word +/// naming a row, and the rows are few, so this bounds a program that forgets +/// to `close` rather than one that needs many: sixty-four entries are one +/// 256-byte table, scanned whole only by an `open` that finds it full. +pub(crate) const CAPACITY: usize = 64; + +/// A descriptor table: one word per descriptor, zero when free. +/// +/// A live word is [`LIVE`], the access in two bits, and the row's index. +/// Claiming and releasing are one compare-and-swap each, so threads opening +/// and closing at once never share a lock, and a descriptor is never handed +/// to two of them. +pub(crate) struct Table { + words: [AtomicU32; N], +} + +const LIVE: u32 = 1 << 31; +const ACCESS_SHIFT: u32 = 16; +const ROW_MASK: u32 = (1 << ACCESS_SHIFT) - 1; + +// Every row's index fits its field. +const _: () = assert!(NAMES.len() <= ROW_MASK as usize + 1); + +impl Table { + pub(crate) const fn new() -> Self { + Self { + words: [const { AtomicU32::new(0) }; N], + } + } + + /// Open a descriptor for `row` with `access`: the lowest free number, as + /// POSIX has it, or `EMFILE` when every one is taken. + pub(crate) fn claim(&self, row: usize, access: Access) -> Result { + let word = LIVE | access.bits() << ACCESS_SHIFT | row as u32; + for (at, slot) in self.words.iter().enumerate() { + if slot + .compare_exchange(0, word, Ordering::AcqRel, Ordering::Relaxed) + .is_ok() + { + return Ok(FIRST + at as c_int); + } + } + Err(errno::EMFILE) + } + + /// The row and access behind `fd`, or `EBADF`. + pub(crate) fn get(&self, fd: c_int) -> Result<(usize, Access), c_int> { + let word = self.slot(fd)?.load(Ordering::Acquire); + if word & LIVE == 0 { + return Err(errno::EBADF); + } + Ok(( + (word & ROW_MASK) as usize, + Access::from_bits(word >> ACCESS_SHIFT), + )) + } + + /// Close `fd`, or `EBADF` when it was not open. + pub(crate) fn release(&self, fd: c_int) -> Result<(), c_int> { + match self.slot(fd)?.swap(0, Ordering::AcqRel) { + 0 => Err(errno::EBADF), + _ => Ok(()), + } + } + + fn slot(&self, fd: c_int) -> Result<&AtomicU32, c_int> { + fd.checked_sub(FIRST) + .and_then(|at| usize::try_from(at).ok()) + .and_then(|at| self.words.get(at)) + .ok_or(errno::EBADF) + } +} + +/// The process's descriptors. +pub(crate) static TABLE: Table = Table::new(); + +/// What the calling thread holds at a row's coordinate. +#[derive(Clone, Copy, Debug)] +pub(crate) struct Holding { + /// The capability's arena slot, a name this thread alone may use. + pub(crate) slot: kore_std::abi::session::SlotRef, + /// It carries `ACCESS`. + pub(crate) readable: bool, + /// It carries `MODIFY`. + pub(crate) writable: bool, +} + +/// **The capability `name` stands for, as the calling thread holds it**: +/// its slot from this thread's boot block, and the rights its record +/// carries. `None` when the thread was not founded holding one of that kind +/// there. The record is the kernel's, read through the arena's read-only +/// alias, and every later effect binds the slot again in the kernel, so this +/// is a description and never a grant. +#[cfg(target_os = "none")] +pub(crate) fn holding(name: &Name) -> Option { + use kore_std::abi::cap::{ResourceKind, Rights}; + use kore_std::runtime::cap::verbs; + let slot = kore_std::runtime::entry::thread_context()? + .bootstrap() + .handle(name.role, name.index)?; + let kind = match name.kind { + Kind::Memory => ResourceKind::MemLease, + }; + let rights = verbs::capability_rights(slot, kind)?; + Some(Holding { + slot, + readable: rights.contains(Rights::ACCESS), + writable: rights.contains(Rights::MODIFY), + }) +} + +/// `open`: a descriptor for the capability `path` names, with `flags`' +/// access. +/// +/// `ENOENT`: no such name. `EINVAL`: a flag other than an access mode. +/// `ENXIO`: the calling thread holds no such capability. `EACCES`: it holds +/// one whose rights do not cover the access. `EMFILE`: no descriptor is free. +#[cfg(target_os = "none")] +pub(crate) fn open(path: &[u8], flags: c_int) -> Result { + let row = resolve(path).ok_or(errno::ENOENT)?; + let access = Access::of(flags)?; + let held = holding(&NAMES[row]).ok_or(errno::ENXIO)?; + if !access.within(held.readable, held.writable) { + return Err(errno::EACCES); + } + TABLE.claim(row, access) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_path_is_a_row_or_nothing() { + assert_eq!(resolve(b"/kore/memory"), Some(0)); + for path in [ + &b""[..], + b"/", + b"/kore", + b"/kore/memory/", + b"/kore/memory\0", + b"kore/memory", + b"/KORE/MEMORY", + b"/dev/zero", + ] { + assert_eq!(resolve(path), None, "{:?}", core::str::from_utf8(path)); + } + assert_eq!((NAMES[0].role, NAMES[0].index), (role::HEAP, 0)); + } + + #[test] + fn only_an_access_mode_is_a_flag() { + assert_eq!( + Access::of(O_RDONLY), + Ok(Access { + read: true, + write: false + }) + ); + assert_eq!( + Access::of(O_WRONLY), + Ok(Access { + read: false, + write: true + }) + ); + assert_eq!( + Access::of(O_RDWR), + Ok(Access { + read: true, + write: true + }) + ); + // The access modes are not bits: 3 is no mode, and neither is a mode + // with O_CREAT (0o100), O_TRUNC (0o1000) or O_CLOEXEC (0o2000000). + for flags in [3, O_RDWR | 0o100, O_WRONLY | 0o1000, 0o2000000, -1] { + assert_eq!(Access::of(flags), Err(errno::EINVAL), "{flags:#o}"); + } + } + + #[test] + fn access_is_held_to_the_capabilitys_rights() { + let read = Access::of(O_RDONLY).unwrap(); + let write = Access::of(O_WRONLY).unwrap(); + let both = Access::of(O_RDWR).unwrap(); + // (readable, writable) of the capability. + assert!(read.within(true, false) && read.within(true, true)); + assert!(!read.within(false, true)); + assert!(write.within(false, true) && !write.within(true, false)); + assert!(both.within(true, true)); + assert!(!both.within(true, false) && !both.within(false, true)); + } + + #[test] + fn the_lowest_free_number_is_handed_out_from_three() { + let table = Table::<4>::new(); + let both = Access::of(O_RDWR).unwrap(); + let read = Access::of(O_RDONLY).unwrap(); + assert_eq!(table.claim(0, both), Ok(3)); + assert_eq!(table.claim(0, read), Ok(4)); + assert_eq!(table.claim(0, both), Ok(5)); + assert_eq!(table.get(4), Ok((0, read))); + assert_eq!(table.release(4), Ok(())); + assert_eq!(table.get(4), Err(errno::EBADF)); + // The hole is filled before a higher number is used. + assert_eq!(table.claim(0, both), Ok(4)); + assert_eq!(table.claim(0, both), Ok(6)); + assert_eq!(table.claim(0, both), Err(errno::EMFILE)); + assert_eq!(table.get(6), Ok((0, both))); + } + + #[test] + fn a_number_that_is_not_open_is_ebadf() { + let table = Table::<4>::new(); + // The streams' numbers, a negative, one past the table, and a free one. + for fd in [0, 1, 2, -1, c_int::MIN, 7, c_int::MAX, 3] { + assert_eq!(table.get(fd), Err(errno::EBADF), "{fd}"); + assert_eq!(table.release(fd), Err(errno::EBADF), "{fd}"); + } + let fd = table.claim(0, Access::of(O_RDONLY).unwrap()).unwrap(); + assert_eq!(table.release(fd), Ok(())); + // A second close is of a descriptor that is no longer open. + assert_eq!(table.release(fd), Err(errno::EBADF)); + } + + #[test] + fn the_row_and_access_survive_the_word() { + let table = Table::<2>::new(); + let row = ROW_MASK as usize; + for flags in [O_RDONLY, O_WRONLY, O_RDWR] { + let access = Access::of(flags).unwrap(); + let fd = table.claim(row, access).unwrap(); + assert_eq!(table.get(fd), Ok((row, access))); + table.release(fd).unwrap(); + } + } + + #[test] + fn threads_never_share_a_descriptor() { + use std::collections::HashSet; + use std::sync::Arc; + let table = Arc::new(Table::<64>::new()); + let access = Access::of(O_RDWR).unwrap(); + let claims: Vec<_> = (0..8) + .map(|_| { + let table = Arc::clone(&table); + std::thread::spawn(move || { + (0..8) + .map(|_| table.claim(0, access).unwrap()) + .collect::>() + }) + }) + .collect(); + let all: Vec = claims + .into_iter() + .flat_map(|thread| thread.join().unwrap()) + .collect(); + assert_eq!(all.len(), 64); + assert_eq!(all.iter().copied().collect::>().len(), 64); + assert_eq!(table.claim(0, access), Err(errno::EMFILE)); + } +} diff --git a/crates/c/src/lib.rs b/crates/c/src/lib.rs new file mode 100644 index 0000000..11eced7 --- /dev/null +++ b/crates/c/src/lib.rs @@ -0,0 +1,82 @@ +//! The runtime of `libkore.so`: libkore's C ABI as one shared object, and the +//! host of an independent C program on Kore. +//! +//! A C program a compiler wrote on Kore (`tcc` at the shell) links +//! `libkore.so` with `-lkore` and includes `kore.h`. From one image the whole +//! system shares, it then has: +//! +//! - `kore.h`'s functions (`kore-ffi`), and the standard C names that +//! `kore/stdlib.h` and `kore/pthread.h` alias, as real symbols (`malloc` +//! to `free`, `pthread_create` to `pthread_exit`), over the process heap this +//! library installs; +//! - a host: the program's `main` runs on a main thread with a C stack, while +//! thread 0 serves its filesystem session ([`api::kore_fs_run`], declared +//! in `kore/program.h`) and ends the process when any thread exits. +//! +//! # The shared object +//! +//! A distribution builds the image from a `cdylib` crate whose `[lib]` is +//! named `kore` and whose source names this crate (`use kore_c as _;`); Kore's +//! is `user/runtime/libkore`. Its link carries the contract consumers rely on, +//! which `tools/check-program.py` checks for every ISA: +//! +//! - `--soname=libkore.so`, the name consumers record as `DT_NEEDED`; +//! - `--no-undefined`: the library binds nothing from the program it links into; +//! - a shared object with no entry point, so it registers as a library; +//! - static initial-exec TLS only, which the Kore user targets select; +//! - exactly the declared ABI exported (`kore.h`, `kore/program.h`, the C names +//! `kore/stdlib.h` and `kore/pthread.h` alias, and the C library's two +//! lifecycle entry points). +//! +//! It is a C program's runtime, never a Rust program's: a Rust executable links +//! `kore-std` with its own entry, and two runtimes in one process would grow one +//! heap supply behind two cursors. +#![cfg_attr(target_os = "none", no_std)] +#![cfg_attr(target_os = "none", feature(thread_local, c_variadic))] +#![deny(unsafe_op_in_unsafe_fn)] + +// The C ABI's exports are this image's reason to exist; naming the crate keeps +// them in the link. +use kore_ffi as _; +// Kore's C library, exported whole from this image. +#[cfg(target_os = "none")] +use kore_c_shim as _; + +pub mod api; +// Off Kore only the host suite reads the vector's storage: the page it is +// read from and the hook that answers it are the Kore program's alone. +#[cfg(any(test, target_os = "none"))] +#[cfg_attr(not(target_os = "none"), allow(dead_code))] +mod args; +#[cfg(any(test, target_os = "none"))] +mod exit; +// Off Kore only `kore_fd_describe` and the host suite read these two: the +// names, the verbs and most refusals are the Kore program's alone. +#[cfg_attr(not(target_os = "none"), allow(dead_code))] +mod fd; +#[cfg(target_os = "none")] +mod lock; +#[cfg(any(test, target_os = "none"))] +#[cfg_attr(not(target_os = "none"), allow(dead_code))] +mod mmap; +#[cfg(target_os = "none")] +mod posix; +#[cfg(target_os = "none")] +mod program; +#[cfg(any(test, target_os = "none"))] +mod stdio; + +/// The process heap over the founder's `role::HEAP` supply, grown on demand. +/// The program's allocations (`malloc`, `kore_alloc_*`) and this library's +/// own draw from it; with no supply every allocation answers null. +#[cfg(target_os = "none")] +#[global_allocator] +static PROCESS_HEAP: kore_std::runtime::heap::Heap = + kore_std::runtime::heap::Heap::new_in(kore_std::runtime::heap::HeapSupplySource); + +/// A panic in this library ends the program, recorded; never a silent park. +#[cfg(target_os = "none")] +#[panic_handler] +fn panic(info: &core::panic::PanicInfo<'_>) -> ! { + program::panicked(info) +} diff --git a/crates/c/src/lock.rs b/crates/c/src/lock.rs new file mode 100644 index 0000000..b54a2f1 --- /dev/null +++ b/crates/c/src/lock.rs @@ -0,0 +1,54 @@ +//! The futex mutex this runtime's threads take turns with: the streams' +//! lock, and the callers of thread 0's memory cell. +//! +//! The classic three states. A waiter's park is bounded by the runtime's +//! slice, because a holder's wake may not reach it: a thread holds a send +//! into a peer's doorbell only where the thread controller relays one. + +use core::sync::atomic::{AtomicU32, Ordering}; +use core::time::Duration; + +use kore_std::runtime::futex::{self, WaitOutcome}; +use kore_std::runtime::pace; + +const FREE: u32 = 0; +const HELD: u32 = 1; +const CONTENDED: u32 = 2; + +/// How long a waiter parks before it tries again: the runtime's slice for a +/// waiter its waker cannot reach (the thread controller's `UNREACHED_SLICE`). +const SLICE: Duration = Duration::from_nanos(pace::CEDE_NS); + +/// A mutex over one futex word. Not reentrant: a holder that needs to be +/// counts its own depth ([`crate::stdio`]). +pub(crate) struct Lock(AtomicU32); + +impl Lock { + pub(crate) const fn new() -> Self { + Self(AtomicU32::new(FREE)) + } + + pub(crate) fn acquire(&self) { + if self + .0 + .compare_exchange(FREE, HELD, Ordering::Acquire, Ordering::Relaxed) + .is_ok() + { + return; + } + while self.0.swap(CONTENDED, Ordering::Acquire) != FREE { + // Bounded: the holder's wake may not reach this thread. + if futex::wait(&self.0, CONTENDED, Some(SLICE)) == WaitOutcome::Unsupported { + // No doorbell to park on: the deadline park any thread can. + let _ = pace::cede(); + } + } + } + + pub(crate) fn release(&self) { + // SeqCst, as the futex contract asks of the store before a wake. + if self.0.swap(FREE, Ordering::SeqCst) == CONTENDED { + futex::wake(&self.0); + } + } +} diff --git a/crates/c/src/mmap.rs b/crates/c/src/mmap.rs new file mode 100644 index 0000000..cebe56a --- /dev/null +++ b/crates/c/src/mmap.rs @@ -0,0 +1,684 @@ +//! `mmap` and `munmap` over a memory capability. +//! +//! `mmap(NULL, len, prot, flags, fd, 0)` on a descriptor for the program's +//! memory supply (`/kore/memory`, [`crate::fd`]) answers `len` bytes of fresh, +//! zeroed memory, as mapping `/dev/zero` does on a POSIX system. Underneath +//! it is three of kore-rt's verbs, the sequence a Wasmtime engine's guest +//! memory already takes (`kore-wasm-platform`): +//! +//! 1. **Carve.** `heap::reserve_from_top` takes a lease off the top of the +//! supply the allocator grows from the bottom of. The two share cursors, so +//! a region and `malloc`'s extents never meet, which a raw carve at an +//! offset of the same capability could not promise. +//! 2. **Place.** The address comes from the free part of the task's link +//! window (`Rt::link_free_span`), from one process-wide cursor. The caller +//! never picks it: a hint is ignored and `MAP_FIXED` is refused. +//! 3. **Map.** The whole lease is mapped there. +//! +//! Thread 0 does all three, for whichever thread asked. Every thread that +//! may allocate is founded holding a share of the supply, and `open` and +//! `mmap` hold the caller's access to that share's rights. But a share +//! carries no `REVOKE` (the founder keeps it, to tear the share down), and +//! a region that can be unmapped again is a carve its holder may revoke. The +//! supply's root is thread 0's, so the caller posts its request there and +//! waits, as a filesystem session does. A mapping is therefore the +//! process's: any thread may unmap it. +//! +//! # What `munmap` returns +//! +//! The supply's cursors and the window's cursor only advance: bytes carved +//! and addresses placed are spent for the life of the program. So `munmap` +//! removes the mapping, and a stale access faults, but it *parks* the region: +//! its lease and its address stay with the program, and the next `mmap` of the +//! same length takes a parked region, zeroed again, before anything new is +//! carved. A program that maps and unmaps one size in a loop therefore uses +//! one region. One that maps ever-different sizes spends supply each time, +//! and ends with `ENOMEM`: the table holds [`REGIONS`] regions, and when all +//! are taken a parked one of another length is given up for good. +//! +//! The request rules and the region table are plain data, so the host suite +//! exercises the code the target runs; the verbs are the metal half. + +use core::ffi::c_int; + +use crate::fd::errno; + +/// The mapping granule: kore's user targets pin a 4 KiB page +/// (`max-page-size=4096`), which is also the granule a supply is carved in +/// (`kore_rt::mem_source::ALIGN`), so a region is exactly its pages. +pub(crate) const PAGE: usize = 4096; + +/// ``'s protections, musl's values. +pub(crate) const PROT_READ: c_int = 1; +pub(crate) const PROT_WRITE: c_int = 2; +/// ``'s flags, musl's values. +pub(crate) const MAP_SHARED: c_int = 0x01; +pub(crate) const MAP_PRIVATE: c_int = 0x02; +pub(crate) const MAP_FIXED: c_int = 0x10; + +/// A request `mmap` can serve. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) struct Request { + /// Whether the mapping is writable. + pub(crate) write: bool, + /// The length asked for, in whole pages' bytes. + pub(crate) asked: usize, +} + +/// The whole pages `len` bytes take, or `None` when that overflows. +pub(crate) fn pages(len: usize) -> Option { + len.checked_add(PAGE - 1).map(|bytes| bytes & !(PAGE - 1)) +} + +impl Request { + /// What `mmap`'s arguments ask for, or the `errno` that refuses them. + /// + /// `EINVAL`: no length, an offset (a supply has no positions a caller can + /// name), neither or both of `MAP_SHARED` and `MAP_PRIVATE`, or a flag + /// with no meaning here. `ENOTSUP`: `MAP_FIXED` (the kernel's side picks + /// every address), and any protection but read or read-write: memory a + /// program wrote is never executable, and a mapping nothing can touch is + /// not one this API reserves. `ENOMEM`: a length that does not round to + /// pages. + /// + /// `MAP_SHARED` and `MAP_PRIVATE` are both accepted and mean the same: + /// the region is fresh and this program's alone, so there is no other + /// mapper for a store to reach or to be kept from. `PROT_WRITE` alone is + /// read-write, which POSIX permits and every Kore ISA's page tables imply. + pub(crate) fn of(len: usize, prot: c_int, flags: c_int, offset: i64) -> Result { + if len == 0 || offset != 0 { + return Err(errno::EINVAL); + } + if flags & MAP_FIXED != 0 { + return Err(errno::ENOTSUP); + } + if flags != MAP_SHARED && flags != MAP_PRIVATE { + return Err(errno::EINVAL); + } + let write = match prot { + PROT_READ => false, + p if p == PROT_WRITE || p == PROT_READ | PROT_WRITE => true, + _ => return Err(errno::ENOTSUP), + }; + let asked = pages(len).ok_or(errno::ENOMEM)?; + Ok(Self { write, asked }) + } +} + +/// How many regions a program may hold, mapped and parked together. A region +/// is a lease and a view in thread 0's arena and a row here; sixteen is the +/// table a Wasmtime engine keeps for the same purpose +/// (`kore_wasm_platform::MAX_MAPPINGS`), small enough to scan, and a program +/// that needs more distinct regions than that at once is better served by +/// one larger mapping it divides itself. +pub(crate) const REGIONS: usize = 16; + +/// One region carved for the program: mapped, or parked for the next `mmap` +/// of its length. `L` is its lease and `V` its view, capabilities dropped +/// with the row. +pub(crate) enum Region { + Vacant, + /// Mapped at `base` for a request of `asked` bytes. + Mapped { + lease: L, + view: V, + base: usize, + asked: usize, + }, + /// Unmapped, keeping its lease and the address it had. + Parked { + lease: L, + base: usize, + /// The lease's own length, which a map must name exactly. + len: usize, + asked: usize, + }, +} + +/// A program's regions. +pub(crate) struct Regions { + rows: [Region; N], +} + +impl Regions { + pub(crate) const fn new() -> Self { + Self { + rows: [const { Region::Vacant }; N], + } + } + + /// A row nothing occupies. + pub(crate) fn vacant(&self) -> Option { + self.rows + .iter() + .position(|row| matches!(row, Region::Vacant)) + } + + /// A parked region that served a request of exactly `asked` bytes. + pub(crate) fn parked(&self, asked: usize) -> Option { + self.rows + .iter() + .position(|row| matches!(row, Region::Parked { asked: was, .. } if *was == asked)) + } + + /// The region mapped at exactly `base`. + pub(crate) fn mapped(&self, base: usize) -> Option { + self.rows + .iter() + .position(|row| matches!(row, Region::Mapped { base: at, .. } if *at == base)) + } + + /// Any parked region: what a full table gives up for a new one. + pub(crate) fn any_parked(&self) -> Option { + self.rows + .iter() + .position(|row| matches!(row, Region::Parked { .. })) + } + + /// Take row `at`, leaving it vacant. + pub(crate) fn take(&mut self, at: usize) -> Region { + core::mem::replace(&mut self.rows[at], Region::Vacant) + } + + /// Fill row `at`. + pub(crate) fn put(&mut self, at: usize, region: Region) { + self.rows[at] = region; + } +} + +#[cfg(target_os = "none")] +pub(crate) use metal::{arm, map, posted, serve, unmap}; + +#[cfg(target_os = "none")] +mod metal { + use core::cell::UnsafeCell; + use core::ffi::c_int; + use core::sync::atomic::{AtomicU32, Ordering}; + use core::time::Duration; + + use kore_ld::place::SharedPlacer; + use kore_std::runtime::cap::{MappedLeaseView, RetireOutcome, StashedLease, verbs}; + use kore_std::runtime::entry::Rt; + use kore_std::runtime::futex::{self, WaitOutcome}; + use kore_std::runtime::{pace, thread}; + + use super::{PAGE, REGIONS, Region, Regions, Request}; + use crate::fd::{Holding, errno}; + use crate::lock::Lock; + + /// Where regions land: the free part of the task's link window, armed + /// once by the host before any other thread exists. Unarmed (a program + /// with no host, or no link window), every `mmap` is `ENOMEM`. + static WINDOW: SharedPlacer = SharedPlacer::unarmed(); + + /// The process's regions, which thread 0 alone touches: the leases and + /// views in it are capabilities of its arena. + struct HostOnly(UnsafeCell); + // SAFETY: only thread 0 forms a reference ([`perform`]'s contract). + unsafe impl Sync for HostOnly {} + static REGIONS_HELD: HostOnly> = + HostOnly(UnsafeCell::new(Regions::new())); + + /// Publish the window regions are placed in: `rt`'s free link span + /// (thread 0's, which is the process's). A task without one leaves the + /// placer unarmed. + pub(crate) fn arm(rt: &Rt) { + if let Ok(span) = rt.link_free_span() { + let _ = WINDOW.arm(span.free, span.limit - span.free); + } + } + + /// Map `lease` whole at `base`, zeroed, with the request's access. + /// + /// The lease is mapped writable to be zeroed whatever was asked: a parked + /// region holds what its last user left, and a fresh one is zeroed too + /// rather than trusted to be. A read-only request is then mapped again + /// without write. + fn map_zeroed( + lease: &StashedLease, + base: usize, + len: usize, + write: bool, + ) -> Option { + let slot = lease.slot()?; + let mut view = verbs::map_owned_lease_at_rw(slot, base as u64, len as u64)?; + // SAFETY: `view` maps `len` writable bytes at its pointer, which + // nothing else has been handed yet. + unsafe { view.as_mut_ptr().write_bytes(0, len) }; + if write { + return Some(view); + } + match view.try_retire_outcome() { + RetireOutcome::Retired | RetireOutcome::AlreadyDead => {} + // Still mapped writable: refusing here would strand the view, so + // the caller gets nothing and the row keeps the lease. + RetireOutcome::Retryable => return None, + } + verbs::map_owned_lease_at(slot, base as u64, len as u64) + } + + /// One request of thread 0. + #[derive(Clone, Copy)] + enum Op { + Map(Request), + Unmap { base: usize, len: usize }, + } + + /// Carry `op` out. Thread 0 only: the supply's root, with the `REVOKE` + /// an unmappable carve needs, is a capability of its arena. + fn perform(op: Op) -> Result { + // SAFETY: thread 0 is the one caller, and it is not re-entered: no + // verb below calls back into this module. + let regions = unsafe { &mut *REGIONS_HELD.0.get() }; + match op { + Op::Map(request) => map_here(regions, request), + Op::Unmap { base, len } => unmap_here(regions, base, len).map(|()| 0), + } + } + + fn map_here( + regions: &mut Regions, + request: Request, + ) -> Result { + let (row, lease, base, len) = match regions.parked(request.asked) { + Some(row) => match regions.take(row) { + Region::Parked { + lease, base, len, .. + } => (row, lease, base, len), + // `parked` answered this row. + _ => return Err(errno::ENOMEM), + }, + None => { + // The row first: a carve with nowhere to be recorded would + // spend supply for nothing. A full table gives up a parked + // region of another length, whose lease ends with its row. + let row = match regions.vacant() { + Some(row) => row, + None => { + let row = regions.any_parked().ok_or(errno::ENOMEM)?; + drop(regions.take(row)); + row + } + }; + let lease = kore_std::runtime::heap::reserve_from_top(request.asked as u64) + .ok_or(errno::ENOMEM)?; + let len = lease + .slot() + .and_then(verbs::lease_geometry) + .and_then(|(_, len)| usize::try_from(len).ok()) + .ok_or(errno::ENOMEM)?; + let base = WINDOW + .choose(len as u64, PAGE as u64) + .ok() + .and_then(|base| usize::try_from(base).ok()) + .ok_or(errno::ENOMEM)?; + (row, lease, base, len) + } + }; + match map_zeroed(&lease, base, len, request.write) { + Some(view) => { + regions.put( + row, + Region::Mapped { + lease, + view, + base, + asked: request.asked, + }, + ); + Ok(base) + } + None => { + regions.put( + row, + Region::Parked { + lease, + base, + len, + asked: request.asked, + }, + ); + Err(errno::ENOMEM) + } + } + } + + fn unmap_here( + regions: &mut Regions, + base: usize, + len: usize, + ) -> Result<(), c_int> { + let asked = super::pages(len).filter(|&bytes| bytes != 0); + let row = regions.mapped(base).ok_or(errno::EINVAL)?; + let Region::Mapped { + lease, + mut view, + base, + asked: was, + } = regions.take(row) + else { + // `mapped` answered this row. + return Err(errno::EINVAL); + }; + if asked != Some(was) { + regions.put( + row, + Region::Mapped { + lease, + view, + base, + asked: was, + }, + ); + return Err(errno::EINVAL); + } + let len = view.len(); + match view.try_retire_outcome() { + RetireOutcome::Retired | RetireOutcome::AlreadyDead => { + regions.put( + row, + Region::Parked { + lease, + base, + len, + asked: was, + }, + ); + Ok(()) + } + RetireOutcome::Retryable => { + regions.put( + row, + Region::Mapped { + lease, + view, + base, + asked: was, + }, + ); + Err(errno::EAGAIN) + } + } + } + + // ── Reaching thread 0 ─────────────────────────────────────────────── + + // The cell's states. A caller owns it at IDLE (under [`CALLERS`]) and at + // SERVED, thread 0 at POSTED. + const IDLE: u32 = 0; + const POSTED: u32 = 1; + const SERVED: u32 = 2; + + /// One request at a time: the caller holding [`CALLERS`] writes `op` and + /// posts, thread 0 writes `answer` and serves. + struct Call { + state: AtomicU32, + op: UnsafeCell, + answer: UnsafeCell>, + } + // SAFETY: `op` and `answer` are touched only by the side `state` hands + // the cell to. + unsafe impl Sync for Call {} + static CALL: Call = Call { + state: AtomicU32::new(IDLE), + op: UnsafeCell::new(Op::Unmap { base: 0, len: 0 }), + answer: UnsafeCell::new(Ok(0)), + }; + + /// Callers take turns at the cell. + static CALLERS: Lock = Lock::new(); + + /// How long a caller waits before it re-reads the cell: thread 0 kicks + /// the main thread when it has answered, and holds no send into a + /// pthread's doorbell, which rides this slice (the streams' rule). + const SLICE: Duration = Duration::from_nanos(pace::CEDE_NS); + + /// Whether a request waits for thread 0. + pub(crate) fn posted() -> bool { + CALL.state.load(Ordering::Acquire) == POSTED + } + + /// Thread 0: answer the posted request, if there is one. `true` when one + /// was answered, and its caller should be woken. + pub(crate) fn serve() -> bool { + if !posted() { + return false; + } + // SAFETY: POSTED hands the cell to thread 0 until it stores SERVED. + let answer = perform(unsafe { *CALL.op.get() }); + // SAFETY: as above. + unsafe { *CALL.answer.get() = answer }; + // SeqCst, as the futex contract asks of the store before a wake. + CALL.state.store(SERVED, Ordering::SeqCst); + futex::wake(&CALL.state); + true + } + + /// Have thread 0 carry `op` out, and wait for its answer. + /// + /// Thread 0 itself (a filesystem callback, a program with no host) does + /// it in place. Any other thread of a program without a host has no one + /// to ask: `ENOMEM`. + fn ask(op: Op) -> Result { + if crate::program::on_host() { + return perform(op); + } + if !crate::program::hosted() { + return Err(errno::ENOMEM); + } + CALLERS.acquire(); + // SAFETY: IDLE under the lock hands the cell to this caller. + unsafe { *CALL.op.get() = op }; + CALL.state.store(POSTED, Ordering::SeqCst); + // Refused only when no post stood, and then thread 0 re-reads the + // cell before it parks. + let _ = thread::kick_parent(); + loop { + match CALL.state.load(Ordering::Acquire) { + SERVED => break, + state => { + if futex::wait(&CALL.state, state, Some(SLICE)) == WaitOutcome::Unsupported { + // No doorbell to park on: the deadline park any + // thread can. + let _ = pace::cede(); + } + } + } + } + // SAFETY: SERVED hands the cell back to this caller. + let answer = unsafe { *CALL.answer.get() }; + CALL.state.store(IDLE, Ordering::Release); + CALLERS.release(); + answer + } + + /// `mmap` on a memory supply the calling thread holds as `held`. + /// + /// The caller's own share is what `open` and this call hold the access + /// to. The region is carved by thread 0, which holds the supply's root: + /// a share a thread is founded with carries no `REVOKE`, and a carve + /// that can be unmapped again needs it. + /// + /// `EACCES`: a writable mapping of a capability without `MODIFY`. + /// `ENOMEM`: every row of the table is a live mapping, the supply or the + /// window has no room, the kernel refused the map, or the program has no + /// host to carve for this thread. A region whose map is refused is + /// parked, so its supply bytes and address serve the next request of its + /// length. + pub(crate) fn map(held: Holding, request: Request) -> Result<*mut u8, c_int> { + if request.write && !held.writable { + return Err(errno::EACCES); + } + ask(Op::Map(request)).map(|base| base as *mut u8) + } + + /// `munmap`: remove the mapping at `base`, which `len` must name whole, + /// and park its region. + /// + /// `EINVAL`: the program has no mapping at exactly `base`, or `len` is + /// not that mapping's length. `EAGAIN`: the kernel did not confirm the + /// unmap; the mapping stands and the call may be made again. + pub(crate) fn unmap(base: usize, len: usize) -> Result<(), c_int> { + ask(Op::Unmap { base, len }).map(|_| ()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + const RW: c_int = PROT_READ | PROT_WRITE; + + #[test] + fn a_length_is_whole_pages() { + assert_eq!(pages(1), Some(PAGE)); + assert_eq!(pages(PAGE), Some(PAGE)); + assert_eq!(pages(PAGE + 1), Some(2 * PAGE)); + assert_eq!(pages(usize::MAX), None); + assert_eq!(pages(usize::MAX - PAGE + 2), None); + assert_eq!(pages(usize::MAX - PAGE + 1), Some(usize::MAX - PAGE + 1)); + } + + #[test] + fn a_readable_region_of_the_supply_is_served() { + assert_eq!( + Request::of(1 << 20, RW, MAP_PRIVATE, 0), + Ok(Request { + write: true, + asked: 1 << 20 + }) + ); + assert_eq!( + Request::of(1, PROT_READ, MAP_SHARED, 0), + Ok(Request { + write: false, + asked: PAGE + }) + ); + // Write alone is read-write. + assert_eq!( + Request::of(PAGE, PROT_WRITE, MAP_SHARED, 0), + Ok(Request { + write: true, + asked: PAGE + }) + ); + } + + #[test] + fn what_has_no_meaning_here_is_refused_by_name() { + // No length, and an offset into a supply. + assert_eq!(Request::of(0, RW, MAP_PRIVATE, 0), Err(errno::EINVAL)); + assert_eq!( + Request::of(PAGE, RW, MAP_PRIVATE, PAGE as i64), + Err(errno::EINVAL) + ); + assert_eq!(Request::of(PAGE, RW, MAP_PRIVATE, -1), Err(errno::EINVAL)); + // Neither, both, or a stranger: MAP_ANONYMOUS is 0x20. + assert_eq!(Request::of(PAGE, RW, 0, 0), Err(errno::EINVAL)); + assert_eq!( + Request::of(PAGE, RW, MAP_SHARED | MAP_PRIVATE, 0), + Err(errno::EINVAL) + ); + assert_eq!( + Request::of(PAGE, RW, MAP_PRIVATE | 0x20, 0), + Err(errno::EINVAL) + ); + // The caller never picks the address. + assert_eq!( + Request::of(PAGE, RW, MAP_PRIVATE | MAP_FIXED, 0), + Err(errno::ENOTSUP) + ); + // No access, executable (4), and bits past the three. + for prot in [0, 4, PROT_READ | 4, RW | 4, 8, -1] { + assert_eq!( + Request::of(PAGE, prot, MAP_PRIVATE, 0), + Err(errno::ENOTSUP), + "{prot}" + ); + } + assert_eq!( + Request::of(usize::MAX, RW, MAP_PRIVATE, 0), + Err(errno::ENOMEM) + ); + } + + type Rows = Regions; + + fn mapped(lease: u32, base: usize, asked: usize) -> Region { + Region::Mapped { + lease, + view: lease, + base, + asked, + } + } + + fn park(rows: &mut Rows, base: usize) { + let row = rows.mapped(base).unwrap(); + let Region::Mapped { + lease, base, asked, .. + } = rows.take(row) + else { + unreachable!() + }; + rows.put( + row, + Region::Parked { + lease, + base, + len: asked, + asked, + }, + ); + } + + #[test] + fn an_unmapped_region_serves_the_next_request_of_its_length() { + let mut rows = Rows::new(); + assert_eq!(rows.vacant(), Some(0)); + rows.put(0, mapped(7, 0x1000, PAGE)); + rows.put(1, mapped(8, 0x9000, 2 * PAGE)); + assert_eq!(rows.vacant(), Some(2)); + assert_eq!((rows.parked(PAGE), rows.any_parked()), (None, None)); + park(&mut rows, 0x1000); + // Parked, it is no longer a mapping, and only its own length takes it. + assert_eq!(rows.mapped(0x1000), None); + assert_eq!(rows.parked(2 * PAGE), None); + assert_eq!(rows.parked(PAGE), Some(0)); + let Region::Parked { lease, base, .. } = rows.take(0) else { + panic!("the parked row") + }; + // The same lease at the same address. + assert_eq!((lease, base), (7, 0x1000)); + assert_eq!(rows.vacant(), Some(0)); + } + + #[test] + fn a_mapping_is_found_by_its_base_alone() { + let mut rows = Rows::new(); + rows.put(1, mapped(1, 0x4000, 4 * PAGE)); + assert_eq!(rows.mapped(0x4000), Some(1)); + // An address inside it is not its base. + assert_eq!(rows.mapped(0x5000), None); + assert_eq!(rows.mapped(0), None); + } + + #[test] + fn a_full_table_has_no_vacant_row() { + let mut rows = Rows::new(); + for at in 0..3 { + rows.put(at, mapped(at as u32, 0x1000 * (at + 1), PAGE)); + } + assert_eq!(rows.vacant(), None); + park(&mut rows, 0x2000); + // A parked row is still occupied: it holds a lease. It is what a + // request of another length takes the place of. + assert_eq!(rows.vacant(), None); + assert_eq!(rows.parked(PAGE), Some(1)); + assert_eq!(rows.parked(2 * PAGE), None); + assert_eq!(rows.any_parked(), Some(1)); + assert!(matches!(rows.take(1), Region::Parked { lease: 1, .. })); + assert_eq!((rows.vacant(), rows.any_parked()), (Some(1), None)); + } +} diff --git a/crates/c/src/posix.rs b/crates/c/src/posix.rs new file mode 100644 index 0000000..49da7bc --- /dev/null +++ b/crates/c/src/posix.rs @@ -0,0 +1,94 @@ +//! The POSIX names over [`crate::fd`] and [`crate::mmap`]: `open`, `close`, +//! `mmap` and `munmap`, as ``, `` and `` +//! declare them. Each sets `errno` and answers `-1` (`MAP_FAILED` for `mmap`) +//! when it refuses. +//! +//! Exported on Kore alone, like the rest of the C library: on a host these +//! names are the host libc's. + +use core::ffi::{CStr, c_char, c_int, c_void}; + +use kore_c_shim::errno::set as set_errno; + +use crate::fd::{self, Kind, NAMES, TABLE, errno}; +use crate::mmap::{self, Request}; + +/// `mmap`'s refusal: `MAP_FAILED`, `(void *)-1`. +const MAP_FAILED: *mut c_void = usize::MAX as *mut c_void; + +fn refused(code: c_int) -> c_int { + set_errno(code); + -1 +} + +/// `open(path, flags)`: a descriptor for the capability `path` names +/// ([`fd::open`]). The mode a variadic `open` may carry belongs to `O_CREAT`, +/// which no name here takes, so it is never read. +/// +/// # Safety +/// `path` is null or a NUL-terminated string. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn open(path: *const c_char, flags: c_int, _mode: ...) -> c_int { + if path.is_null() { + return refused(errno::EFAULT); + } + // SAFETY: the caller's contract: a NUL-terminated string. + let path = unsafe { CStr::from_ptr(path) }.to_bytes(); + fd::open(path, flags).unwrap_or_else(refused) +} + +/// `close(fd)`: forget the descriptor. The capability it named stays the +/// program's, and mappings made through it stand. +#[unsafe(no_mangle)] +pub extern "C" fn close(fd: c_int) -> c_int { + match TABLE.release(fd) { + Ok(()) => 0, + Err(code) => refused(code), + } +} + +/// `mmap(addr, len, prot, flags, fd, offset)` ([`mmap`]'s module docs). +/// `addr` is a hint POSIX lets an implementation ignore, and it is. +/// +/// `EBADF`: `fd` is not open. `EACCES`: it was not opened for reading, or a +/// writable mapping was asked of one not opened for writing. `ENXIO`: the +/// calling thread does not hold the capability. Otherwise as +/// [`Request::of`] and [`mmap::map`] refuse. +#[unsafe(no_mangle)] +pub extern "C" fn mmap( + _addr: *mut c_void, + len: usize, + prot: c_int, + flags: c_int, + fd: c_int, + offset: i64, +) -> *mut c_void { + let mapped = TABLE.get(fd).and_then(|(row, access)| { + let request = Request::of(len, prot, flags, offset)?; + if !access.read || (request.write && !access.write) { + return Err(errno::EACCES); + } + let name = &NAMES[row]; + let held = fd::holding(name).ok_or(errno::ENXIO)?; + match name.kind { + Kind::Memory => mmap::map(held, request), + } + }); + match mapped { + Ok(base) => base.cast(), + Err(code) => { + set_errno(code); + MAP_FAILED + } + } +} + +/// `munmap(addr, len)`: unmap the program's mapping at `addr`, whole +/// ([`mmap::unmap`]). +#[unsafe(no_mangle)] +pub extern "C" fn munmap(addr: *mut c_void, len: usize) -> c_int { + match mmap::unmap(addr.addr(), len) { + Ok(()) => 0, + Err(code) => refused(code), + } +} diff --git a/crates/c/src/program.rs b/crates/c/src/program.rs new file mode 100644 index 0000000..657040d --- /dev/null +++ b/crates/c/src/program.rs @@ -0,0 +1,679 @@ +//! The host of an independent C program: `libkore.so`'s half of the program +//! lifecycle Kore's C library starts and ends. +//! +//! A C program on Kore links its C library's start files whatever else it +//! links. Their `_start` hands the program to the C library, which, when the +//! program also links this library, hands it on to [`__kore_program_start`]. +//! From there: +//! +//! - **The program runs on its main thread**, a worker with a C stack of +//! [`KORE_MAIN_STACK_BYTES`](crate::api::KORE_MAIN_STACK_BYTES), not on +//! thread 0. Thread 0's stack is the +//! founding's, sized for a runtime, and thread 0 is the process owner: the +//! one thread that may stop the others. +//! - **Thread 0 is the host.** It keeps the task's executor and its service +//! faces, and serves what the program's threads ask of it through one +//! request cell and one word, the shape `kore-lua`'s VM bridge and `tcc` +//! share: the program's filesystem session ([`kore_fs_run`](crate::api::kore_fs_run)) +//! and its end. +//! - **Any thread can end the program.** `exit` and `_Exit` reach +//! [`__kore_program_exit`] once the C library has run its `atexit` +//! handlers; the first request wins. The host stops every other thread, +//! runs the finalizers when none was still running, and ends the process +//! with the C status through the C library's own exit. +//! +//! ```text +//! thread 0 (the host) main thread (C) +//! ─────────────────── ─────────────── +//! found the main thread ────────────────────▶ program(): main(), then exit +//! park on the wake channel … kore_fs_run(step, user) +//! ◀──────── SESSION_POSTED, kick_parent ───── futex::wait on the word +//! run the session (its C callbacks run here) +//! SESSION_SERVED, kick ─────────────────────▶ returns the session's status +//! … exit(status): atexit handlers +//! ◀──────── the exit word, kick_parent ────── the thread ends +//! join it, raise the barrier, finalize +//! end(status): the C library's exit +//! ``` +//! +//! Every park is a doorbell park; nothing here spins. The ordering is +//! `kore-lua`'s: a poster stores, then kicks; the host stands its post, +//! re-reads both words, then parks, so a kick that found no post standing +//! announced a store the re-read sees. Every thread of the program can kick +//! the host: the main thread holds the wake channel's send, and each thread it +//! starts is founded holding one too (`thread_controller`). +//! +//! A launch that cannot carry the host (no channel space or heap, a stack +//! window that cannot place the main thread) runs the program on thread 0, as +//! the C library alone would: allocation, threads, time and random work, a +//! filesystem session is refused, and only thread 0's exit ends the program. + +extern crate alloc; + +use core::cell::{Cell, UnsafeCell}; +use core::ffi::{c_int, c_void}; +use core::future::poll_fn; +use core::pin::pin; +use core::sync::atomic::{AtomicBool, AtomicU32, AtomicUsize, Ordering}; +use core::task::Poll; + +use kore_ffi::{KoreFsError, KoreFsStep, KoreStatus, Session}; + +use crate::exit::{Exit, ExitWord}; +use kore_std::ExitCode; +use kore_std::abi::session::Status; +use kore_std::fs::Fs; +use kore_std::runtime::chan::{NoticeRx, RecvError}; +use kore_std::runtime::entry::{self, Rt}; +use kore_std::runtime::executor::{self, Executor}; +use kore_std::runtime::futex::{self, WaitOutcome}; +use kore_std::runtime::thread::{self, DoorbellWorker, WorkerStack}; +use kore_std::runtime::transport::TrapTransport; +use kore_std::runtime::{crash, startup}; + +/// The C library's program: runs `main`, then `exit`s with its value. +pub type Program = unsafe extern "C" fn() -> !; +/// The C library's exit: commits `status` on the calling thread's own handle. +pub type End = unsafe extern "C" fn(c_int) -> !; + +/// The program, as [`__kore_program_start`] received it, for the main thread. +static PROGRAM: AtomicUsize = AtomicUsize::new(0); +/// The C library's exit, as [`__kore_program_start`] received it, for an +/// [`abort`] on thread 0. +static END: AtomicUsize = AtomicUsize::new(0); +/// Set by the first [`__kore_program_start`]; a second never founds a host. +static STARTED: AtomicBool = AtomicBool::new(false); +/// The host serves: the main thread exists and thread 0 parks for it. Set +/// before the main thread is founded, since its first call may need it. +static HOSTED: AtomicBool = AtomicBool::new(false); + +/// This thread's part in the program. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum Role { + /// A thread the program started, or thread 0 before the program runs. + Other, + /// Thread 0, once it holds the program: the host, or the program itself + /// when the launch could not carry a host. + Host, + /// The thread `main` runs on. + Main, +} + +#[thread_local] +static ROLE: Cell = Cell::new(Role::Other); + +/// Whether the calling thread is the host: thread 0 holding the program. +pub(crate) fn on_host() -> bool { + ROLE.get() == Role::Host +} + +/// Whether a host serves: thread 0 answers what other threads post. +pub(crate) fn hosted() -> bool { + HOSTED.load(Ordering::Acquire) +} + +/// Whether the calling thread is the main thread, the one the host can kick. +pub(crate) fn on_main() -> bool { + ROLE.get() == Role::Main +} + +// ── The end ───────────────────────────────────────────────────────────── + +/// The program's end, once asked for (see [`ExitWord`]). +static EXIT: ExitWord = ExitWord::new(); + +// ── The session cell ──────────────────────────────────────────────────── + +// The word's states. The main thread owns the cell at IDLE and SERVED, the +// host at POSTED; every hand-over is a Release store matched by an Acquire. +const IDLE: u32 = 0; +const POSTED: u32 = 1; +const SERVED: u32 = 2; + +struct Request { + state: AtomicU32, + payload: UnsafeCell, +} + +struct Payload { + // main thread → host: the session's callback program. + step: Option, + user: *mut c_void, + // host → main thread: how the session ended. + status: KoreStatus, + error: KoreFsError, +} + +// SAFETY: `payload` is touched only by the side the word hands it to (above). +unsafe impl Sync for Request {} + +static SESSION: Request = Request { + state: AtomicU32::new(IDLE), + payload: UnsafeCell::new(Payload { + step: None, + user: core::ptr::null_mut(), + status: KoreStatus::Ok, + error: KoreFsError::None, + }), +}; + +impl Request { + /// # Safety + /// The caller observed (Acquire) a state that hands it the cell, and holds + /// no other reference across the store that hands it back. + #[allow(clippy::mut_from_ref)] + unsafe fn payload(&self) -> &mut Payload { + unsafe { &mut *self.payload.get() } + } +} + +/// Anything the host must answer: an end, a session, or a mapping. +fn pending() -> bool { + EXIT.posted().is_some() + || SESSION.state.load(Ordering::Acquire) == POSTED + || crate::mmap::posted() +} + +// ── Entry points the C library calls ──────────────────────────────────── + +/// Run the program. The C library's start hands it here when the program +/// links libkore: `program` runs `main` and then `exit`; `end` is the C +/// library's exit, which commits a status on the calling thread's own handle. +/// Never returns. +/// +/// # Safety +/// Called once, by the C library's start on the founded thread and before the +/// program has run. Neither `program` nor `end` returns. +#[unsafe(no_mangle)] +pub unsafe extern "C" fn __kore_program_start(program: Option, end: Option) -> ! { + let (Some(program), Some(end)) = (program, end) else { + // A C library that hands over nothing to run has broken its half of + // this contract; say so rather than run on. + thread::exit(ExitCode::INTERNAL) + }; + if STARTED.swap(true, Ordering::AcqRel) { + // A second start founds no second host. + unsafe { program() } + } + ROLE.set(Role::Host); + PROGRAM.store(program as usize, Ordering::Release); + END.store(end as usize, Ordering::Release); + match Founded::found() { + Some(host) => host.serve(end), + None => unsafe { program() }, + } +} + +/// End the program with `status`. The C library's `exit` calls this after its +/// `atexit` handlers with `finalize` nonzero, `_Exit` with zero, on whatever +/// thread called them. +/// +/// Returns only when the caller must commit the exit itself: it is thread 0, +/// every other thread is stopped, and, for `exit`, the finalizers have run if +/// none was still running. On any other thread the request goes to the host +/// and the calling thread ends. +#[unsafe(no_mangle)] +pub extern "C" fn __kore_program_exit(status: c_int, finalize: c_int) { + let finalize = finalize != 0; + match ROLE.get() { + // Thread 0 itself: the program when there is no host, or a callback + // or finalizer the host is running. It cannot wait for its own loop, + // so it relays what the launcher has room for now; the barrier is + // this thread's. + Role::Host => { + crate::stdio::flush_now(); + startup::barrier(finalize) + } + role if HOSTED.load(Ordering::Acquire) => { + let _ = EXIT.post(Exit { + status, + finalize, + from_main: role == Role::Main, + }); + // Refused only when no post stood, and then the host re-reads the + // word before it parks: the store above reaches it either way. + let _ = thread::kick_parent(); + thread::exit(ExitCode::OK) + } + // No host: only thread 0 owns the process. Anywhere else the barrier + // refuses and ends this thread alone. + _ => startup::barrier(finalize), + } +} + +/// `abort` (C11 7.22.4.1): the program's abnormal end. It ends with the +/// aborted status, 134 — what a shell reports for a program `SIGABRT` ended — +/// and, like `_Exit`, runs no `atexit` handler or finalizer. No stream +/// buffers, so what the program wrote before has already left it: the host's +/// loop relays all of it before the end when another thread aborts, and a +/// callback's `abort` on the host itself relays what the launcher has room +/// for at once (`stdio::flush_now`). +/// +/// The C library's `_Exit` lives in the program, so this takes `_Exit`'s +/// path itself: any other thread hands the status to the host and ends, and +/// thread 0, the barrier raised, commits the end the C library handed over. +#[unsafe(no_mangle)] +pub extern "C" fn abort() -> ! { + let status = ExitCode::ABORTED.get(); + __kore_program_exit(status, 0); + match END.load(Ordering::Acquire) { + // A program the C library never started: thread 0 ends the task. + 0 => thread::exit(ExitCode::ABORTED), + end => { + // SAFETY: `__kore_program_start` stored an `End`, which never + // returns; every other thread is stopped. + let end: End = unsafe { core::mem::transmute(end) }; + unsafe { end(status) } + } + } +} + +/// The main thread's half of [`kore_fs_run`](crate::api::kore_fs_run): post +/// the session to the host and wait for its answer. +/// +/// # Safety +/// `kore_fs_run`'s contract for `step` and `user`. +pub(crate) unsafe fn session(step: KoreFsStep, user: *mut c_void) -> (KoreStatus, KoreFsError) { + if ROLE.get() != Role::Main || !HOSTED.load(Ordering::Acquire) { + return (KoreStatus::Refused, KoreFsError::None); + } + // The main thread is the only poster, and it waits out each request: the + // cell is idle whenever control is here. + { + // SAFETY: IDLE hands the cell to the main thread. + let payload = unsafe { SESSION.payload() }; + payload.step = Some(step); + payload.user = user; + } + SESSION.state.store(POSTED, Ordering::Release); + // Refused only when no post stood, and then the host re-reads the word + // before it parks. + let _ = thread::kick_parent(); + loop { + match SESSION.state.load(Ordering::Acquire) { + SERVED => break, + state => { + if futex::wait(&SESSION.state, state, None) == WaitOutcome::Unsupported { + // The main thread is founded with its own doorbell, so it + // can always park; if it cannot, spinning is the one + // thing left, and this host will not. End the program. + fatal() + } + } + } + } + let answer = { + // SAFETY: SERVED hands the cell back to the main thread. + let payload = unsafe { SESSION.payload() }; + payload.step = None; + payload.user = core::ptr::null_mut(); + (payload.status, payload.error) + }; + SESSION.state.store(IDLE, Ordering::Release); + answer +} + +/// This library's panic: the record first, then the end. A panic on thread 0 +/// stops the program there; anywhere else it asks the host to, so the process +/// ends with the panic rather than running on without the panicking thread. +pub(crate) fn panicked(info: &core::panic::PanicInfo<'_>) -> ! { + crash::report(info); + match ROLE.get() { + Role::Host => { + crate::stdio::flush_now(); + startup::barrier(false) + } + role if HOSTED.load(Ordering::Acquire) => { + let _ = EXIT.post(Exit { + status: ExitCode::PANIC.get(), + finalize: false, + from_main: role == Role::Main, + }); + let _ = thread::kick_parent(); + } + _ => {} + } + thread::exit(ExitCode::PANIC) +} + +/// A host invariant broke on a program thread: end the program with +/// `INTERNAL`, never by spinning. +fn fatal() -> ! { + let _ = EXIT.post(Exit { + status: ExitCode::INTERNAL.get(), + finalize: false, + from_main: ROLE.get() == Role::Main, + }); + let _ = thread::kick_parent(); + thread::exit(ExitCode::INTERNAL) +} + +// ── The host ──────────────────────────────────────────────────────────── + +/// The host's wake channel broke or a park was refused: it cannot hear the +/// program, so it ends it. +struct Fatal; + +/// Thread 0 with its program founded: the runtime context, the executor's +/// doorbell, the launcher's stream when the launch carried one, and the main +/// thread. +struct Founded { + rt: Rt, + doorbell: executor::Doorbell, + output: Option<&'static crate::stdio::Endpoint>, + worker: DoorbellWorker, +} + +impl Founded { + /// Found the main thread. `None` when the launch cannot carry the host. + fn found() -> Option { + let rt = entry::thread_context()?; + let doorbell = rt.doorbell()?; + // The C streams reach the launcher through this thread: map its + // stream and install the sink before any thread can write. + let output = crate::stdio::install(&rt); + // `main`'s arguments, read from the launcher's page while this is + // still the only thread. A line that is no argument vector cannot + // be handed to `main`: say why on `stderr` and end here, before the + // program runs. + if let Err(malformed) = crate::args::capture(&rt) { + crate::stdio::report(alloc::format!("kore: {malformed}\n").as_bytes()); + crate::stdio::flush_now(); + // SAFETY: the C library's exit, which `__kore_program_start` + // stored before it called this. + let end: End = unsafe { core::mem::transmute(END.load(Ordering::Acquire)) }; + unsafe { end(ExitCode::USAGE.get()) } + } + // Where `mmap` places: this task's free link span, published before + // any thread that could map exists. + crate::mmap::arm(&rt); + HOSTED.store(true, Ordering::Release); + // A mint grant of its own: the first `pthread_create` on the main + // thread starts kore-rt's thread controller there, which mints the + // doorbells its threads park on (`thread_controller`). Without one the + // controller still runs, taking the deadline park. + let grant = thread::channel_space_endowment(); + match thread::spawn_with_doorbell_on( + main_thread, + grant.as_slice(), + WorkerStack::Bytes(crate::api::KORE_MAIN_STACK_BYTES), + ) { + Some(worker) => Some(Self { + rt, + doorbell, + output, + worker, + }), + None => { + HOSTED.store(false, Ordering::Release); + // No host will drain the stream: writes report `EOF`, as + // they do for a launch that carried none. + if output.is_some() { + crate::stdio::abandon(); + } + None + } + } + } + + /// Serve the program until it ends, then end the process. + fn serve(self, end: End) -> ! { + let Self { + rt, + doorbell, + output, + worker, + } = self; + let transport = TrapTransport::new(); + let exit = { + let mut executor = Executor::new(entry::transport(), doorbell); + executor.run(async { + let exit = host(&rt, &transport, &worker, output).await; + // Whatever ended the program, what it wrote reaches the + // launcher, and then the stream's close, before the process + // ends: the launcher shows all of it before the status. + if let Some(output) = output { + crate::stdio::finish(output, &|| { + let _ = worker.kick(); + }) + .await; + } + exit + }) + }; + let exit = exit.unwrap_or(Exit { + status: ExitCode::INTERNAL.get(), + finalize: false, + from_main: false, + }); + if exit.from_main { + // The main thread asked and is ending itself: wait for it, so a + // program that ran on it alone is stopped cleanly. + let _ = worker.join(); + } else { + // Still running: the barrier stops it, and its channels end with + // the process. Its join ticket must not try to reclaim it first. + core::mem::forget(worker); + } + startup::barrier(exit.finalize); + // SAFETY: the C library's exit, as `__kore_program_start` received it. + unsafe { end(exit.status) } + } +} + +/// The main thread's entry: the program, as the C library handed it over. +fn main_thread() { + ROLE.set(Role::Main); + // SAFETY: `__kore_program_start` stored a `Program` before founding this + // thread, and the founding's session ordered that store before this load. + let program: Program = unsafe { core::mem::transmute(PROGRAM.load(Ordering::Acquire)) }; + unsafe { program() } +} + +/// The host's loop: answer each session the main thread posts, until any +/// thread asks for the end. +async fn host( + rt: &Rt, + transport: &TrapTransport, + worker: &DoorbellWorker, + output: Option<&'static crate::stdio::Endpoint>, +) -> Result { + let mut wake = NoticeRx::new(transport, worker.wake_recv()); + let mut filesystem = Filesystem::new(rt, transport); + // The one writer the host can kick, woken when its room is made. + let wake_main = || { + let _ = worker.kick(); + }; + loop { + if let Some(exit) = EXIT.posted() { + return Ok(exit); + } + if let Some(output) = output { + // One bounded pass: the end and the next session are served + // between passes, however fast the program writes. + crate::stdio::drain(output, &wake_main).await; + if let Some(exit) = EXIT.posted() { + return Ok(exit); + } + } + // A thread's `mmap` or `munmap`: thread 0 holds the supply's root. + if crate::mmap::serve() { + wake_main(); + continue; + } + if SESSION.state.load(Ordering::Acquire) == POSTED { + // SAFETY: POSTED hands the cell to the host until it stores SERVED. + let (step, user) = { + let payload = unsafe { SESSION.payload() }; + (payload.step, payload.user) + }; + let Some(step) = step else { + return Err(Fatal); + }; + // An end requested mid-session ends the program without waiting + // for the session: the barrier stops the waiting main thread, and + // the abandoned session's owner is quarantined, as cancellation + // leaves it. + let session = pin!(filesystem.run(step, user)); + let ended = pin!(until_exit(&mut wake, output, &wake_main)); + let outcome = race(session, ended).await; + let Some((status, error)) = outcome else { + continue; + }; + { + // SAFETY: still POSTED; the host owns the cell. + let payload = unsafe { SESSION.payload() }; + payload.status = status; + payload.error = error; + } + SESSION.state.store(SERVED, Ordering::Release); + let _ = worker.kick(); + continue; + } + // Armed before the recheck `park` makes, so a writer that published + // meanwhile either kicks or is seen. + crate::stdio::arm(); + park(&mut wake, || pending() || crate::stdio::pending()).await?; + } +} + +/// Poll `first` and then `second` until one completes: `Some` with the first's +/// output, or `None` when the second finished first. +async fn race( + mut first: core::pin::Pin<&mut A>, + mut second: core::pin::Pin<&mut B>, +) -> Option { + poll_fn(|cx| { + if let Poll::Ready(output) = first.as_mut().poll(cx) { + return Poll::Ready(Some(output)); + } + if second.as_mut().poll(cx).is_ready() { + return Poll::Ready(None); + } + Poll::Pending + }) + .await +} + +/// How soon a receive refused for want of a standing post looks again: the +/// tree's figure for a rotating notice receiver (the WASI host's handoff). +const WOULD_BLOCK_RETRY_NS: u64 = 100_000; + +/// Park until a kick, a spurious wake, or `ready` holds, having stood the +/// wake post before re-reading what `ready` reads. +async fn park( + wake: &mut NoticeRx<'_, TrapTransport>, + ready: impl Fn() -> bool, +) -> Result<(), Fatal> { + loop { + match wake.try_recv() { + Ok(Some(_)) => return Ok(()), + Ok(None) => { + if ready() { + return Ok(()); + } + let Some(post) = wake.standing_post() else { + return Err(Fatal); + }; + let _ = executor::next_wake_on(post).await; + } + Err(RecvError::Refused(Status::WouldBlock)) => { + // No post stands to wake on: look again shortly, as the + // tree's other receivers do, rather than wait for a wake + // nothing armed. + let _ = executor::next_wake_for(WOULD_BLOCK_RETRY_NS).await; + } + Err(_) => return Err(Fatal), + } + } +} + +/// Resolve once an end is posted, parking on the wake channel meanwhile and +/// draining what the program writes: the session being served is not a +/// reason to wake, only an end or output is. On a broken channel it never +/// resolves, since the host can no longer hear an end; the session it races +/// then runs to its own completion. Dropped when the session ends first, +/// which the drain survives (`stdio::drain`). Each pass yields, so the +/// session it races is polled however fast the program writes. +async fn until_exit( + wake: &mut NoticeRx<'_, TrapTransport>, + output: Option<&'static crate::stdio::Endpoint>, + wake_main: &dyn Fn(), +) { + let ended = || EXIT.posted().is_some(); + while !ended() { + if let Some(output) = output { + crate::stdio::drain(output, wake_main).await; + executor::yield_now().await; + } + // Another thread's mapping does not wait for the session to end, + // and the session is polled between two of them. + if crate::mmap::serve() { + wake_main(); + executor::yield_now().await; + continue; + } + crate::stdio::arm(); + if park(wake, || { + ended() || crate::stdio::pending() || crate::mmap::posted() + }) + .await + .is_err() + { + core::future::pending::<()>().await; + } + } +} + +/// The program's filesystem: attached on first use, one native owner for the +/// program's life. +struct Filesystem<'r, 't> { + rt: &'r Rt, + transport: &'t TrapTransport, + session: Option>, + /// The launch carried no face, or the owner could not be formed: every + /// later session is refused the same way without asking again. + refused: Option<(KoreStatus, KoreFsError)>, +} + +impl<'r, 't> Filesystem<'r, 't> { + fn new(rt: &'r Rt, transport: &'t TrapTransport) -> Self { + Self { + rt, + transport, + session: None, + refused: None, + } + } + + async fn run(&mut self, step: KoreFsStep, user: *mut c_void) -> (KoreStatus, KoreFsError) { + if let Some(refused) = self.refused { + return refused; + } + if self.session.is_none() { + let Some(fs) = Fs::attach(self.rt, self.transport).await else { + let refused = (KoreStatus::Refused, KoreFsError::Unsupported); + self.refused = Some(refused); + return refused; + }; + match Session::new(fs) { + Ok(session) => self.session = Some(session), + Err(_) => { + let refused = (KoreStatus::Capacity, KoreFsError::None); + self.refused = Some(refused); + return refused; + } + } + } + let Some(session) = self.session.as_mut() else { + return (KoreStatus::Refused, KoreFsError::None); + }; + // SAFETY: `kore_fs_run`'s caller took on `Session::run`'s contract for + // `step` and `user`, and waits until this future completes or is + // dropped with the program's end. + match unsafe { session.run(step, user) }.await { + Ok(()) => (KoreStatus::Ok, KoreFsError::None), + Err(error) => error.foreign(), + } + } +} diff --git a/crates/c/src/stdio.rs b/crates/c/src/stdio.rs new file mode 100644 index 0000000..7e6f712 --- /dev/null +++ b/crates/c/src/stdio.rs @@ -0,0 +1,960 @@ +//! A C program's `stdout` and `stderr`. +//! +//! The launcher's stream (the launch's `PROFILE_INTERACTIVE` window) is +//! mapped on thread 0, the host, and is bound to it: only thread 0 may publish +//! into it. Every thread writes the C streams, so their bytes reach the +//! stream through the host, by one in-process ring: +//! +//! - **A writer** holds the streams' lock for its whole C call: kore-c-shim +//! takes it around each stream call, as POSIX's `flockfile` does, so one +//! call's output is never split by another thread's. Under the lock the +//! writer is the ring's one producer. It waits for room for each piece, +//! copies it in, and kicks the host only if the host armed itself before +//! parking, so a loop of `printf` costs no trap per call. The host can +//! kick the main thread but holds no send into a pthread's doorbell, so +//! every wait for room is bounded by the runtime's slice: a writer the +//! host cannot reach re-reads the ring each slice, as the thread +//! controller's unreached clients do. +//! - **The host** drains the ring a pass at a time — what was queued when +//! the pass began — into the stream's output records, as many as the +//! launcher has room for, and waits for their capacity when they are full: +//! a slow terminal holds the program back rather than losing its output. +//! Between passes it serves the program's end and its filesystem sessions, +//! so a program that never stops printing still ends when it asks to. +//! - **The host's own writes** (a filesystem step's callbacks run on it) +//! cannot wait for the host. They are kept in order beside the ring, each +//! stamped with the ring position it follows, and drained in that order. +//! The host never takes the streams' lock, which a writer waiting on the +//! host may hold. +//! - **The end** drains what was queued and closes the stream before the +//! process ends, so the launcher shows the program's output before its exit +//! status. An end committed on the host itself (a callback's `exit` or +//! `abort`, a panic there) cannot wait, and relays at once what the +//! launcher has room for. A write after the end, or once the stream is +//! gone, reports `EOF`. +//! +//! The ring and its record codec are plain data over atomics ([`Ring`]), so +//! the host suite exercises the code the target runs; the endpoint, the lock's +//! parks and the kicks are the metal half (`metal`). + +use core::cell::UnsafeCell; +#[cfg(target_os = "none")] +use core::sync::atomic::AtomicBool; +use core::sync::atomic::{AtomicU32, Ordering}; + +/// Which C stream a record carries. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum Stream { + Out, + Err, +} + +impl Stream { + const fn tag(self) -> u8 { + match self { + Self::Out => 1, + Self::Err => 2, + } + } + + const fn from_tag(tag: u8) -> Option { + match tag { + 1 => Some(Self::Out), + 2 => Some(Self::Err), + _ => None, + } + } +} + +/// Bytes of one record's header: its stream's tag and its length. +const HEADER: usize = 2; +/// The most payload one record carries: its length is one byte. +pub(crate) const RECORD_PAYLOAD: usize = u8::MAX as usize; + +/// The ring's bytes ([`RING`]). A power of two, so a position is an index by +/// a mask, and far below `u32`'s range, so the free-running positions' +/// difference never wraps past a lap. +const RING_BYTES: usize = 1 << 14; + +/// The ring bytes `len` payload bytes take, in whole records. +pub(crate) const fn framed(len: usize) -> usize { + len + HEADER * len.div_ceil(RECORD_PAYLOAD) +} + +/// The largest piece the ring keeps whole: half of it, so a writer waiting +/// for that much room never waits for more room than the host can make +/// while another piece is queued ahead of it. A longer write goes in pieces +/// of this size, under the one lock its call holds. +pub(crate) const WHOLE_WRITE: usize = + RING_BYTES / 2 - HEADER * (RING_BYTES / 2).div_ceil(RECORD_PAYLOAD); + +const _: () = assert!(RING_BYTES.is_power_of_two()); +const _: () = assert!(framed(WHOLE_WRITE) <= RING_BYTES / 2); + +/// A cursor on a cache line of its own: the writer's `tail` and the host's +/// `head` move on different cores, and must not share a line. +#[repr(align(64))] +struct Cursor(AtomicU32); + +/// One ring of framed records over free-running `u32` positions: `tail` is +/// written only by the lock holder, `head` only by the host. +pub(crate) struct Ring { + bytes: UnsafeCell<[u8; N]>, + /// The first byte the host has not consumed. + head: Cursor, + /// One past the last byte a writer published. + tail: Cursor, +} + +// SAFETY: the bytes between `head` and `tail` belong to the host, the rest to +// the lock holder; each side publishes its cursor after its last access and +// reads the other's with Acquire before its first. +unsafe impl Sync for Ring {} + +impl Ring { + pub(crate) const fn new() -> Self { + Self { + bytes: UnsafeCell::new([0; N]), + head: Cursor(AtomicU32::new(0)), + tail: Cursor(AtomicU32::new(0)), + } + } + + /// The host's cursor, for a writer to wait on. + pub(crate) fn head(&self) -> &AtomicU32 { + &self.head.0 + } + + /// Bytes published and not consumed, at most: `head` is read first, and + /// `tail` never falls behind any `head` read before it. + pub(crate) fn queued(&self) -> usize { + let head = self.head.0.load(Ordering::Acquire); + self.tail.0.load(Ordering::Acquire).wrapping_sub(head) as usize + } + + /// Free bytes: exact under the lock, where `tail` is this thread's; + /// conservative outside it, where the host and another writer may both + /// have moved between the two reads. + pub(crate) fn room(&self) -> usize { + N.saturating_sub(self.queued()) + } + + /// Room for `bytes` more, framed. + pub(crate) fn fits(&self, bytes: usize) -> bool { + self.room() >= framed(bytes) + } + + /// The position the next record will be published at. + pub(crate) fn tail(&self) -> u32 { + self.tail.0.load(Ordering::Acquire) + } + + /// Publish `payload` as records of `stream`, whole. + /// + /// # Safety + /// The caller is the ring's one producer (it holds the streams' lock) and + /// saw [`fits`](Self::fits) for `payload` under that lock. + pub(crate) unsafe fn publish(&self, stream: Stream, payload: &[u8]) { + let mut at = self.tail.0.load(Ordering::Relaxed); + for record in payload.chunks(RECORD_PAYLOAD) { + // SAFETY: the caller's contract: `[tail, tail + framed)` is free, + // so the host reads none of these bytes until the store below. + unsafe { + self.put(at, stream.tag()); + self.put(at.wrapping_add(1), record.len() as u8); + for (i, byte) in record.iter().enumerate() { + self.put(at.wrapping_add((HEADER + i) as u32), *byte); + } + } + at = at.wrapping_add((HEADER + record.len()) as u32); + } + // SeqCst, not Release: the host's park recheck reads `tail` after + // arming, and the writer reads the armed flag after this store — the + // Dekker pair that makes a kick unnecessary unless the host armed. + self.tail.0.store(at, Ordering::SeqCst); + } + + /// The record at the host's cursor, if any: its stream and its payload, + /// copied into `out`. The host is the one consumer. + pub(crate) fn peek(&self, out: &mut [u8; RECORD_PAYLOAD]) -> Option<(Stream, usize)> { + let head = self.head.0.load(Ordering::Relaxed); + if self.tail.0.load(Ordering::Acquire) == head { + return None; + } + // SAFETY: `[head, tail)` holds whole records the Acquire above made + // visible, and only this consumer advances `head`. + unsafe { + let stream = Stream::from_tag(self.get(head))?; + let len = self.get(head.wrapping_add(1)) as usize; + for (i, byte) in out.iter_mut().take(len).enumerate() { + *byte = self.get(head.wrapping_add((HEADER + i) as u32)); + } + Some((stream, len)) + } + } + + /// Consume the record [`peek`](Self::peek) returned, `len` payload bytes. + pub(crate) fn consume(&self, len: usize) { + let head = self.head.0.load(Ordering::Relaxed); + self.head + .0 + .store(head.wrapping_add((HEADER + len) as u32), Ordering::Release); + } + + /// Drop every record published so far, as the one consumer. + pub(crate) fn discard(&self) { + let tail = self.tail.0.load(Ordering::Acquire); + self.head.0.store(tail, Ordering::Release); + } + + unsafe fn put(&self, at: u32, byte: u8) { + // SAFETY: the caller owns byte `at` (see `publish`). + unsafe { (*self.bytes.get())[at as usize & (N - 1)] = byte } + } + + unsafe fn get(&self, at: u32) -> u8 { + // SAFETY: the caller owns byte `at` (see `peek`). + unsafe { (*self.bytes.get())[at as usize & (N - 1)] } + } +} + +/// The program's one ring. +#[cfg(target_os = "none")] +pub(crate) static RING: Ring = Ring::new(); + +/// The host is parked, or about to, and a writer must kick it (see +/// [`Ring::publish`]). +#[cfg(target_os = "none")] +static ARMED: AtomicBool = AtomicBool::new(false); + +/// The room the main thread waits for, framed, or 0: the one writer the host +/// can kick, kicked once that much is free rather than at every record. +#[cfg(target_os = "none")] +static MAIN_NEEDS: AtomicU32 = AtomicU32::new(0); + +/// The launcher's stream is gone, or there is no host to drain the ring: +/// writes fail rather than wait for a drain that will not come. +#[cfg(target_os = "none")] +static LOST: AtomicBool = AtomicBool::new(false); + +/// The program has ended: nothing written from here is relayed. +#[cfg(target_os = "none")] +static FINISHED: AtomicBool = AtomicBool::new(false); + +/// Writes report `EOF` from here on. +#[cfg(target_os = "none")] +fn closed() -> bool { + LOST.load(Ordering::Acquire) || FINISHED.load(Ordering::Acquire) +} + +/// Whether the host has anything to drain. +#[cfg(target_os = "none")] +pub(crate) fn pending() -> bool { + !closed() && (RING.queued() != 0 || metal::host_pending()) +} + +/// A write the host made while it could not drain: its bytes, and the ring +/// position it follows. +#[cfg(target_os = "none")] +pub(crate) struct Held { + pub(crate) after: u32, + pub(crate) stream: Stream, + pub(crate) bytes: alloc::vec::Vec, +} + +/// Whether a held write is the next item to drain: it belongs at ring +/// position `held_after`, so it goes once the host's cursor reaches that +/// point, after every record published before it and before every one +/// published after. A position the cursor has already passed (a write made +/// during a call whose first records were already relayed) goes at once. +/// Positions are free-running, and a stamp never strays half the `u32` +/// range from the cursor. +pub(crate) fn held_is_next(held_after: u32, head: u32) -> bool { + head.wrapping_sub(held_after) as i32 >= 0 +} + +extern crate alloc; + +#[cfg(target_os = "none")] +pub(crate) use metal::{Endpoint, abandon, arm, drain, finish, flush_now, install, report}; + +#[cfg(target_os = "none")] +mod metal { + use super::*; + use alloc::collections::VecDeque; + use alloc::vec::Vec; + use core::cell::{Cell, RefCell}; + use core::mem::MaybeUninit; + use core::sync::atomic::AtomicU64; + use core::time::Duration; + use kore_c_shim::stdio::{Standard, StandardSink, install_standard_sink}; + use kore_std::abi::process::{PROCESS_OUTPUT_CHUNK_BYTES, ProcessOutputEvent}; + use kore_std::process::{ + MappedProcessStdioChild, ProcessStdioParkStep, ProcessStdioWaitFor, ProcessStdioWake, + }; + use kore_std::runtime::futex::{self, WaitOutcome}; + use kore_std::runtime::{executor, pace, thread}; + + /// How long a writer waits before it re-reads what it waits for: the + /// runtime's slice for a waiter its waker cannot reach (the thread + /// controller's `UNREACHED_SLICE`). The host kicks the main thread + /// sooner; a pthread rides the slice. + const SLICE: Duration = Duration::from_nanos(pace::CEDE_NS); + + /// The host's half: the launcher's stream and the writes the host held. + /// Thread 0 alone touches it (the stream is bound to that thread), and + /// never across an await. + pub(crate) struct Endpoint { + stdio: RefCell, + held: RefCell>, + /// Bytes of the next item (the front held write, or the ring's head + /// record) already published. A drain can be dropped between two + /// records it publishes (it races a filesystem session), so it + /// consumes an item only once all of it is out, and resumes past + /// what it already sent rather than send it twice. + sent: Cell, + } + + /// Thread 0's endpoint, in static storage: the stream is mapped before + /// the main thread is founded, and a launch without a heap still has + /// one. + struct EndpointCell(UnsafeCell>); + // SAFETY: written once by the host before any other thread exists, and + // read only on thread 0 (`crate::program::on_host`) once `MAPPED` is set. + unsafe impl Sync for EndpointCell {} + static ENDPOINT: EndpointCell = EndpointCell(UnsafeCell::new(MaybeUninit::uninit())); + static MAPPED: AtomicBool = AtomicBool::new(false); + + fn host() -> Option<&'static Endpoint> { + if !crate::program::on_host() || !MAPPED.load(Ordering::Acquire) { + return None; + } + // SAFETY: on thread 0, after `install` wrote it (see `EndpointCell`). + Some(unsafe { (*ENDPOINT.0.get()).assume_init_ref() }) + } + + /// A line on `stderr` from the host itself: what it must say when the + /// program cannot run. Kept in order and relayed as any write made on + /// the host is. Nothing is said when the launch carried no stream. + pub(crate) fn report(line: &[u8]) { + let _ = sink(Standard::Error, line); + } + + pub(crate) fn host_pending() -> bool { + host().is_some_and(|endpoint| !endpoint.held.borrow().is_empty()) + } + + /// The C streams' sink and their lock. + static SINK: StandardSink = StandardSink { + write: sink, + lock: lock_streams, + unlock: unlock_streams, + }; + + /// Map the launcher's stream and send the C streams to it. `None` when + /// the launch carried no stream: `stdout` then reports `EOF`, as it does + /// for every program that has none. + pub(crate) fn install(rt: &kore_std::runtime::entry::Rt) -> Option<&'static Endpoint> { + let stdio = kore_std::process::map_stdio(rt)?; + // SAFETY: thread 0, before any other thread exists, and only once + // (`__kore_program_start` founds one host). + let endpoint = unsafe { + (*ENDPOINT.0.get()).write(Endpoint { + stdio: RefCell::new(stdio), + held: RefCell::new(VecDeque::new()), + sent: Cell::new(0), + }) + }; + MAPPED.store(true, Ordering::Release); + install_standard_sink(&SINK); + Some(endpoint) + } + + /// The host could not be founded after the stream was mapped: no one + /// will drain it, so every write reports `EOF`. + pub(crate) fn abandon() { + LOST.store(true, Ordering::SeqCst); + } + + // ── The streams' lock ─────────────────────────────────────────────── + + /// The streams' lock, reentrant per thread ([`DEPTH`]): a stream call + /// holds it across all its writes, and the sink each write reaches takes + /// it again. + static LOCK: crate::lock::Lock = crate::lock::Lock::new(); + + /// How many times this thread holds [`LOCK`]. + #[thread_local] + static DEPTH: Cell = Cell::new(0); + + /// The call holding [`LOCK`], if any: [`IN_CALL`] beside the ring's + /// `tail` when it began. The host's own writes made meanwhile are + /// stamped there, so they go before the call rather than between its + /// pieces. + static CALL: AtomicU64 = AtomicU64::new(0); + const IN_CALL: u64 = 1 << u32::BITS; + + /// Take the streams' lock. The host never does: a writer that holds it + /// may be waiting for the host to drain. + fn lock_streams() { + if crate::program::on_host() { + return; + } + let depth = DEPTH.get(); + if depth == 0 { + LOCK.acquire(); + // The holder is the ring's one producer: `tail` is exact here. + CALL.store(IN_CALL | u64::from(RING.tail()), Ordering::SeqCst); + } + DEPTH.set(depth + 1); + } + + fn unlock_streams() { + if crate::program::on_host() { + return; + } + match DEPTH.get() { + // Released more often than taken: the caller's mistake, and not + // a reason to release another thread's hold. + 0 => {} + 1 => { + DEPTH.set(0); + CALL.store(0, Ordering::SeqCst); + LOCK.release(); + } + depth => DEPTH.set(depth - 1), + } + } + + // ── Writers ───────────────────────────────────────────────────────── + + fn sink(standard: Standard, bytes: &[u8]) -> bool { + let stream = match standard { + Standard::Output => Stream::Out, + Standard::Error => Stream::Err, + }; + if bytes.is_empty() { + return true; + } + if closed() { + return false; + } + if let Some(endpoint) = host() { + return hold(endpoint, stream, bytes); + } + // The call's lock, again: this thread is the ring's one producer for + // every piece. + lock_streams(); + let written = bytes.chunks(WHOLE_WRITE).all(|piece| write(stream, piece)); + unlock_streams(); + written + } + + /// One piece, under the streams' lock: wait for its room, then copy it + /// in. + fn write(stream: Stream, piece: &[u8]) -> bool { + loop { + if closed() { + return false; + } + // Read before the room check: a wait on this value returns at + // once if the host has moved since. + let head = RING.head().load(Ordering::SeqCst); + if RING.fits(piece.len()) { + // SAFETY: the lock makes this thread the one producer, and + // the room was checked under it. + unsafe { RING.publish(stream, piece) }; + if ARMED.swap(false, Ordering::SeqCst) { + let _ = thread::kick_parent(); + } + return true; + } + // The ring is full of records the host has not drained. A host + // parked on its wake (armed) is kicked; one that is not is + // draining already, or waiting for the launcher's room, and + // moves its cursor when it has drained. + let main = crate::program::on_main(); + if main { + MAIN_NEEDS.store(framed(piece.len()) as u32, Ordering::SeqCst); + } + if ARMED.swap(false, Ordering::SeqCst) { + let _ = thread::kick_parent(); + } + let outcome = futex::wait(RING.head(), head, Some(SLICE)); + if main { + MAIN_NEEDS.store(0, Ordering::SeqCst); + } + if outcome == WaitOutcome::Unsupported { + let _ = pace::cede(); + } + } + } + + /// The host's own write: kept, stamped, for its next drain. Stamped at + /// the start of a call in progress, so it does not split that call. + fn hold(endpoint: &Endpoint, stream: Stream, bytes: &[u8]) -> bool { + let mut copy = Vec::new(); + if copy.try_reserve_exact(bytes.len()).is_err() { + return false; + } + copy.extend_from_slice(bytes); + let mut held = endpoint.held.borrow_mut(); + if held.try_reserve(1).is_err() { + return false; + } + let call = CALL.load(Ordering::SeqCst); + let after = if call & IN_CALL != 0 { + call as u32 + } else { + RING.tail() + }; + held.push_back(Held { + after, + stream, + bytes: copy, + }); + true + } + + // ── The host ──────────────────────────────────────────────────────── + + /// How one item went out. + enum Pumped { + /// All of it. + Done, + /// The launcher has no room for the next record: wait, or stop. + Full, + /// The launcher's stream is gone. + Lost, + } + + /// Publish what is left of one item, `len` bytes that `fill` copies out + /// from an offset, as stream records while the launcher has room, past + /// what [`Endpoint::sent`] says is out. Never waits. + fn pump( + endpoint: &Endpoint, + stream: Stream, + len: usize, + fill: &dyn Fn(usize, &mut [u8]) -> usize, + ) -> Pumped { + let mut stdio = endpoint.stdio.borrow_mut(); + while endpoint.sent.get() < len { + if stdio.output_capacity_left() == 0 { + return Pumped::Full; + } + let from = endpoint.sent.get(); + let mut chunk = [0u8; PROCESS_OUTPUT_CHUNK_BYTES]; + let want = (len - from).min(PROCESS_OUTPUT_CHUNK_BYTES); + let filled = fill(from, &mut chunk[..want]); + if filled == 0 { + // The item vanished under the drain: nothing can resume it. + return Pumped::Lost; + } + let event = match stream { + Stream::Out => ProcessOutputEvent::stdout(&chunk[..filled]), + Stream::Err => ProcessOutputEvent::stderr(&chunk[..filled]), + }; + let Ok(event) = event else { + return Pumped::Lost; + }; + match stdio.send_output_now(event) { + Ok(ProcessStdioWake::Unreached(_)) | Err(_) => return Pumped::Lost, + Ok(_) => endpoint.sent.set(from + filled), + } + } + endpoint.sent.set(0); + Pumped::Done + } + + /// [`pump`] one item whole, waiting for the launcher's room between its + /// records. `false` once the stream is gone. Dropped at a wait, it has + /// published whole records only, and `sent` says how far. + async fn send( + endpoint: &Endpoint, + stream: Stream, + len: usize, + fill: &dyn Fn(usize, &mut [u8]) -> usize, + ) -> bool { + loop { + match pump(endpoint, stream, len, fill) { + Pumped::Done => return true, + Pumped::Lost => return false, + Pumped::Full => { + if !room(endpoint).await { + return false; + } + } + } + } + } + + /// Copy the front held write's bytes from `from` into `out`. + fn held_bytes(endpoint: &Endpoint, from: usize, out: &mut [u8]) -> usize { + let held = endpoint.held.borrow(); + let Some(front) = held.front() else { + return 0; + }; + let Some(bytes) = front.bytes.get(from..from + out.len()) else { + return 0; + }; + out.copy_from_slice(bytes); + out.len() + } + + /// The next item due, if any: the front held write when the host's + /// cursor reached its stamp, else the ring's head record if it lies + /// before `end`. + enum Next { + Held(Stream, usize), + Record(Stream, usize), + None, + } + + fn next(endpoint: &Endpoint, end: u32, record: &mut [u8; RECORD_PAYLOAD]) -> Next { + let head = RING.head().load(Ordering::Acquire); + if let Some(front) = endpoint.held.borrow().front() + && held_is_next(front.after, head) + { + return Next::Held(front.stream, front.bytes.len()); + } + if head == end { + return Next::None; + } + match RING.peek(record) { + Some((stream, len)) => Next::Record(stream, len), + None => Next::None, + } + } + + /// A record went out: consume it, and wake the main thread once the + /// room it waits for is free. Fenced, so the cursor's store is ordered + /// before the read of what main needs, the other half of the writer's + /// pair. + fn consumed(len: usize, wake_main: &dyn Fn()) { + RING.consume(len); + core::sync::atomic::fence(Ordering::SeqCst); + let needs = MAIN_NEEDS.load(Ordering::SeqCst) as usize; + if needs != 0 && RING.room() >= needs { + wake_main(); + } + } + + /// The launcher's stream is gone: drop everything queued, and let every + /// writer see it. + fn lose(endpoint: &Endpoint, wake_main: &dyn Fn()) { + LOST.store(true, Ordering::SeqCst); + endpoint.held.borrow_mut().clear(); + endpoint.sent.set(0); + RING.discard(); + core::sync::atomic::fence(Ordering::SeqCst); + if MAIN_NEEDS.load(Ordering::SeqCst) != 0 { + wake_main(); + } + } + + /// How long a rotating notice receiver's handoff is given before the park + /// is prepared again: the WASI host's figure. + const HANDOFF_NS: u64 = 100_000; + + /// Wait until the launcher has room for one more record. `false` once its + /// stream is gone. The endpoint is borrowed to prepare and settle each + /// park, never across the wait (the WASI host's shape). + async fn room(endpoint: &Endpoint) -> bool { + loop { + let step = { + let mut stdio = endpoint.stdio.borrow_mut(); + // The drain snapshot comes before the capacity check, so room + // the launcher makes between them is seen by the park's + // recheck. + let drain = stdio.begin_drain(); + if stdio.output_capacity_left() > 0 { + return true; + } + match stdio.prepare_park(&drain, ProcessStdioWaitFor::Capacity) { + Ok(step) => step, + Err(_) => return false, + } + }; + match step { + ProcessStdioParkStep::Changed => {} + ProcessStdioParkStep::Handoff => { + let _ = executor::next_wake_for(HANDOFF_NS).await; + } + ProcessStdioParkStep::Park(ticket) => { + let reason = executor::next_wake_on(ticket.post()).await; + if endpoint + .stdio + .borrow_mut() + .settle_park(ticket, reason) + .is_err() + { + return false; + } + } + } + } + } + + /// One pass: what the ring held when it began, and the host's writes + /// held by then, in order. Bounded on both, so the host serves the + /// program between passes: a filesystem step's callbacks hold writes + /// while a pass waits for the launcher's room, and those wait for the + /// next pass. Cancellation-safe at every wait: an item leaves its queue + /// only once all of it is published. + pub(crate) async fn drain(endpoint: &Endpoint, wake_main: &dyn Fn()) { + let end = RING.tail(); + let mut held_left = endpoint.held.borrow().len(); + let mut record = [0u8; RECORD_PAYLOAD]; + loop { + if LOST.load(Ordering::Acquire) { + lose(endpoint, wake_main); + return; + } + let sent = match next(endpoint, end, &mut record) { + // A write held after the pass began: the next pass's, and so + // is everything after it. + Next::Held(..) if held_left == 0 => return, + Next::Held(stream, len) => { + let fill = |from: usize, out: &mut [u8]| held_bytes(endpoint, from, out); + let sent = send(endpoint, stream, len, &fill).await; + if sent { + endpoint.held.borrow_mut().pop_front(); + held_left -= 1; + } + sent + } + Next::Record(stream, len) => { + let bytes = record; + let fill = |from: usize, out: &mut [u8]| { + out.copy_from_slice(&bytes[from..from + out.len()]); + out.len() + }; + let sent = send(endpoint, stream, len, &fill).await; + if sent { + consumed(len, wake_main); + } + sent + } + Next::None => return, + }; + if !sent { + lose(endpoint, wake_main); + return; + } + } + } + + /// The host arms itself before it rechecks what it would park for: a + /// writer that published before this reads it and kicks, and one that + /// publishes after is seen by the recheck (the Dekker pair with + /// [`Ring::publish`]'s `SeqCst` store and the writer's swap). + pub(crate) fn arm() { + ARMED.store(true, Ordering::SeqCst); + core::sync::atomic::fence(Ordering::SeqCst); + } + + /// The program's end on the host's loop: drain what was queued and close + /// the stream. A write from here on reports `EOF`. + pub(crate) async fn finish(endpoint: &Endpoint, wake_main: &dyn Fn()) { + FINISHED.store(true, Ordering::SeqCst); + drain(endpoint, wake_main).await; + if LOST.load(Ordering::Acquire) || !room(endpoint).await { + return; + } + let _ = endpoint + .stdio + .borrow_mut() + .send_output_now(ProcessOutputEvent::closed()); + } + + /// The program's end on the host itself, which cannot wait (a callback's + /// `exit` or `abort`, a panic there): relay at once what the launcher has + /// room for, and close the stream if all of it went. A write from here on + /// reports `EOF`. + pub(crate) fn flush_now() { + let Some(endpoint) = host() else { + return; + }; + if closed() { + return; + } + FINISHED.store(true, Ordering::SeqCst); + // A borrow still held (a panic inside the drain itself) leaves the + // stream as it is rather than panic again. + if endpoint.stdio.try_borrow_mut().is_err() || endpoint.held.try_borrow_mut().is_err() { + return; + } + let end = RING.tail(); + let mut record = [0u8; RECORD_PAYLOAD]; + loop { + let pumped = match next(endpoint, end, &mut record) { + Next::Held(stream, len) => { + let fill = |from: usize, out: &mut [u8]| held_bytes(endpoint, from, out); + let pumped = pump(endpoint, stream, len, &fill); + if matches!(pumped, Pumped::Done) { + endpoint.held.borrow_mut().pop_front(); + } + pumped + } + Next::Record(stream, len) => { + let bytes = record; + let fill = |from: usize, out: &mut [u8]| { + out.copy_from_slice(&bytes[from..from + out.len()]); + out.len() + }; + let pumped = pump(endpoint, stream, len, &fill); + if matches!(pumped, Pumped::Done) { + RING.consume(len); + } + pumped + } + Next::None => break, + }; + if !matches!(pumped, Pumped::Done) { + return; + } + } + let mut stdio = endpoint.stdio.borrow_mut(); + if stdio.output_capacity_left() > 0 { + let _ = stdio.send_output_now(ProcessOutputEvent::closed()); + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn drain(ring: &Ring) -> std::vec::Vec<(Stream, std::vec::Vec)> { + let mut out = std::vec::Vec::new(); + let mut record = [0u8; RECORD_PAYLOAD]; + while let Some((stream, len)) = ring.peek(&mut record) { + out.push((stream, record[..len].to_vec())); + ring.consume(len); + } + out + } + + /// A write longer than a record is framed into records of at most a + /// record's payload, in order, and the ring accounts every header. + #[test] + fn a_write_is_framed_into_records_and_read_back_in_order() { + let ring = Ring::<1024>::new(); + let bytes: std::vec::Vec = (0..600u32).map(|i| i as u8).collect(); + assert!(ring.fits(bytes.len())); + unsafe { ring.publish(Stream::Out, &bytes) }; + assert_eq!(ring.queued(), framed(600)); + assert_eq!(framed(600), 600 + 3 * HEADER); + unsafe { ring.publish(Stream::Err, b"oops") }; + let records = drain(&ring); + let joined: std::vec::Vec = records + .iter() + .filter(|(stream, _)| *stream == Stream::Out) + .flat_map(|(_, bytes)| bytes.iter().copied()) + .collect(); + assert_eq!(joined, bytes); + assert_eq!(records.len(), 4); + assert_eq!(records[3], (Stream::Err, b"oops".to_vec())); + assert!( + records + .iter() + .all(|(_, bytes)| bytes.len() <= RECORD_PAYLOAD) + ); + assert_eq!(ring.queued(), 0); + } + + /// The positions run free across many laps and past `u32::MAX`; every + /// record still reads back whole, and room is exactly what is unconsumed. + #[test] + fn the_ring_wraps_its_bytes_and_its_positions() { + let ring = Ring::<64>::new(); + ring.head.0.store(u32::MAX - 20, Ordering::Relaxed); + ring.tail.0.store(u32::MAX - 20, Ordering::Relaxed); + for round in 0..200u32 { + let payload: std::vec::Vec = + (0..(round % 40) + 1).map(|i| (i + round) as u8).collect(); + assert!(ring.fits(payload.len()), "round {round}"); + unsafe { ring.publish(Stream::Out, &payload) }; + assert!(!ring.fits(64), "a full lap never fits"); + let records = drain(&ring); + let joined: std::vec::Vec = records.into_iter().flat_map(|(_, b)| b).collect(); + assert_eq!(joined, payload, "round {round}"); + } + } + + /// Room is the framed size: a write fits only while its records and + /// their headers do. + #[test] + fn room_counts_headers() { + let ring = Ring::<16>::new(); + assert!(ring.fits(14)); + assert!(!ring.fits(15)); + unsafe { ring.publish(Stream::Out, &[7; 10]) }; + assert!(ring.fits(2)); + assert!(!ring.fits(3)); + } + + /// A cursor read out of order — the host's moved past a `tail` read + /// before it — never reads as room the ring does not have. + #[test] + fn a_stale_read_never_finds_room_that_is_not_there() { + let ring = Ring::<16>::new(); + // A reader that saw `tail` = 0, then a `head` the host has advanced + // to 10, would compute a wrapped, enormous backlog. + ring.tail.0.store(0, Ordering::Relaxed); + ring.head.0.store(10, Ordering::Relaxed); + assert!(!ring.fits(1)); + // The real order, head first, is exact. + ring.tail.0.store(14, Ordering::Relaxed); + assert_eq!(ring.queued(), 4); + assert!(ring.fits(10)); + } + + /// Discarding consumes every published record at once. + #[test] + fn discard_drops_everything_published() { + let ring = Ring::<64>::new(); + unsafe { ring.publish(Stream::Out, b"lost") }; + unsafe { ring.publish(Stream::Err, b"too") }; + ring.discard(); + assert_eq!(ring.queued(), 0); + assert!(drain(&ring).is_empty()); + } + + /// The largest piece kept whole leaves room for another queued ahead of + /// it, whatever was framed in between. + #[test] + fn a_whole_write_fits_half_the_ring() { + assert!(framed(WHOLE_WRITE) <= RING_BYTES / 2); + assert!(framed(WHOLE_WRITE + RECORD_PAYLOAD) > RING_BYTES / 2); + } + + /// The host's held writes go after the ring records that preceded them + /// and before the ones that followed. + #[test] + fn a_held_write_drains_after_what_preceded_it() { + let ring = Ring::<256>::new(); + unsafe { ring.publish(Stream::Out, b"a") }; + let stamp = ring.tail(); + unsafe { ring.publish(Stream::Out, b"b") }; + let mut record = [0u8; RECORD_PAYLOAD]; + assert!(!held_is_next(stamp, ring.head().load(Ordering::Relaxed))); + let (_, len) = ring.peek(&mut record).unwrap(); + assert_eq!(&record[..len], b"a"); + ring.consume(len); + assert!(held_is_next(stamp, ring.head().load(Ordering::Relaxed))); + } + + /// A write stamped where a call began goes at once if the cursor already + /// passed that point, and never if the point is still ahead, across the + /// positions' wrap. + #[test] + fn a_held_write_behind_the_cursor_goes_at_once() { + for base in [0u32, u32::MAX - 3] { + let head = base.wrapping_add(10); + assert!(held_is_next(base, head), "passed at {base}"); + assert!(held_is_next(head, head), "reached at {base}"); + assert!(!held_is_next(head.wrapping_add(1), head), "ahead at {base}"); + } + } +} diff --git a/crates/c/tests/program.c b/crates/c/tests/program.c new file mode 100644 index 0000000..fda382c --- /dev/null +++ b/crates/c/tests/program.c @@ -0,0 +1,44 @@ +/* kore/program.h as an independent C program uses it: a session whose + * callbacks are kore.h's own KoreFsStep, run through kore_fs_run, and a + * descriptor described through kore_fd_describe. Compiled + * freestanding for every Kore ISA by tools/check-bindings.py. */ +#include + +struct reader { + int done; +}; + +static void open_root(struct KoreFsContext *context, const struct KoreFsCompletion *completion, + void *user) +{ + struct reader *reader = user; + if (completion->operation == KORE_OP_START) { + (void)kore_fs_open(context, (const uint8_t *)"", 0, KORE_FS_READ); + return; + } + reader->done = 1; +} + +/* The entry's exact type, so a drift in either header fails the build. */ +static KoreStatus (*const run)(KoreFsStep, void *, KoreFsError *) = kore_fs_run; + +int program_session(void) +{ + struct reader reader = {0}; + KoreFsError error = KORE_FS_ERROR_NONE; + KoreStatus status = run(open_root, &reader, &error); + return status == KORE_STATUS_OK && error == KORE_FS_ERROR_NONE && reader.done; +} + +/* A descriptor's description: the exact entry type, its fields and the + * constants its fields take. */ +static KoreStatus (*const describe)(int, KoreFdInfo *) = kore_fd_describe; + +int program_memory(int fd) +{ + KoreFdInfo info = {0}; + if (describe(fd, &info) != KORE_STATUS_OK) + return 0; + return info.kind == KORE_FD_MEMORY && (info.access & (KORE_FD_READ | KORE_FD_WRITE)) != 0 && + info.length != 0; +} diff --git a/crates/ffi/src/futex.rs b/crates/ffi/src/futex.rs index c76621d..6e04454 100644 --- a/crates/ffi/src/futex.rs +++ b/crates/ffi/src/futex.rs @@ -91,7 +91,8 @@ pub unsafe extern "C" fn kore_futex_wait( Ok(time) => time.as_nanos(), Err(_) => return KoreFutexOutcome::Unsupported, }; - let slice = deadline.map_or(1_000_000, |end| end.saturating_sub(now).min(1_000_000)); + let slice = crate::thread_local::CANCEL_SLICE_NS; + let slice = deadline.map_or(slice, |end| end.saturating_sub(now).min(slice)); let outcome = futex::wait(word, expected, Some(Duration::from_nanos(slice))); unsafe { crate::kore_thread_testcancel() }; if outcome != futex::WaitOutcome::TimedOut { diff --git a/crates/ffi/src/session.rs b/crates/ffi/src/session.rs index 6176f38..658b83f 100644 --- a/crates/ffi/src/session.rs +++ b/crates/ffi/src/session.rs @@ -25,6 +25,21 @@ pub enum RunError { Cleanup(Refusal), } +impl RunError { + /// What a foreign caller reads for a failed run, in the submission-status + /// vocabulary: `Quarantined` for an owner that may never run again, and + /// `Refused` with the refused close's error, whose view the owner kept. + pub fn foreign(&self) -> (KoreStatus, KoreFsError) { + match self { + Self::Quarantined => (KoreStatus::Quarantined, KoreFsError::None), + Self::Cleanup(Refusal::Fs(error)) => (KoreStatus::Refused, classify(*error)), + // A close answers in the filesystem's vocabulary or quarantines; + // no other refusal reaches cleanup. + Self::Cleanup(_) => (KoreStatus::Refused, KoreFsError::Unknown), + } + } +} + impl<'w, 't> Session<'w, 't> { /// Consume the real native face. Identifier exhaustion returns it unchanged. /// Close any previously opened external views before handing off this face: diff --git a/crates/ffi/src/thread.rs b/crates/ffi/src/thread.rs index a5a550c..9e05906 100644 --- a/crates/ffi/src/thread.rs +++ b/crates/ffi/src/thread.rs @@ -98,7 +98,11 @@ fn retire(argument: usize) { // SAFETY: called exactly once by native owner after successful reclamation. unsafe { let context = NonNull::new_unchecked(argument as *mut StartContext); - context.as_ref().control.retired.store(1, Ordering::Release); + let control = &context.as_ref().control; + // Store, then wake (the futex's order): a joiner parked on the word + // re-reads it once it is registered, so neither can miss the other. + control.retired.store(1, Ordering::SeqCst); + kore::futex::wake_all(&control.retired); release(context); } } @@ -196,16 +200,24 @@ pub unsafe extern "C" fn kore_thread_join( return KORE_THREAD_EDEADLK; } // Wait without claiming the native join record: cancellation must leave the - // target joinable. Only the controller can retire native capabilities. + // target joinable. Only the controller can retire native capabilities, and + // it wakes `retired` when it does. loop { unsafe { crate::kore_thread_testcancel() }; let control = unsafe { &ticket.as_ref().control }; - if control.retired.load(Ordering::Acquire) != 0 { + if control.retired.load(Ordering::SeqCst) != 0 { break; } - // Independent native workers need not own a doorbell. Use the - // controller's deadline park, which requires no peer wake capability. - if !kore::runtime::pace::cede() { + // A cancellation request is a flag, not a wake: a joiner that can be + // cancelled parks one slice at a time, as a cancellable futex wait + // does. Any other joiner parks until the controller retires the target. + let slice = crate::thread_local::cancellable() + .then(|| core::time::Duration::from_nanos(crate::thread_local::CANCEL_SLICE_NS)); + if kore::futex::wait(&control.retired, 0, slice) == kore::futex::WaitOutcome::Unsupported + // A native worker with no doorbell takes the deadline park, which + // needs no wake capability at all. + && !kore::runtime::pace::cede() + { return KORE_THREAD_EAGAIN; } } diff --git a/crates/ffi/src/thread_local.rs b/crates/ffi/src/thread_local.rs index d160bc3..64bd577 100644 --- a/crates/ffi/src/thread_local.rs +++ b/crates/ffi/src/thread_local.rs @@ -15,6 +15,12 @@ pub const KORE_THREAD_CANCEL_ENABLE: i32 = 0; pub const KORE_THREAD_CANCEL_DISABLE: i32 = 1; pub const KORE_THREAD_CANCEL_DEFERRED: i32 = 0; pub const KORE_THREAD_CANCEL_ASYNCHRONOUS: i32 = 1; +/// How long one park lasts at a cancellation point that waits: a request is +/// a flag no wake carries, so a cancellable wait re-checks it this often. +/// One millisecond is the cadence the native runtime parks an unreachable +/// waiter at (`kore-rt`'s `pace::CEDE_NS`): a cancellation is acted on within +/// one, and an idle cancellable waiter wakes no more often than that. +pub(crate) const CANCEL_SLICE_NS: u64 = 1_000_000; pub type KoreThreadKey = u64; pub type KoreThreadDestructor = Option; @@ -487,7 +493,7 @@ pub unsafe extern "C" fn pthread_setcanceltype(kind: i32, old: *mut i32) -> i32 /// Same contract as kore_thread_testcancel. #[cfg(all(feature = "c-stdlib", target_os = "none"))] #[unsafe(no_mangle)] -pub unsafe extern "C" fn pthread_testcancel() -> () { +pub unsafe extern "C" fn pthread_testcancel() { unsafe { crate::kore_thread_testcancel() } } diff --git a/docs/languages.md b/docs/languages.md index 07bd6ae..0086c3f 100644 --- a/docs/languages.md +++ b/docs/languages.md @@ -20,9 +20,9 @@ A Rust host attaches `Fs` using its real founding face, then moves it into `kore_ffi::Session`, after closing any external views opened before handoff. The host awaits `Session::run(step, user)` on the existing Kore executor. No second executor, global allocator or global filesystem root is -installed. The C/Zig callback is linked into that same application's image. -This is not yet an independent C `main`/CRT, registry-published static SDK, or a -POSIX syscall veneer. +installed. The C/Zig callback is linked into that same application's image. A C +program with no Rust host gets one from `libkore.so` ([below](#an-independent-c-program-libkoreso)). +Neither is a registry-published static SDK or a POSIX syscall veneer. Each callback receives an opaque context and the preceding completion. Initially that completion is `KORE_OP_START`. Queue one operation and return. Submission @@ -64,6 +64,189 @@ if requests (such as zero-byte reads) complete immediately. C/Zig callbacks must return promptly: the adapter does not preempt arbitrary foreign code inside a callback. +## An independent C program: `libkore.so` + +A C program a compiler writes on Kore (`tcc` at Kore's shell) has no Rust +host. `kore-c` is its runtime. A distribution builds `libkore.so` from a +one-line `cdylib` named `kore` over it, linked with `--soname=libkore.so +--no-undefined` (Kore's `user/runtime/libkore`). It ships the library beside +the headers, which `kore-c` publishes to that crate's build script as +`DEP_KORE_INCLUDE`, so they come from the same revision. A program links it with +`-lkore` — Kore's TCC links it into every program, its `-lc` reading a script +that joins `libkore.so` to the program's own `libc.a` — and gets: + +- every `kore.h` function, and the C names `kore/stdlib.h` and + `kore/pthread.h` alias as real symbols (`c-stdlib`), over the process heap + `kore-c` installs on the launch's `HEAP` supply; +- the rest of the C library the program's headers declare (Kore's hosted + ``, ``, `` and ``, beside TCC): + `kore-c-shim`'s `printf` family, every conversion C has but the wide ones and + every float digit exact, `stdout` and `stderr`, the `strto*` parsers, + `qsort`, `strerror`, the string routines and a per-thread `errno`, and + `kore-c`'s own `abort`; +- descriptors over the capabilities it holds (``, ``, + ``: `open`, `close`, `mmap`, `munmap`), and + `kore_fd_describe` (`kore/program.h`), described below; +- a host on thread 0, entered from the C library's start; +- `kore_fs_run(step, user, &error)` (`kore/program.h`), which runs a + `KoreFsStep` program as the program's filesystem session. + +The host is the shape `kore-lua`'s VM bridge has. `main` runs on a main +thread, a `DoorbellWorker` with a 1 MiB guarded stack +(`KORE_MAIN_STACK_BYTES`). Thread 0 is the process owner, and keeps the +executor and the launch's service faces. The two share one request cell and +one word. Every park is a doorbell park; nothing spins. + +- **Filesystem.** `kore_fs_run` from the main thread posts the session and + waits. The host attaches the launch's filesystem face on first use, rooted + at the directory the program was launched in, and runs `kore-ffi`'s + `Session` on its executor. The callbacks run there, as on any native + executor, while the caller waits: they must not depend on its + thread-local state, nor wait for another program thread — while one runs + the host serves nothing else, the streams included. A thread that cannot park (any but the main thread) + is refused, as is a launch with no face. A quarantined session can never + run again. +- **The streams.** `stdout` and `stderr` reach the launch's interactive + stream (`PROFILE_INTERACTIVE`), the terminal the program was run from, + which only thread 0 may publish into. Any thread's write goes whole into + one in-process ring under a futex lock, which a stream call holds across + all its writes (POSIX's `flockfile`): the holder is the ring's one + producer, so one call's output is never split by another thread's, and it + kicks the host only if the host armed itself before parking. Every wait + for room is bounded by the runtime's slice: the host kicks the main thread + when its room is free, and a pthread, which the host cannot reach, + re-reads the ring each slice. The host drains the + ring into the stream's records as the launcher has room, parks on the + stream's capacity when it has none — a slow terminal holds the program + back rather than losing its output — and wakes the writers waiting for + space, a pass at a time, serving the program's end and its sessions + between passes. The host's own writes (a filesystem step's callbacks) are + kept in order beside the ring, before any call another thread has in + progress. A launch without the stream has no sink, and a + write to either stream returns `EOF`. +- **The end.** The C library's `exit` and `_Exit` call + `__kore_program_exit(status, finalize)` after their `atexit` handlers, on + whatever thread called them. The first request wins and reaches the host, + since threads the program starts inherit a send into its wake channel. + - The host waits for a main thread that asked to end itself. + - It then raises kore-rt's process barrier, stopping every other thread + and running the finalizers only if none was still running. Once a + program has started a thread, `kore-ffi`'s thread controller is one of + them, so such a program's finalizers do not run; `atexit` handlers + always do. + - It commits the C status, the wire's `i32`, through the C library's own + exit. + - An end requested mid-session does not wait for the session. + - A panic in the library ends the program the same way, its record + published first. + - `abort` takes `_Exit`'s path with the aborted status, 134, what a shell + reports for a program `SIGABRT` ended: no `atexit` handler runs. + - An end the host's loop serves drains the ring and closes the stream + before the status, so the launcher shows all of the program's output + first. An end committed on the host itself (a callback's `exit` or + `abort`, a panic there) cannot wait: it relays what the launcher has room + for, and closes the stream if that was all of it. +- **Without a host.** A launch that cannot carry one (no channel space or + heap) runs the program on thread 0, as its C library alone would. Then the + filesystem session is refused, and only thread 0's exit ends the program. + +### Descriptors: C's spelling for a held capability + +A program's authority is the capabilities it was founded holding. Each sits +at a fixed place: the boot block names it by a role and an index, and the +capability is a slot of the thread's arena. `open` is the readable way to say +which one is meant, and nothing more: + +```c +int fd = open("/kore/memory", O_RDWR); +char *region = mmap(NULL, 1 << 20, PROT_READ | PROT_WRITE, MAP_PRIVATE, fd, 0); +... +munmap(region, 1 << 20); +close(fd); +``` + +- **The path is not a file.** `open` looks the string up in one static table + (`crates/c/src/fd.rs`) whose rows are a name and the bootstrap coordinate it + stands for. No filesystem is asked, and no name outside the table exists: + `ENOENT`. The first row is `/kore/memory`, `role::HEAP@0`, the memory supply + the allocator grows on. +- **It grants nothing.** `open` succeeds only for a capability the calling + thread holds (`ENXIO` otherwise), with access its rights cover (`EACCES`). + The only flags are the access modes. `close` forgets the number; the + capability stays the program's, and mappings made through it stand. +- **A descriptor stores the coordinate, never the capability.** Capabilities + are per thread on Kore, and `main` and every `pthread` have their own boot + block. Each use resolves the coordinate in the *calling* thread's. Numbers + are the process's, as C expects, from 3 (0 to 2 are the streams'), and no + authority crosses a thread through one. +- **`kore_fd_describe(fd, &info)`** reads what the calling thread holds + there: its kind (`KORE_FD_MEMORY`), what its rights permit, and for a + memory supply its whole length. + +`mmap` on the memory supply answers fresh, zeroed memory, as mapping +`/dev/zero` does elsewhere. It is three of kore-rt's verbs, the sequence a +Wasmtime engine's guest memory takes: a lease carved from the supply's top, +clear of the allocator, which grows from its bottom; an address from the +free part of the task's link window (`Rt::link_free_span`); the whole lease +mapped there. The rules follow from that: + +- **The caller never picks the address.** A hint is ignored, and `MAP_FIXED` + is `ENOTSUP`. +- **Read, or read and write.** `PROT_EXEC` is `ENOTSUP`: memory a program + wrote is never executable. `MAP_SHARED` and `MAP_PRIVATE` mean the same, + since the region is this program's alone. There is no offset, and no + `MAP_ANONYMOUS`: memory comes from a capability, so a descriptor names it. +- **Thread 0 maps, for whichever thread asked.** Every thread that may + allocate is founded holding a share of the supply, and `open` and `mmap` + hold the caller's access to that share's rights. A share carries no + `REVOKE`, though, and a region that can be unmapped again is a carve its + holder may revoke, so the caller posts its request to thread 0, which + holds the supply's root, and waits as a filesystem session does. A mapping + is therefore the process's: any thread may `munmap` it, whole, by its base + and its length. +- **`munmap` parks the region.** The supply's cursors and the window's only + advance, so carved bytes and placed addresses never return to them. The + mapping is removed and a stale access faults, but the program keeps the + lease and its address, and the next `mmap` of the same length takes it, + zeroed again, before anything new is carved. One size mapped and unmapped + in a loop uses one region. Ever-different sizes spend supply each time and + end with `ENOMEM`. A program holds sixteen regions, mapped and parked + together. + +**`main` gets its arguments.** The launcher's page carries the name it +resolved the program by and the rest of the command line. The host reads it +once, on thread 0 before the main thread exists, and splits the rest as every +Kore program that wants words does: a POSIX shell's quoting, without +expansion. `argv[0]` is the name, or the empty string when the launcher gave +none, `argv[argc]` is null, and the strings are the program's to change. A +line that does not split (an unclosed quote, a lone trailing backslash) is +not handed to `main`: the host says why on `stderr` and the program ends with +status 5 before it runs. `envp` is the empty vector, since a Kore task has no +environment. A launch that carried no page runs with `argc` zero. + +The C library's half is three weak references. Its start calls +`__kore_program_start(program, end)` when the program links libkore, asks +`__kore_program_arguments(&argv)` for the vector before it calls `main`, and +its exit calls `__kore_program_exit`. A program without libkore is unchanged. On +Kore those are the start files Kore ships with TCC. `tools/check-program.py` +(`make cross`, and CI per ISA) links the shared object as a distribution does +and checks it: + +- a library with no entry point; +- its soname, nothing needed or undefined; +- static initial-exec TLS only; +- exactly the declared ABI exported: `kore.h`, `kore/program.h`, the + `c-stdlib` names, the two lifecycle entry points, and the C library names + Kore's hosted headers declare (`C_LIBRARY`). + +Kore's `make keytest-tcc-x86` compiles a C program against these headers at +Kore's shell, links `libkore.so`, and runs it through every surface above. It +also compiles Kore's `hello.c`, written against the standard headers alone, +reads every line it prints off the terminal, and ends a program by `abort` +(Kore `res/TCC.md` §10). That is boot evidence; this repository's checks are +compiler and link evidence. Task preparation (`kore_task_*`) still needs a +Rust host to admit its batch: the independent host does not admit one. + ## Failure and lifetime Filesystem and provider errors have explicit generated foreign enums. Monotonic diff --git a/docs/provenance.md b/docs/provenance.md index 6554b1d..737162c 100644 --- a/docs/provenance.md +++ b/docs/provenance.md @@ -318,3 +318,36 @@ three ISAs. The Zig namespace and the `stdlib.h`/`pthread.h` aliases derive from that header's function list alone, so they are current as well. Not run locally: `zig fmt`, `test-foreign`, the rest of `bindings-check` and `kore-lua`'s Kore-target build; CI's host job runs them. + +## A C program's runtime, over Kore #1494's kore-rt + +The export records Kore `0018956a3aca19dc6acea8d28fb8ec9fadefcb09`, a commit +on MicroPerceptron/kore#1494's branch, which brings TinyCC to Kore's shell. +Eleven of the 26 commits since `7074bda2` reach the closure, which keeps its +26 packages and 382 files: + +- `kore-rt` carries what `kore-c`'s host needs: + - `startup::barrier`, the process barrier without the exit; + - managed threads that inherit their founder's wake send; + - `crash::report`, a panic's report without the thread's end; + - also a per-thread C `errno` word, guarded worker stacks sized in bytes, + and the verbs and bundle call a shell's private install takes. +- `kore-rt`'s thread controller, which `kore-ffi`'s pthreads run on, parks + on the futex and never polls (Kore `0018956a`). Its threads get their own + doorbells and the controller relays the wakes they cannot deliver to each + other. A thread's end reaches it through the exit hook `thread::exit` + runs, and its reap parks on a join arm. Here `kore-ffi`'s join parks on + its retirement word, which the controller wakes. `kore-c`'s main thread + holds a channel-space grant to mint the controller's doorbells from. +- `kore-abi` carries the install request and its replies, and the claimant + band sized by driver rows (`MAX_CLAIMANT_FACES`), which the device service + uses. +- The services' `boot.toml` digests were resealed three times. + +No exported manifest changed, so `native/Cargo.lock` did not. + +`kore-c` is new: the runtime of `libkore.so`, and the host of an independent +C program (`docs/languages.md`). It is the first libkore package a C program +on Kore links without a Rust host. Its shared object is a distribution's +one-line `cdylib` over it. `tools/check-program.py` links that object for the +three Kore ISAs and checks its shape and its exact exported ABI. diff --git a/extern/c/include/kore/program.h b/extern/c/include/kore/program.h new file mode 100644 index 0000000..b8e3f05 --- /dev/null +++ b/extern/c/include/kore/program.h @@ -0,0 +1,116 @@ +/* libkore.so: what an independent C program on Kore adds to kore.h. Link with -lkore. */ + +#ifndef KORE_PROGRAM_H +#define KORE_PROGRAM_H + +#pragma once + +/* Generated by cbindgen 0.29.4 from kore-c. Do not edit. */ + +#include "../kore.h" + +/** + * The stack `main` runs on: a guarded strip of the task's stack window, + * committed whole when the program starts. A frame deeper than this faults + * on the guard page below it, as a stack overflow does. + * + * The size of the strip `tcc`, a large C program whose depth its input + * decides, measured its need under Kore's runtime at and runs in; ISO C's + * own guarantee of 127 nested blocks (C11 §5.2.4.1) needs a small part of + * it. Larger strips come out of the 16 MiB window the program's SPMD gangs + * share, and would be committed at every launch. + */ +#define KORE_MAIN_STACK_BYTES (1 << 20) + +/** + * [`KoreFdInfo::kind`]: the memory supply the program's allocator grows + * on. `mmap` answers a fresh, zeroed region of it. + */ +#define KORE_FD_MEMORY 1 + +/** + * [`KoreFdInfo::access`]: the capability may be read. + */ +#define KORE_FD_READ (1 << 0) + +/** + * [`KoreFdInfo::access`]: the capability may be written. + */ +#define KORE_FD_WRITE (1 << 1) + +/** + * What a descriptor names, as the calling thread holds it + * ([`kore_fd_describe`]). + */ +typedef struct KoreFdInfo { + /** + * What kind of capability it is: `KORE_FD_MEMORY`. + */ + uint32_t kind; + /** + * What the capability's own rights permit this thread, as + * `KORE_FD_READ | KORE_FD_WRITE` bits: the rights it was founded with, + * which the access `open` asked for can only narrow. + */ + uint32_t access; + /** + * For `KORE_FD_MEMORY`, the supply's whole length in bytes: what the + * allocator and every mapping share, not what is still free. + */ + uint64_t length; +} KoreFdInfo; + +#ifdef __cplusplus +extern "C" { +#endif // __cplusplus + +/** + * Run `step` as this program's filesystem session, through the filesystem + * face its launch carried, rooted where it was launched (its working + * directory). The callbacks and their pointers obey `KoreFsStep`'s contract: + * each callback runs on the program's host thread, as on any native + * executor, while the caller waits, so a callback must not depend on the + * caller's thread-local state. + * + * Returns `KORE_STATUS_OK` once the callback program ended and every view it + * left open closed. `KORE_STATUS_QUARANTINED`: a reply was lost or + * uncertain, and no session can run again in this program. + * `KORE_STATUS_REFUSED`: a close was refused (`*error` names why, and the + * view stays open), the launch carried no filesystem face + * (`KORE_FS_ERROR_UNSUPPORTED`), or the caller is not the thread `main` runs + * on, which alone waits for the host. `KORE_STATUS_CAPACITY`: the native + * owner could not be formed. `KORE_STATUS_INVALID_ARGUMENT`: `step` is null. + * `*error` is written whenever `error` is not null. + * + * # Safety + * `step` obeys `KoreFsStep`'s pointer, serialization and no-unwind contract; + * `user` remains valid, and the callbacks' alone, until this returns. `error` + * is null or valid for one aligned write. + */ +KoreStatus kore_fs_run(KoreFsStep step, void *user, KoreFsError *error); + +/** + * Describe the capability behind `fd`, a descriptor `open` answered + * (``): `open` finds a capability this program was founded holding + * by its name, and this reads what was found. Nothing is granted or changed. + * + * Capabilities are held per thread, so the answer is the calling thread's: + * the same descriptor may describe a capability on the thread that opened + * it and be refused on one founded without it. + * + * Returns `KORE_STATUS_OK` with `*out` written. + * `KORE_STATUS_INVALID_ARGUMENT`: `out` is null. `KORE_STATUS_STALE_HANDLE`: + * `fd` is not an open descriptor. `KORE_STATUS_REFUSED`: the calling thread + * holds no such capability, or this is not a Kore program. A refusal leaves + * `*out` unchanged. + * + * # Safety + * `out` is null or valid for one aligned write. + */ +KoreStatus kore_fd_describe(int fd, struct KoreFdInfo *out); + +#ifdef __cplusplus +} // extern "C" +#endif // __cplusplus + +#endif /* KORE_PROGRAM_H */ diff --git a/native/Cargo.lock b/native/Cargo.lock index 8eef346..22b245d 100644 --- a/native/Cargo.lock +++ b/native/Cargo.lock @@ -247,6 +247,14 @@ dependencies = [ "kore-window", ] +[[package]] +name = "kore-c-shim" +version = "0.1.0" +dependencies = [ + "kore-rt", + "rustc_apfloat", +] + [[package]] name = "kore-collections" version = "0.1.0" @@ -583,6 +591,16 @@ dependencies = [ "bitflags", ] +[[package]] +name = "rustc_apfloat" +version = "0.2.3+llvm-462a31f5a5ab" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "486c2179b4796f65bfe2ee33679acf0927ac83ecf583ad6c91c3b4570911b9ad" +dependencies = [ + "bitflags", + "smallvec", +] + [[package]] name = "rustix" version = "1.1.6" @@ -622,6 +640,12 @@ version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" +[[package]] +name = "smallvec" +version = "1.16.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "149309498a5f7334b12ddae0cad8aba81ce785fe91755068867e017cf98fee46" + [[package]] name = "stable_deref_trait" version = "1.2.1" diff --git a/native/Cargo.toml b/native/Cargo.toml index 20a77a3..3164724 100644 --- a/native/Cargo.toml +++ b/native/Cargo.toml @@ -3,6 +3,7 @@ resolver = "3" members = [ "crates/abi", "crates/app", + "crates/c-shim", "crates/collections", "crates/dagfs", "crates/driver", diff --git a/native/crates/abi/src/command.rs b/native/crates/abi/src/command.rs index d62b18c..f34eca2 100644 --- a/native/crates/abi/src/command.rs +++ b/native/crates/abi/src/command.rs @@ -7,14 +7,22 @@ use crate::image::{ImageName, NAME_BYTES}; -/// Snapshot wire version. -pub const COMMAND_SNAPSHOT_ABI: u8 = 1; +/// Snapshot wire version. 2: an entry may carry no usage. A launcher +/// carries the usage of the rows its command line names and only the name +/// and summary of the rest ([`CommandSnapshotBuilder::push_listed`]), so the +/// record grows with the catalogue's names and summaries rather than with +/// every row's usage, and has room for the programs a task installs. A `1` +/// reader refuses an empty usage. +pub const COMMAND_SNAPSHOT_ABI: u8 = 2; /// The largest task-local command set accepted by the decoder. Raised /// from 32 when the desktop's `editor` and `power-ui` rows joined the two /// native filesystem verbs, and from 40 when Files and the documents editor -/// became two rows; the byte budget (`PROCESS_CONTEXT_BYTES`) is the other -/// bound, proven by exec's host test over the real catalogue. -pub const MAX_COMMAND_SNAPSHOT_ENTRIES: usize = 48; +/// became two rows; and from 48 for the programs a task installs, which +/// its launcher lists beside the catalogue (47 rows and a shell's share of +/// the installed pool no longer fit 48). The byte budget +/// (`PROCESS_CONTEXT_BYTES`) is the other bound; exec's host test proves +/// both over the real catalogue with that share installed. +pub const MAX_COMMAND_SNAPSHOT_ENTRIES: usize = 64; /// The launcher holds a power-control face and therefore handles these two /// privileged controls in-process. pub const COMMAND_SNAPSHOT_SYSTEM_POWER: u8 = 1 << 0; @@ -38,6 +46,8 @@ pub enum CommandSnapshotError { #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub struct CommandDescription<'a> { pub local_name: &'a str, + /// The command's usage, or empty when the launcher carried none for + /// this row. pub usage: &'a str, pub summary: &'a str, } @@ -67,13 +77,35 @@ impl<'a> CommandSnapshotBuilder<'a> { }) } - /// Append one local namespace row. The executable capability paired with - /// this metadata deliberately is not accepted by this API. + /// Append one local namespace row with its usage. The executable + /// capability paired with this metadata deliberately is not accepted by + /// this API. pub fn push( &mut self, local_name: &ImageName, usage: &[u8], summary: &[u8], + ) -> Result<(), CommandSnapshotError> { + validate_text(usage, false)?; + self.append(local_name, usage, summary) + } + + /// Append one local namespace row without its usage: a row the reader + /// lists but was not asked to describe. + pub fn push_listed( + &mut self, + local_name: &ImageName, + summary: &[u8], + ) -> Result<(), CommandSnapshotError> { + self.append(local_name, &[], summary) + } + + /// `usage` is valid text, or empty for a row that carries none. + fn append( + &mut self, + local_name: &ImageName, + usage: &[u8], + summary: &[u8], ) -> Result<(), CommandSnapshotError> { if usize::from(self.count) >= MAX_COMMAND_SNAPSHOT_ENTRIES { return Err(CommandSnapshotError::TooManyEntries); @@ -82,7 +114,6 @@ impl<'a> CommandSnapshotBuilder<'a> { .as_str() .ok_or(CommandSnapshotError::InvalidText)?; validate_text(name.as_bytes(), true)?; - validate_text(usage, false)?; validate_text(summary, false)?; if name.len() > u8::MAX as usize || usage.len() > u8::MAX as usize @@ -260,7 +291,10 @@ fn decode_entry( cursor += usage_len; let summary = &bytes[cursor..cursor + summary_len]; validate_text(name, true)?; - validate_text(usage, false)?; + // A row the launcher only listed carries no usage. + if !usage.is_empty() { + validate_text(usage, false)?; + } validate_text(summary, false)?; Ok(( CommandDescription { @@ -318,6 +352,47 @@ mod tests { assert!(snapshot.find(b"touch").is_none()); } + #[test] + fn a_listed_row_carries_its_name_and_summary_and_no_usage() { + let mut bytes = [0u8; 256]; + let mut builder = CommandSnapshotBuilder::new(&mut bytes, 0).unwrap(); + builder + .push_listed(&image("hello"), b"dynamic-linker witness") + .unwrap(); + builder + .push(&image("ls"), b"ls [-l] [path]", b"list directory entries") + .unwrap(); + let listed_only = 4 + "hello".len() + "dynamic-linker witness".len(); + let len = builder.finish(); + let snapshot = CommandSnapshot::parse(&bytes[..len]).unwrap(); + assert_eq!( + snapshot.find(b"hello"), + Some(CommandDescription { + local_name: "hello", + usage: "", + summary: "dynamic-linker witness", + }) + ); + assert_eq!(snapshot.find(b"ls").unwrap().usage, "ls [-l] [path]"); + // The usage a listed row leaves out is exactly what it saves. + assert_eq!( + len, + 8 + listed_only + 4 + "ls".len() + "ls [-l] [path]".len() + 22 + ); + // A row that is described carries a usage: `push` takes no empty one, + // and a summary is never optional. + let mut bytes = [0u8; 64]; + let mut builder = CommandSnapshotBuilder::new(&mut bytes, 0).unwrap(); + assert_eq!( + builder.push(&image("one"), b"", b"first"), + Err(CommandSnapshotError::InvalidText) + ); + assert_eq!( + builder.push_listed(&image("one"), b""), + Err(CommandSnapshotError::InvalidText) + ); + } + #[test] fn duplicate_and_malformed_rows_fail_closed() { let mut bytes = [0u8; 128]; diff --git a/native/crates/abi/src/device.rs b/native/crates/abi/src/device.rs index 60b8e89..463489e 100644 --- a/native/crates/abi/src/device.rs +++ b/native/crates/abi/src/device.rs @@ -21,8 +21,10 @@ mod census; mod i2c_admission; +mod state; pub use census::*; pub use i2c_admission::*; +pub use state::*; /// An I/O virtual address — a device-domain address translated by the /// IOMMU. **Never** a physical address: userland holds only this, so it can @@ -89,6 +91,26 @@ pub enum DeviceSpecies { /// its banks. pub const MAX_ROOT_COMPLEXES: usize = 8; +/// Claimant faces one device broker can mint: one per driver row, the +/// founder minting face `n` for driver row `n` (`MintFace`, +/// `res/DEVICE_SERVICE.md` §10). Five today: input, block, accel, net and +/// Xe. +/// +/// The bound is shared, not chosen. The kernel founder seats driver row +/// `n`'s one executor bell at a fixed slot of its common pin island and +/// asserts the island holds this many. Exec's manifest gate refuses a +/// driver row past it. The broker sizes its face table, its claimant band +/// ([`CLAIMANT_FACE_SUPPLY_BYTES`](crate::task::CLAIMANT_FACE_SUPPLY_BYTES)) +/// and its landing band (two base-arena slots per face, below the +/// startup-link band, which it asserts) from it. Raising it costs one +/// ring stride per broker, two broker slots and one founder bell slot per +/// face; the founder's pins are the first of those to run out. +/// +/// Deliberately not the image service's face count: that one grows with +/// the executable catalogue, and a broker band sized by it would grow with +/// every program the system can run. +pub const MAX_CLAIMANT_FACES: usize = 5; + impl DeviceSpecies { /// Decode the wire byte; `None` for an unknown species (fail-closed — /// a broker that cannot name the species must not derive). diff --git a/native/crates/abi/src/device/state.rs b/native/crates/abi/src/device/state.rs new file mode 100644 index 0000000..5dab25c --- /dev/null +++ b/native/crates/abi/src/device/state.rs @@ -0,0 +1,922 @@ +//! The read-only **live state** of the PCI functions the census +//! ([`super::census`]) lists, and the shape of one function's BARs. +//! +//! The census is boot-frozen identity. This is what changes after boot, and +//! what a driver writer asks before touching a device: whether a driver +//! claimed the function, which founded row drives its DMA, which BARs exist, +//! which of them a driver may map and which are mapped now, and how the +//! function can interrupt. A BAR is its index, its size and the length a +//! driver may map; **no address crosses** (res/DEVICE_SERVICE.md §1.1), so +//! nothing here says where a BAR lives. +//! +//! Descriptions only, as a census row is: a state grants no inventory, +//! claim, mapping or route, and reading one changes nothing. It is a +//! snapshot. A claim can land or a route can be released the moment after +//! the kernel answers, so no consumer may treat it as permission. +//! +//! The path is the census's: the kernel answers one +//! [`SystemMetric::DeviceState`](crate::session::SystemMetric::DeviceState) +//! or [`SystemMetric::DeviceBar`](crate::session::SystemMetric::DeviceBar) +//! observation on the `SystemObserve` capability the monitor holds; the +//! monitor samples them on each request (never cached, since they are live) +//! and vends [`DeviceStatePage`]s and one [`DeviceBarTable`]; exec relays +//! them to a launcher; the launcher stamps a [`DeviceInventory`] into the +//! process-context prefix of an executable whose bootstrap row declares it. + +use zerocopy::byteorder::{LittleEndian, U16, U64}; +use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout, Unaligned}; + +use super::census::{DeviceCensus, DeviceCensusPage, MAX_DEVICE_CENSUS_ROWS}; + +/// BARs a PCI function has (PCI Local Bus Specification 3.0 §6.2.5: six +/// Base Address Registers in a type 0 header). +pub const DEVICE_BARS: usize = 6; +/// Every BAR's bit, for the three per-BAR bitmaps of a [`DeviceState`]. +pub const DEVICE_BAR_MASK: u8 = (1 << DEVICE_BARS) - 1; +/// `DeviceState::driver` when no founded row drives the function's DMA. +pub const DEVICE_ROW_NONE: u8 = 0xFF; +/// States per vended page: 8 header bytes + 128 = 136 bytes, the census +/// page's size, so a page rides the monitor and exec reply wires as one. +pub const DEVICE_STATE_PAGE_ROWS: usize = 16; +/// Pages the states of a full census span. +pub const DEVICE_STATE_PAGES: usize = MAX_DEVICE_CENSUS_ROWS.div_ceil(DEVICE_STATE_PAGE_ROWS); +/// Layout version of [`DeviceInventory`]. A zeroed record reads 0 and is +/// *unavailable*. +pub const DEVICE_INVENTORY_ABI: u16 = 1; +/// [`SystemMetric::DeviceBar`](crate::session::SystemMetric::DeviceBar)'s +/// operand is `index | bar << DEVICE_BAR_OPERAND_SHIFT`: the catalog index +/// in the low 13 bits (every index a census can hold fits, asserted below), +/// the BAR number in the top three. +pub const DEVICE_BAR_OPERAND_SHIFT: u32 = 13; + +/// The operand naming BAR `bar` of the function at catalog `index`; `None` +/// when either is out of the operand's range. +pub const fn device_bar_operand(index: u16, bar: u8) -> Option { + if index >> DEVICE_BAR_OPERAND_SHIFT != 0 || bar as usize >= DEVICE_BARS { + return None; + } + Some(index | (bar as u16) << DEVICE_BAR_OPERAND_SHIFT) +} + +/// The `(index, bar)` a [`device_bar_operand`] names; `None` for a BAR +/// number no function has. +pub const fn device_bar_of(operand: u16) -> Option<(u16, u8)> { + let bar = (operand >> DEVICE_BAR_OPERAND_SHIFT) as u8; + if bar as usize >= DEVICE_BARS { + return None; + } + Some((operand & ((1 << DEVICE_BAR_OPERAND_SHIFT) - 1), bar)) +} + +/// One function's live state. 8 bytes, frozen: one observation's value +/// word, byte for byte. +#[repr(C)] +#[derive( + Clone, Copy, Debug, PartialEq, Eq, FromBytes, IntoBytes, KnownLayout, Immutable, Unaligned, +)] +pub struct DeviceState { + /// [`Self::PRESENT`] and the bits below it. + pub flags: u8, + /// The founded row (the founding record's index) whose task activated + /// DMA on this function, or [`DEVICE_ROW_NONE`]. The kernel's own + /// shutdown ledger, so it names who drives the device rather than who + /// claimed it: a function a row claimed and delegated reads the row it + /// was delegated to, and a driver that never activates DMA reads none. + pub driver: u8, + /// Bit `n`: BAR `n` exists as memory the kernel resolved. An I/O BAR is + /// not one: no driver may reach port space through a BAR. + pub bars_present: u8, + /// Bit `n`: a driver may map a prefix of BAR `n`. + pub bars_mappable: u8, + /// Bit `n`: BAR `n` is mapped by a driver now. + pub bars_mapped: u8, + /// Vectors the function's MSI capability can deliver (1, 2, 4, 8, 16 or + /// 32); 0 when it has none. + pub msi_vectors: u8, + /// Entries in the function's MSI-X table; 0 when it has none. + pub msix_entries: U16, +} + +impl DeviceState { + /// A populated state; clear marks the zero state. + pub const PRESENT: u8 = 1 << 0; + /// A driver holds the function: its claim is committed. + pub const CLAIMED: u8 = 1 << 1; + /// An interrupt route is established on the function now. + pub const ROUTED: u8 = 1 << 2; + /// A driver can establish a remapped MSI-family route (MSI or MSI-X), + /// the only kind userland may hold. + pub const ROUTE_MSI: u8 = 1 << 3; + /// A driver can establish a remapped MSI-X route. + pub const ROUTE_MSIX: u8 = 1 << 4; + /// The function has a legacy interrupt pin; [`Self::pin`] says which. + pub const HAS_PIN: u8 = 1 << 5; + /// Bits 6 and 7: the legacy pin, 0 for INTA through 3 for INTD. + pub const PIN_SHIFT: u32 = 6; + + pub const EMPTY: Self = Self { + flags: 0, + driver: 0, + bars_present: 0, + bars_mappable: 0, + bars_mapped: 0, + msi_vectors: 0, + msix_entries: U16::ZERO, + }; + + pub const fn is_present(&self) -> bool { + self.flags & Self::PRESENT != 0 + } + + pub const fn claimed(&self) -> bool { + self.flags & Self::CLAIMED != 0 + } + + /// The legacy interrupt pin, 0 for INTA through 3 for INTD. + pub const fn pin(&self) -> Option { + if self.flags & Self::HAS_PIN == 0 { + return None; + } + Some(self.flags >> Self::PIN_SHIFT) + } + + /// The founded row driving the function's DMA. + pub const fn driver(&self) -> Option { + if self.driver == DEVICE_ROW_NONE { + None + } else { + Some(self.driver) + } + } + + pub const fn validate(&self) -> bool { + if !self.is_present() { + return self.flags == 0 + && self.driver == 0 + && self.bars_present == 0 + && self.bars_mappable == 0 + && self.bars_mapped == 0 + && self.msi_vectors == 0 + && self.msix_entries.get() == 0; + } + // A pin without its flag would be a second spelling of "no pin". + (self.flags & Self::HAS_PIN != 0 || self.flags >> Self::PIN_SHIFT == 0) + && self.bars_present & !DEVICE_BAR_MASK == 0 + // Only a BAR that exists can be mapped, and only one a driver + // may map is. + && self.bars_mappable & !self.bars_present == 0 + && self.bars_mapped & !self.bars_mappable == 0 + && (self.msi_vectors == 0 + || (self.msi_vectors.is_power_of_two() && self.msi_vectors <= 32)) + && self.msix_entries.get() <= super::DeviceRecord::MAX_MSIX_TABLE_ENTRIES + } + + /// The state as one `SystemObserve` report, `(value, aux)`, with the + /// function's catalog `index` echoed in `aux[63:48]` so a reader can + /// prove the report answers the function it asked for. + pub const fn to_observation(&self, index: u16) -> (u64, u64) { + let value = u64::from_le_bytes([ + self.flags, + self.driver, + self.bars_present, + self.bars_mappable, + self.bars_mapped, + self.msi_vectors, + self.msix_entries.get().to_le_bytes()[0], + self.msix_entries.get().to_le_bytes()[1], + ]); + (value, (index as u64) << 48) + } + + /// Inverse of [`to_observation`](Self::to_observation); `None` when the + /// echoed index is not `index`, the rest of `aux` is not zero, or the + /// state is not a valid populated one. + pub const fn from_observation(index: u16, value: u64, aux: u64) -> Option { + if aux != (index as u64) << 48 { + return None; + } + let bytes = value.to_le_bytes(); + let state = Self { + flags: bytes[0], + driver: bytes[1], + bars_present: bytes[2], + bars_mappable: bytes[3], + bars_mapped: bytes[4], + msi_vectors: bytes[5], + msix_entries: U16::new(u16::from_le_bytes([bytes[6], bytes[7]])), + }; + if state.is_present() && state.validate() { + Some(state) + } else { + None + } + } +} + +/// One vended page of states, row for row beside the census's rows. +/// 136 bytes, frozen. +#[repr(C)] +#[derive( + Clone, Copy, Debug, PartialEq, Eq, FromBytes, IntoBytes, KnownLayout, Immutable, Unaligned, +)] +pub struct DeviceStatePage { + /// Page ordinal, `0..DEVICE_STATE_PAGES`. + pub page: u8, + /// Populated states in this page, `0..=DEVICE_STATE_PAGE_ROWS`. + pub rows_valid: u8, + /// States the whole table holds: the census's `count`. + pub count: U16, + pub reserved: [u8; 4], + pub states: [DeviceState; DEVICE_STATE_PAGE_ROWS], +} + +impl DeviceStatePage { + pub const EMPTY: Self = Self { + page: 0, + rows_valid: 0, + count: U16::ZERO, + reserved: [0; 4], + states: [DeviceState::EMPTY; DEVICE_STATE_PAGE_ROWS], + }; + + /// Page `page` of a table of `count` states, empty: the assembler's + /// start. `None` past the table's last page, though a table of no states + /// still has page 0. + pub const fn new(page: u8, count: u16) -> Option { + let count = if count as usize > MAX_DEVICE_CENSUS_ROWS { + MAX_DEVICE_CENSUS_ROWS + } else { + count as usize + }; + let pages = count.div_ceil(DEVICE_STATE_PAGE_ROWS); + let last = if pages == 0 { 0 } else { pages - 1 }; + if page as usize > last { + return None; + } + Some(Self { + page, + rows_valid: 0, + count: U16::new(count as u16), + reserved: [0; 4], + states: [DeviceState::EMPTY; DEVICE_STATE_PAGE_ROWS], + }) + } + + /// The catalog index of this page's first state. + pub const fn first(&self) -> usize { + self.page as usize * DEVICE_STATE_PAGE_ROWS + } + + /// States this page holds once complete. + pub const fn expected(&self) -> usize { + let left = (self.count.get() as usize).saturating_sub(self.first()); + if left > DEVICE_STATE_PAGE_ROWS { + DEVICE_STATE_PAGE_ROWS + } else { + left + } + } + + /// Append the next state, from one kernel observation of the function at + /// the catalog index this page reaches next; `false` when the page is + /// full or the report does not decode. + pub fn push(&mut self, value: u64, aux: u64) -> bool { + let at = self.rows_valid as usize; + if at >= self.expected() { + return false; + } + let Some(state) = DeviceState::from_observation((self.first() + at) as u16, value, aux) + else { + return false; + }; + self.states[at] = state; + self.rows_valid += 1; + true + } + + pub fn validate(&self) -> bool { + let count = self.count.get() as usize; + if self.page as usize >= DEVICE_STATE_PAGES + || count > MAX_DEVICE_CENSUS_ROWS + || self.reserved != [0; 4] + || self.rows_valid as usize != self.expected() + { + return false; + } + let valid = self.rows_valid as usize; + self.states + .iter() + .enumerate() + .all(|(i, state)| state.validate() && state.is_present() == (i < valid)) + } +} + +/// One BAR's shape. 16 bytes, frozen. Lengths only: where the BAR lives is +/// not here and never will be. +#[repr(C)] +#[derive( + Clone, Copy, Debug, PartialEq, Eq, FromBytes, IntoBytes, KnownLayout, Immutable, Unaligned, +)] +pub struct DeviceBar { + /// The BAR's whole size in bytes; 0 for a BAR the function has not. + pub size: U64, + /// Bytes from the BAR's start a driver may map: the whole BAR, or the + /// part before the page its MSI-X table or pending bits begin in, which + /// stay the kernel's. 0 when a driver may map none of it. + pub mappable: U64, +} + +impl DeviceBar { + pub const EMPTY: Self = Self { + size: U64::ZERO, + mappable: U64::ZERO, + }; + + /// Bits of `aux` the mappable length takes; the BAR number is echoed + /// above them. + const AUX_LENGTH_BITS: u32 = 56; + + pub const fn is_present(&self) -> bool { + self.size.get() != 0 + } + + pub const fn validate(&self) -> bool { + self.mappable.get() <= self.size.get() + } + + /// The BAR as one `SystemObserve` report, `(value, aux)`: its size, and + /// its mappable length with the BAR number echoed in `aux[58:56]`. + /// `None` when a length does not leave the echo room, which no BAR of a + /// machine with a 56-bit physical address space or less can reach. + pub const fn to_observation(&self, bar: u8) -> Option<(u64, u64)> { + if self.mappable.get() >> Self::AUX_LENGTH_BITS != 0 || bar as usize >= DEVICE_BARS { + return None; + } + Some(( + self.size.get(), + self.mappable.get() | (bar as u64) << Self::AUX_LENGTH_BITS, + )) + } + + /// Inverse of [`to_observation`](Self::to_observation); `None` when the + /// echoed BAR is not `bar` or more can be mapped than the BAR holds. + pub const fn from_observation(bar: u8, value: u64, aux: u64) -> Option { + if aux >> Self::AUX_LENGTH_BITS != bar as u64 { + return None; + } + let shape = Self { + size: U64::new(value), + mappable: U64::new(aux & ((1 << Self::AUX_LENGTH_BITS) - 1)), + }; + if shape.validate() { Some(shape) } else { None } + } +} + +/// The BARs of one function. 104 bytes, frozen. +#[repr(C)] +#[derive( + Clone, Copy, Debug, PartialEq, Eq, FromBytes, IntoBytes, KnownLayout, Immutable, Unaligned, +)] +pub struct DeviceBarTable { + /// The function's catalog index: its row in the census. + pub index: U16, + /// 1 for a populated table; 0 marks the zero table. + pub present: u8, + pub reserved: [u8; 5], + pub bars: [DeviceBar; DEVICE_BARS], +} + +impl DeviceBarTable { + pub const EMPTY: Self = Self { + index: U16::ZERO, + present: 0, + reserved: [0; 5], + bars: [DeviceBar::EMPTY; DEVICE_BARS], + }; + + /// The table of the function at catalog `index`, every BAR absent: the + /// assembler's start. + pub const fn new(index: u16) -> Self { + Self { + index: U16::new(index), + present: 1, + reserved: [0; 5], + bars: [DeviceBar::EMPTY; DEVICE_BARS], + } + } + + pub const fn is_present(&self) -> bool { + self.present == 1 + } + + /// Set BAR `bar` from one kernel observation; `false` when the report + /// does not decode. + pub fn set_bar(&mut self, bar: u8, value: u64, aux: u64) -> bool { + let (Some(slot), Some(shape)) = ( + self.bars.get_mut(bar as usize), + DeviceBar::from_observation(bar, value, aux), + ) else { + return false; + }; + *slot = shape; + true + } + + pub fn validate(&self) -> bool { + match self.present { + 0 => *self == Self::EMPTY, + 1 => self.reserved == [0; 5] && self.bars.iter().all(DeviceBar::validate), + _ => false, + } + } +} + +/// What a device tool is launched with: the census, each of its rows' live +/// state, and the BARs of the one function the command line named, if it +/// named one the census holds. 3008 bytes, inside the process-context +/// prefix (asserted below). +#[repr(C)] +#[derive( + Clone, Copy, Debug, PartialEq, Eq, FromBytes, IntoBytes, KnownLayout, Immutable, Unaligned, +)] +pub struct DeviceInventory { + /// [`DEVICE_INVENTORY_ABI`]; 0 = unavailable. + pub abi: U16, + pub reserved: [u8; 6], + pub census: DeviceCensus, + /// `states[i]` is the live state of `census.rows[i]`, for every row the + /// census holds; the zero state past them. + pub states: [DeviceState; MAX_DEVICE_CENSUS_ROWS], + /// The named function's BARs, or the zero table. + pub bars: DeviceBarTable, +} + +impl DeviceInventory { + pub const EMPTY: Self = Self { + abi: U16::ZERO, + reserved: [0; 6], + census: DeviceCensus::EMPTY, + states: [DeviceState::EMPTY; MAX_DEVICE_CENSUS_ROWS], + bars: DeviceBarTable::EMPTY, + }; + + /// An available inventory over `census`, with no state and no BARs yet. + pub const fn new(census: DeviceCensus) -> Self { + Self { + abi: U16::new(DEVICE_INVENTORY_ABI), + reserved: [0; 6], + census, + states: [DeviceState::EMPTY; MAX_DEVICE_CENSUS_ROWS], + bars: DeviceBarTable::EMPTY, + } + } + + pub const fn available(&self) -> bool { + self.abi.get() == DEVICE_INVENTORY_ABI + } + + /// Fold one vended page of states in; `false`, changing nothing, when + /// the page does not validate or is not a page of this census's table. + pub fn insert_states(&mut self, page: &DeviceStatePage) -> bool { + if !self.available() || !page.validate() || page.count != self.census.count { + return false; + } + let first = page.first(); + let valid = page.rows_valid as usize; + self.states[first..first + valid].copy_from_slice(&page.states[..valid]); + true + } + + /// Whether every census row has its state. + pub fn complete(&self) -> bool { + let count = self.census.rows().len(); + self.available() && self.states[..count].iter().all(DeviceState::is_present) + } + + pub fn validate(&self) -> bool { + if self.abi.get() == 0 { + return *self == Self::EMPTY; + } + let count = self.census.rows().len(); + self.abi.get() == DEVICE_INVENTORY_ABI + && self.reserved == [0; 6] + && self.census.available() + && self.census.validate() + && self + .states + .iter() + .enumerate() + .all(|(i, state)| state.validate() && (i < count || !state.is_present())) + && self.bars.validate() + && (!self.bars.is_present() || (self.bars.index.get() as usize) < count) + } +} + +/// Bytes of the device payload a monitor or exec reply carries. +pub const DEVICE_PAGE_BYTES: usize = 136; + +/// The device payload slot of the monitor's and exec's reply wires: one +/// census page, one page of states, or one function's BARs, as the reply's +/// tag says, and zero on every other reply. Bytes, so the three records +/// share the slot without one posing as another: each accessor decodes and +/// validates its own. +#[repr(transparent)] +#[derive( + Clone, Copy, Debug, PartialEq, Eq, FromBytes, IntoBytes, KnownLayout, Immutable, Unaligned, +)] +pub struct DevicePage(pub [u8; DEVICE_PAGE_BYTES]); + +impl DevicePage { + pub const EMPTY: Self = Self([0; DEVICE_PAGE_BYTES]); + + pub fn of_census(page: &DeviceCensusPage) -> Self { + let mut out = Self::EMPTY; + out.0.copy_from_slice(page.as_bytes()); + out + } + + /// The census page this slot carries, validated. + pub fn census(&self) -> Option { + DeviceCensusPage::read_from_bytes(&self.0) + .ok() + .filter(DeviceCensusPage::validate) + } + + pub fn of_states(page: &DeviceStatePage) -> Self { + let mut out = Self::EMPTY; + out.0.copy_from_slice(page.as_bytes()); + out + } + + /// The page of states this slot carries, validated. + pub fn states(&self) -> Option { + DeviceStatePage::read_from_bytes(&self.0) + .ok() + .filter(DeviceStatePage::validate) + } + + pub fn of_bars(table: &DeviceBarTable) -> Self { + let mut out = Self::EMPTY; + out.0[..core::mem::size_of::()].copy_from_slice(table.as_bytes()); + out + } + + /// The BAR table this slot carries: a populated, valid one, with the + /// slot's remaining bytes zero. + pub fn bars(&self) -> Option { + let (table, rest) = DeviceBarTable::read_from_prefix(&self.0).ok()?; + (table.is_present() && table.validate() && rest.iter().all(|byte| *byte == 0)) + .then_some(table) + } +} + +/// A PCI function's address as a person writes it: `ssss:bb:dd.f`, or +/// `bb:dd.f` in segment 0, every field hexadecimal but the function, which +/// is one digit `0` to `7`. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct PciAddress { + pub segment: u16, + /// `bus << 8 | device << 3 | function`, a census row's `locator`. + pub locator: u16, +} + +impl PciAddress { + /// Bytes of the longest spelling, `ssss:bb:dd.f`. + pub const MAX_TEXT: usize = 12; + + /// Parse `text`, the whole of it. `None` unless it is exactly an + /// address: a launcher looks for one among a command line's words, so a + /// word that merely resembles one must not name a device. + pub fn parse(text: &[u8]) -> Option { + let (rest, function) = text.split_last_chunk::<2>()?; + let [b'.', function @ b'0'..=b'7'] = *function else { + return None; + }; + let mut fields = rest.rsplitn(3, |byte| *byte == b':'); + let device = hex_field(fields.next()?, 2)?; + let bus = hex_field(fields.next()?, 2)?; + let segment = match fields.next() { + Some(field) => hex_field(field, 4)?, + None => 0, + }; + if device > 0x1F { + return None; + } + Some(Self { + segment, + locator: bus << 8 | device << 3 | (function - b'0') as u16, + }) + } + + /// The census row this address names. + pub fn row_in(&self, census: &DeviceCensus) -> Option { + census + .rows() + .iter() + .position(|row| row.segment.get() == self.segment && row.locator.get() == self.locator) + .map(|index| index as u16) + } +} + +/// `field` as exactly `digits` hexadecimal digits. +fn hex_field(field: &[u8], digits: usize) -> Option { + if field.len() != digits { + return None; + } + field.iter().try_fold(0u16, |value, byte| { + let digit = (*byte as char).to_digit(16)?; + Some(value << 4 | digit as u16) + }) +} + +const _: () = assert!(core::mem::size_of::() == 8); +const _: () = assert!(core::mem::size_of::() == 8 + 8 * DEVICE_STATE_PAGE_ROWS); +const _: () = assert!( + core::mem::size_of::() == core::mem::size_of::() +); +const _: () = assert!(core::mem::size_of::() == DEVICE_PAGE_BYTES); +const _: () = assert!(core::mem::size_of::() == DEVICE_PAGE_BYTES); +const _: () = assert!(core::mem::size_of::() <= DEVICE_PAGE_BYTES); +const _: () = assert!(core::mem::size_of::() == 16); +const _: () = assert!(core::mem::size_of::() == 8 + 16 * DEVICE_BARS); +const _: () = assert!(core::mem::size_of::() == 3008); +const _: () = + assert!(core::mem::size_of::() <= crate::process::PROCESS_CONTEXT_BYTES); +const _: () = assert!(DEVICE_STATE_PAGES <= u8::MAX as usize); +// Every index a census row can carry fits the BAR operand beside the BAR. +const _: () = assert!(MAX_DEVICE_CENSUS_ROWS <= 1 << DEVICE_BAR_OPERAND_SHIFT); +const _: () = assert!(DEVICE_BARS <= 1 << (u16::BITS - DEVICE_BAR_OPERAND_SHIFT)); + +#[cfg(test)] +mod tests { + use super::*; + use crate::device::DeviceCensusRow; + + fn state(flags: u8) -> DeviceState { + DeviceState { + flags: DeviceState::PRESENT | flags, + driver: DEVICE_ROW_NONE, + bars_present: 0b01_0001, + bars_mappable: 0b00_0001, + bars_mapped: 0, + msi_vectors: 1, + msix_entries: U16::new(0), + } + } + + fn census(count: u16) -> DeviceCensus { + let mut census = DeviceCensus::new(count, 1); + for index in 0..count { + census.rows[index as usize] = DeviceCensusRow { + locator: U16::new(index << 3), + present: 1, + ..DeviceCensusRow::EMPTY + }; + } + census + } + + #[test] + fn a_state_round_trips_one_observation() { + let mut state = state(DeviceState::CLAIMED | DeviceState::ROUTE_MSI | DeviceState::HAS_PIN); + state.flags |= 2 << DeviceState::PIN_SHIFT; + state.driver = 7; + state.msix_entries = U16::new(0); + let (value, aux) = state.to_observation(41); + assert_eq!(DeviceState::from_observation(41, value, aux), Some(state)); + assert_eq!(state.pin(), Some(2)); + assert_eq!(state.driver(), Some(7)); + assert!(state.claimed()); + // Another function's report, or one with stray auxiliary bits, is + // not this function's state. + assert_eq!(DeviceState::from_observation(40, value, aux), None); + assert_eq!(DeviceState::from_observation(41, value, aux | 1), None); + } + + #[test] + fn an_impossible_state_is_refused() { + let ok = state(0); + assert!(ok.validate()); + // The zero state is the empty row, never a kernel answer. + assert!(DeviceState::EMPTY.validate()); + assert_eq!(DeviceState::from_observation(0, 0, 0), None); + for broken in [ + DeviceState { + bars_present: 0x40, + ..ok + }, + DeviceState { + bars_mappable: 0b10, + ..ok + }, + DeviceState { + bars_mapped: 0b01_0000, + ..ok + }, + DeviceState { + msi_vectors: 3, + ..ok + }, + DeviceState { + msi_vectors: 64, + ..ok + }, + DeviceState { + flags: ok.flags | 1 << DeviceState::PIN_SHIFT, + ..ok + }, + DeviceState { + msix_entries: U16::new(crate::device::DeviceRecord::MAX_MSIX_TABLE_ENTRIES + 1), + ..ok + }, + ] { + assert!(!broken.validate(), "{broken:?}"); + } + } + + #[test] + fn a_page_takes_exactly_its_rows_in_order() { + // 20 states: page 0 holds 16, page 1 the last 4, and there is no + // page 2. + let mut first = DeviceStatePage::new(0, 20).unwrap(); + for index in 0..16u16 { + let (value, aux) = state(0).to_observation(index); + assert!(first.push(value, aux)); + } + let (value, aux) = state(0).to_observation(16); + assert!(!first.push(value, aux), "a full page takes no more"); + assert!(first.validate()); + + let mut second = DeviceStatePage::new(1, 20).unwrap(); + // The next state is index 16's; index 17's report is out of order. + let (value, aux) = state(0).to_observation(17); + assert!(!second.push(value, aux)); + assert!(!second.validate(), "a short page is not a page"); + for index in 16..20u16 { + let (value, aux) = state(0).to_observation(index); + assert!(second.push(value, aux)); + } + assert!(second.validate()); + assert!(DeviceStatePage::new(2, 20).is_none()); + // A machine with no function still has an empty page 0. + assert!(DeviceStatePage::new(0, 0).unwrap().validate()); + assert!(DeviceStatePage::new(1, 0).is_none()); + } + + #[test] + fn a_bar_round_trips_and_never_maps_more_than_it_holds() { + let bar = DeviceBar { + size: U64::new(0x10_0000), + mappable: U64::new(0x8000), + }; + let (value, aux) = bar.to_observation(4).unwrap(); + assert_eq!(DeviceBar::from_observation(4, value, aux), Some(bar)); + assert_eq!(DeviceBar::from_observation(3, value, aux), None); + // An absent BAR is the zero shape, and still answers for its number. + let (value, aux) = DeviceBar::EMPTY.to_observation(5).unwrap(); + assert_eq!( + DeviceBar::from_observation(5, value, aux), + Some(DeviceBar::EMPTY) + ); + assert_eq!(DeviceBar::from_observation(0, 0x1000, 0x2000), None); + assert!(bar.to_observation(6).is_none()); + let wide = DeviceBar { + size: U64::new(u64::MAX), + mappable: U64::new(1 << 56), + }; + assert!(wide.to_observation(0).is_none()); + } + + #[test] + fn a_bar_operand_names_a_function_and_a_bar() { + let operand = device_bar_operand(119, 5).unwrap(); + assert_eq!(device_bar_of(operand), Some((119, 5))); + assert_eq!(device_bar_operand(0, 0), Some(0)); + assert!(device_bar_operand(1 << DEVICE_BAR_OPERAND_SHIFT, 0).is_none()); + assert!(device_bar_operand(0, 6).is_none()); + assert!(device_bar_of(7 << DEVICE_BAR_OPERAND_SHIFT).is_none()); + } + + #[test] + fn an_inventory_holds_a_state_per_census_row() { + let mut inventory = DeviceInventory::new(census(20)); + assert!(inventory.validate()); + assert!(!inventory.complete()); + for page in 0..2u8 { + let mut states = DeviceStatePage::new(page, 20).unwrap(); + for index in states.first()..states.first() + states.expected() { + let (value, aux) = state(0).to_observation(index as u16); + assert!(states.push(value, aux)); + } + assert!(inventory.insert_states(&states)); + } + assert!(inventory.complete()); + assert!(inventory.validate()); + // A page of another census's table is refused whole. + let other = DeviceStatePage::new(0, 21).unwrap(); + assert!(!inventory.insert_states(&other)); + + let mut bars = DeviceBarTable::new(19); + let shape = DeviceBar { + size: U64::new(0x4000), + mappable: U64::new(0x4000), + }; + let (value, aux) = shape.to_observation(0).unwrap(); + assert!(bars.set_bar(0, value, aux)); + assert!(!bars.set_bar(1, value, aux), "the echo names BAR 0"); + inventory.bars = bars; + assert!(inventory.validate()); + // BARs of a function the census does not hold are not this + // inventory's. + inventory.bars.index = U16::new(20); + assert!(!inventory.validate()); + assert!(DeviceInventory::EMPTY.validate()); + assert!(!DeviceInventory::EMPTY.available()); + } + + #[test] + fn a_page_slot_decodes_only_the_record_it_carries() { + let mut states = DeviceStatePage::new(0, 2).unwrap(); + for index in 0..2u16 { + let (value, aux) = state(0).to_observation(index); + assert!(states.push(value, aux)); + } + let slot = DevicePage::of_states(&states); + assert_eq!(slot.states(), Some(states)); + + let mut table = DeviceBarTable::new(3); + let shape = DeviceBar { + size: U64::new(0x1000), + mappable: U64::new(0x1000), + }; + let (value, aux) = shape.to_observation(2).unwrap(); + assert!(table.set_bar(2, value, aux)); + let slot = DevicePage::of_bars(&table); + assert_eq!(slot.bars(), Some(table)); + // Bytes past the table are not part of it. + let mut stray = slot; + stray.0[DEVICE_PAGE_BYTES - 1] = 1; + assert_eq!(stray.bars(), None); + + let page = census(3).page(0).unwrap(); + assert_eq!(DevicePage::of_census(&page).census(), Some(page)); + // The zero slot is no record at all. + assert_eq!(DevicePage::EMPTY.bars(), None); + assert_eq!( + DevicePage::EMPTY.census().map(|page| page.rows_valid), + Some(0) + ); + } + + #[test] + fn an_address_is_exactly_an_address() { + let address = PciAddress::parse(b"0000:00:1f.3").unwrap(); + assert_eq!((address.segment, address.locator), (0, 0x1F << 3 | 3)); + assert_eq!(PciAddress::parse(b"00:1f.3"), Some(address)); + assert_eq!( + PciAddress::parse(b"0001:ab:02.0"), + Some(PciAddress { + segment: 1, + locator: 0xAB << 8 | 2 << 3 + }) + ); + for not in [ + &b""[..], + b"1f.3", + b"00:1f", + b"00:1f.8", + b"00:20.0", + b"0:1f.3", + b"000:00:1f.3", + b"00:1f.3x", + b"x0:1f.3", + b"0000:00:1f:3", + b"00000:00:1f.3", + b"8086:a828", + ] { + assert_eq!( + PciAddress::parse(not), + None, + "{:?}", + core::str::from_utf8(not) + ); + } + let census = census(4); + assert_eq!( + PciAddress { + segment: 0, + locator: 3 << 3 + } + .row_in(&census), + Some(3) + ); + assert_eq!( + PciAddress { + segment: 1, + locator: 3 << 3 + } + .row_in(&census), + None + ); + } +} diff --git a/native/crates/abi/src/pkg.rs b/native/crates/abi/src/pkg.rs index bac3b8f..ac109a8 100644 --- a/native/crates/abi/src/pkg.rs +++ b/native/crates/abi/src/pkg.rs @@ -27,13 +27,27 @@ use zerocopy::{FromBytes, Immutable, IntoBytes, KnownLayout, Unaligned}; /// Layout version. A zeroed record reads `abi == 0` and is *not a /// manifest*; a reader refuses rather than guessing. -pub const PACKAGE_ABI: u16 = 1; -/// Members one manifest may declare. The bootstrap archive today carries -/// 33 executables and 3 libraries (`BOOTSTRAP_EXECUTABLES` / -/// `BOOTSTRAP_LIBRARIES` in exec), and the system package `kore` must list -/// every one of them; 48 leaves a third free before the record grows a -/// second page. `size_of::()` is pinned below. -pub const MAX_PACKAGE_MEMBERS: usize = 48; +/// +/// 2: [`MAX_PACKAGE_MEMBERS`] grew from 48 to 96, so the record grew with +/// it. Version 1 manifests were only ever written — exec rewrites the +/// system package's in place at every boot, and no installer reads one +/// yet — so nothing has to read both shapes; the bump is what lets a future +/// reader tell them apart rather than misread one as the other. +pub const PACKAGE_ABI: u16 = 2; +/// Members one manifest may declare: every bootstrap executable and library +/// (`BOOTSTRAP_EXECUTABLES` / `BOOTSTRAP_LIBRARIES` in exec), which the +/// system package `kore` must list in full. +/// +/// **Where the number comes from.** The record is read whole by one +/// `ReadVersion` transfer (`MAX_TRANSFER`, 4096 bytes): 48 bytes of header +/// and name, then 40 a member, so one page holds 100. 96 is the most of +/// that in whole 32-member steps. 48 was the first ceiling, reached exactly +/// when the catalogue held 45 executables and 3 libraries; the C compiler +/// was the 49th member. **What breaks if it moves.** Larger needs a second +/// transfer to read the record (a reader change, and another +/// [`PACKAGE_ABI`]); smaller fails exec's build, which asserts the +/// catalogue fits. `size_of::()` is pinned below. +pub const MAX_PACKAGE_MEMBERS: usize = 96; /// A member or package name: the store's own name bound /// ([`FS_NAME_BYTES`](crate::svc::FS_NAME_BYTES)), zero-padded, so a /// manifest name is always a legal DagFS name and never needs clamping. @@ -255,12 +269,13 @@ impl PackageManifest { } const _: () = assert!(core::mem::size_of::() == 8 + PACKAGE_NAME_BYTES); -/// 16-byte header + 32-byte name + 48 × 40-byte members = 1968 bytes: one +/// 16-byte header + 32-byte name + 96 × 40-byte members = 3888 bytes: one /// `ReadVersion` transfer (`MAX_TRANSFER` = 4096) reads a whole manifest. const _: () = assert!( core::mem::size_of::() == 16 + PACKAGE_NAME_BYTES + MAX_PACKAGE_MEMBERS * (8 + PACKAGE_NAME_BYTES) ); +const _: () = assert!(core::mem::size_of::() <= 4096); #[cfg(test)] mod tests { @@ -281,7 +296,7 @@ mod tests { fn a_manifest_round_trips_through_its_bytes_and_validates() { let manifest = system_package(); let bytes = manifest.as_bytes(); - assert_eq!(bytes.len(), 1968); + assert_eq!(bytes.len(), 3888); let read = PackageManifest::read_from_bytes(bytes).unwrap(); assert_eq!(read, manifest); assert_eq!(read.validate(), Ok(())); @@ -297,7 +312,7 @@ mod tests { #[test] fn a_zeroed_record_is_not_a_manifest() { - let zero = PackageManifest::read_from_bytes(&[0u8; 1968]).unwrap(); + let zero = PackageManifest::read_from_bytes(&[0u8; 3888]).unwrap(); assert_eq!(zero.validate(), Err(ManifestError::Abi(0))); } @@ -333,7 +348,7 @@ mod tests { } #[test] - fn push_refuses_duplicates_and_the_forty_ninth_member() { + fn push_refuses_duplicates_and_a_member_past_the_ceiling() { let mut manifest = system_package(); assert!( manifest diff --git a/native/crates/abi/src/process.rs b/native/crates/abi/src/process.rs index 48b0841..b1e72eb 100644 --- a/native/crates/abi/src/process.rs +++ b/native/crates/abi/src/process.rs @@ -565,8 +565,11 @@ pub const PROCESS_OUTPUT_BYTES: usize = PROCESS_IO_BYTES - HEADER_BYTES - PROCES /// would take that word for zero-or-corrupt. Each change shifts an offset or /// a meaning and therefore burns a version. 13: the system snapshot's /// scheduler-timer arm census, paid from the argument buffer together with -/// the output bytes its status row needs, so no report lost a row. -pub const PROCESS_IO_ABI: u16 = 13; +/// the output bytes its status row needs, so no report lost a row. 14: the +/// second reserved word gains a meaning, the length of the command name the +/// launcher resolved the program by, which rides the argument buffer after +/// the tail ([`ProcessIo::command`]): a `13` reader requires that word zero. +pub const PROCESS_IO_ABI: u16 = 14; /// One launch-time snapshot of exec's bounded process table. /// @@ -717,9 +720,12 @@ pub struct ProcessIo { /// holds the frame service's diagnostic face. net: crate::monitor::NetSnapshot, // `reserved[0]` was intentionally available for an additive ABI field. - // It now carries the immutable launcher-context live length; the remaining - // words stay reserved and must be zero. + // It now carries the immutable launcher-context live length, and + // `reserved[1]` the command name's length; the remaining words stay + // reserved and must be zero. reserved: [U32; 4], + /// The argument tail, `arg_len` bytes, then the command name, + /// `reserved[1]` bytes; zero after them. args: [u8; PROCESS_ARG_BYTES], /// Alternating payload: launcher context before START, child output after /// execution. The task lifecycle is the exclusive-ownership boundary. @@ -776,8 +782,10 @@ impl ProcessIo { && self.flags.get() == 0 && self.telemetry.validate() && self.filesystem.validate() - && self.reserved[1..].iter().all(|v| v.get() == 0) + && self.reserved[2..].iter().all(|v| v.get() == 0) && usize::from(self.arg_len.get()) <= PROCESS_ARG_BYTES + && usize::try_from(self.reserved[1].get()) + .is_ok_and(|name| name <= PROCESS_ARG_BYTES - usize::from(self.arg_len.get())) && usize::try_from(self.reserved[0].get()).is_ok_and(|len| len <= PROCESS_CONTEXT_BYTES) && usize::from(self.output_len.get()) <= PROCESS_OUTPUT_BYTES && (self.reserved[0].get() == 0 || self.output_len.get() == 0) @@ -791,6 +799,36 @@ impl ProcessIo { Ok(&self.args[..usize::from(self.arg_len.get())]) } + /// The name the launcher resolved this program by: the first word of the + /// command line it was run from, which the tail in [`Self::args`] does + /// not hold. Empty when the launcher gave none. A label: it selects + /// nothing and proves nothing about the image. + pub fn command(&self) -> Result<&[u8], ProcessIoError> { + if !self.validate() { + return Err(ProcessIoError::Invalid); + } + let tail = usize::from(self.arg_len.get()); + Ok(&self.args[tail..tail + self.reserved[1].get() as usize]) + } + + /// Publish the command name before the child starts. It shares the + /// argument buffer with the tail, so a name the buffer has no room for + /// after the tail is refused whole and the page is unchanged. + pub fn write_command(&mut self, name: &[u8]) -> Result<(), ProcessIoError> { + if !self.validate() { + return Err(ProcessIoError::Invalid); + } + let tail = usize::from(self.arg_len.get()); + let end = tail + .checked_add(name.len()) + .filter(|end| *end <= PROCESS_ARG_BYTES) + .ok_or(ProcessIoError::ArgumentsTooLong)?; + self.args[tail..].fill(0); + self.args[tail..end].copy_from_slice(name); + self.reserved[1].set(name.len() as u32); + Ok(()) + } + /// Immutable launcher context for this invocation. It is data, never an /// authority selector; a program must still hold every capability needed /// for the operation it describes. @@ -1248,6 +1286,55 @@ mod tests { assert_eq!(page.output(), Err(ProcessIoError::Invalid)); } + #[test] + fn the_command_name_rides_after_the_tail_and_leaves_it_alone() { + let mut page = ProcessIo::new_zeroed(); + page.initialize_with_context( + b"-v 'two words'", + b"context", + crate::monitor::TelemetrySnapshot::EMPTY, + ) + .unwrap(); + // A launcher that names nothing leaves the name empty. + assert_eq!(page.command().unwrap(), b""); + page.write_command(b"answer").unwrap(); + assert_eq!(page.command().unwrap(), b"answer"); + assert_eq!(page.args().unwrap(), b"-v 'two words'"); + assert_eq!(page.context().unwrap(), b"context"); + // A second name replaces the first whole, a shorter one included. + page.write_command(b"a").unwrap(); + assert_eq!(page.command().unwrap(), b"a"); + // The buffer is the tail's and the name's together: a name that + // does not fit after the tail is refused and nothing changes. + let long = [b'n'; PROCESS_ARG_BYTES]; + assert_eq!( + page.write_command(&long), + Err(ProcessIoError::ArgumentsTooLong) + ); + assert_eq!(page.command().unwrap(), b"a"); + assert_eq!(page.args().unwrap(), b"-v 'two words'"); + let fits = &long[..PROCESS_ARG_BYTES - page.args().unwrap().len()]; + page.write_command(fits).unwrap(); + assert_eq!(page.command().unwrap(), fits); + // The next launch starts with no name. + page.initialize(b"", crate::monitor::TelemetrySnapshot::EMPTY) + .unwrap(); + assert_eq!(page.command().unwrap(), b""); + } + + #[test] + fn a_command_name_past_the_buffer_fails_closed() { + let mut page = ProcessIo::new_zeroed(); + page.initialize(b"tail", crate::monitor::TelemetrySnapshot::EMPTY) + .unwrap(); + page.reserved[1].set((PROCESS_ARG_BYTES - 3) as u32); + assert_eq!(page.command(), Err(ProcessIoError::Invalid)); + assert_eq!(page.args(), Err(ProcessIoError::Invalid)); + page.reserved[1].set(0); + page.reserved[2].set(1); + assert_eq!(page.args(), Err(ProcessIoError::Invalid)); + } + #[test] fn malformed_context_length_fails_closed() { let mut page = ProcessIo::new_zeroed(); diff --git a/native/crates/abi/src/session.rs b/native/crates/abi/src/session.rs index ac3aa27..c010a5a 100644 --- a/native/crates/abi/src/session.rs +++ b/native/crates/abi/src/session.rs @@ -2588,6 +2588,28 @@ pub const REVOKE_KIND_EXEC_RETURN: u8 = 7; /// window survives, the latch re-anchors on it. pub const REVOKE_KIND_UNSHARE_IOVA_LIVE: u8 = 8; +/// The [`Invocation::kind`] selector for the consuming **writable lease → +/// frozen lease** transition through [`OpCode::Revoke`]: the exclusive +/// intake of a registrar that must validate bytes it did not write +/// (`res/IMAGE.md` §6b). +/// +/// The subject is a held or same-session `MemLease` carrying the complete +/// lifecycle grant `ACCESS|MODIFY|DERIVE|REVOKE`. Commit consumes that +/// capability and its complete descendant lineage — every view, carve and +/// route derived from it dies, wherever it travelled — removes every +/// CPU/DMA/guest alias, and writes a fresh `MemLease` over the same window +/// into `result`, carrying the same grant. From then on no lessor of the +/// backing may write the window, the successor and its carves included: the +/// family's claim table holds it immutable for as long as the successor +/// lives. A carve whose parent or sibling still holds a writable view of the +/// window is refused before anything commits, and so is an executable or +/// swap-sealed lease. +/// +/// The successor keeps `MODIFY` because a registrar carves the window's text +/// and finalizes it, which a frozen window admits ([`REVOKE_KIND_EXEC`] +/// leaves it immutable too); `MODIFY` never again reaches a write. +pub const REVOKE_KIND_FREEZE: u8 = 9; + /// Argument payload for a **page delegation** derive (`Derive`, /// [`kind`](Invocation::kind)` = Region` on a held/staged `Region` /// subject; res/IMAGE.md §9a): re-home the subject region @@ -3123,6 +3145,21 @@ pub enum SystemMetric { IommuUnit = 35, /// Indexed capabilities and scope counts, Unit::features() observation. IommuUnitFeatures = 36, + /// One PCI function's live state; operand = its catalog index, as for + /// [`DeviceRow`](Self::DeviceRow). The pair is + /// [`DeviceState::to_observation`](crate::device::DeviceState::to_observation): + /// whether a driver claimed it, the founded row driving its DMA, which + /// BARs exist, may be mapped and are mapped, and how it can interrupt. + /// A snapshot, never permission; no address. + DeviceState = 37, + /// One BAR's shape; operand = + /// [`device_bar_operand`](crate::device::device_bar_operand) of the + /// function's catalog index and the BAR number. The pair is + /// [`DeviceBar::to_observation`](crate::device::DeviceBar::to_observation): + /// the BAR's size and the length a driver may map. Lengths only: where + /// the BAR lives never crosses. A BAR the function has not answers the + /// zero shape. + DeviceBar = 38, } /// Records the kernel retains for [`SystemMetric::IommuFaultRecord`]. @@ -3494,6 +3531,8 @@ impl SystemObserveReq { matches!( metric, SystemMetric::DeviceRow + | SystemMetric::DeviceState + | SystemMetric::DeviceBar | SystemMetric::ProcessorCapture | SystemMetric::ProcessorIdentity | SystemMetric::ProcessorTopology diff --git a/native/crates/abi/src/svc.rs b/native/crates/abi/src/svc.rs index 703543e..469b2d9 100644 --- a/native/crates/abi/src/svc.rs +++ b/native/crates/abi/src/svc.rs @@ -205,6 +205,13 @@ impl MonitorReqWire { /// records. The monitor assembles the census from its own /// `SystemObserve` capability; the request carries no authority. pub const OP_DEVICE_CENSUS: u8 = 2; + /// One page of the census functions' live state + /// (`crate::device::state`): `reserved[0]` is the page ordinal. Sampled + /// when asked, never cached; the request carries no authority. + pub const OP_DEVICE_STATES: u8 = 5; + /// One function's BAR shapes: `reserved[0..2]` is its catalog index, + /// little-endian. Lengths only. + pub const OP_DEVICE_BARS: u8 = 6; } /// Monitor reply: a complete bounded snapshot or a closed refusal. The @@ -222,9 +229,11 @@ pub struct MonitorRepWire { /// Tagged observation payload: ProcessorPage for processor replies, or /// iommu_status::Snapshot encoded by its wire-page methods for IOMMU replies. pub processor: crate::processor_status::ProcessorPage, - /// [`TAG_DEVICE_CENSUS`](Self::TAG_DEVICE_CENSUS) only; the zero page - /// on every other tag (the codec refuses a stray one). - pub census: crate::device::DeviceCensusPage, + /// The device record of [`TAG_DEVICE_CENSUS`](Self::TAG_DEVICE_CENSUS), + /// [`TAG_DEVICE_STATES`](Self::TAG_DEVICE_STATES) or + /// [`TAG_DEVICE_BARS`](Self::TAG_DEVICE_BARS); the zero slot on every + /// other tag (the codec refuses a stray one). + pub device: crate::device::DevicePage, } impl MonitorRepWire { @@ -233,6 +242,8 @@ impl MonitorRepWire { pub const TAG_SNAPSHOT: u8 = 1; pub const TAG_REFUSED: u8 = 2; pub const TAG_DEVICE_CENSUS: u8 = 3; + pub const TAG_DEVICE_STATES: u8 = 6; + pub const TAG_DEVICE_BARS: u8 = 7; pub const REFUSE_DENIED: u8 = 1; } @@ -1502,6 +1513,40 @@ impl TaskReqWire { /// `slot` names the face; no plan, no target, no grants. Seal-exempt, /// via==0 only: provisioning is the administrator's, not a task's. pub const OP_PROVISION_HOME: u8 = 25; + /// Install one file from the caller's own tree as a **private** + /// executable generation (`res/EXECUTABLE_CAPABILITY_MIGRATION.md` §7.4): + /// exec reads the file through the caller's conveyed view into the + /// caller's own memory supply, freezes that supply, registers it, and + /// grants an executable capability for it. + /// + /// The request is a client face's atomic context bundle without an + /// executable member: the supply on the ProcessIo lane, the view on the + /// filesystem-view lane, and on the first auxiliary lane the receive end + /// of a channel whose root the caller keeps — the generation's lifetime: + /// revoking that root, or the caller's death, retires it. The proof lane + /// and the other auxiliary lanes are vacant. `slot` carries the supply's + /// capability tag, so exec selects exactly this request's delivery and + /// never a stale one (tags are never reissued). The path rides the + /// supply itself ([`InstallPathHeader`]) and is overwritten by the + /// file's bytes, so the record keeps its size. Every other field is + /// zero. A file longer than the supply is answered + /// [`TAG_INSTALL_SUPPLY_SHORT`](TaskRepWire::TAG_INSTALL_SUPPLY_SHORT) + /// with the size it needs. + /// + /// Reading suffices, by the filesystem's own rule: `BIND_EXECUTABLE` + /// governs the shared executable namespace, not private code a task + /// makes executable from its own lease. The generation binds no global + /// name: only the capability's holder reaches it. + pub const OP_INSTALL_FILE: u8 = 26; + /// One page of the census functions' live state + /// (`crate::device::state`), relayed from the monitor as + /// [`OP_DEVICE_CENSUS`](Self::OP_DEVICE_CENSUS)'s pages are, but never + /// cached: `slot` is the page ordinal. Bind-inert data, no grants, + /// answered on any launcher pair. + pub const OP_DEVICE_STATES: u8 = 27; + /// One function's BAR shapes, relayed from the monitor: `slot` is the + /// function's catalog index. Lengths only; no grants. + pub const OP_DEVICE_BARS: u8 = 28; /// Every retired `TaskReqWire` discriminant, in one definition the /// decoder cites (the [`derive_kind_known`](crate::session::derive_kind_known) @@ -1568,6 +1613,66 @@ impl TaskReqWire { pub const INSTALL_FROM_TREE: u8 = 1 << 1; } +/// The path an [`OP_INSTALL_FILE`](TaskReqWire::OP_INSTALL_FILE) request +/// installs, which the caller writes at offset 0 of the supply it conveys: +/// this header, then [`len`](Self::len) UTF-8 bytes. The path is relative +/// to the conveyed view and `/`-separated. Exec copies it out before the +/// first file byte lands and the file then overwrites it, so neither the +/// record nor the frozen image carries it. +/// +/// A path rides the supply rather than the record because a legal path is +/// longer than every free field: the record's `target` holds 64 bytes, the +/// deepest legal path [`FS_SEARCH_PATH_BYTES`], and the pair already uses +/// all but 128 bytes of its ring (`kore_svc_exec::wire`). +#[repr(C)] +#[derive( + Clone, Copy, Debug, PartialEq, Eq, FromBytes, IntoBytes, KnownLayout, Immutable, Unaligned, +)] +pub struct InstallPathHeader { + /// Live path bytes after the header (`1..=MAX_PATH`). + pub len: U16, + /// Reserved; must be zero. + pub reserved: [u8; 6], +} + +impl InstallPathHeader { + /// Wire size of the header (repr(C), 2 + 6). + pub const BYTES: usize = 8; + /// The longest path a request carries: the deepest legal filesystem + /// path, the bound a search hit spells. + pub const MAX_PATH: usize = FS_SEARCH_PATH_BYTES; + /// Bytes a request's path occupies at the head of its supply. + pub const MAX_BYTES: usize = Self::BYTES + Self::MAX_PATH; + + /// The header for `path`, or `None` for an empty path or one past + /// [`MAX_PATH`](Self::MAX_PATH). + pub fn for_path(path: &[u8]) -> Option { + if path.is_empty() || path.len() > Self::MAX_PATH { + return None; + } + Some(Self { + len: U16::new(path.len() as u16), + reserved: [0; 6], + }) + } + + /// The path a supply's head carries: a well-formed header followed by + /// that many UTF-8 bytes, or `None`. Component rules (no empty, `.` or + /// `..` component; the filesystem's name bound) are the reader's to + /// apply, against the filesystem it walks. + pub fn path(head: &[u8]) -> Option<&str> { + let (header, rest) = Self::read_from_prefix(head).ok()?; + let len = usize::from(header.len.get()); + if header.reserved != [0; 6] || len == 0 || len > Self::MAX_PATH { + return None; + } + core::str::from_utf8(rest.get(..len)?).ok() + } +} + +const _: () = assert!(core::mem::size_of::() == InstallPathHeader::BYTES); +const _: () = assert!(InstallPathHeader::MAX_PATH <= u16::MAX as usize); + /// The exec reply wire form (`TaskResponse`'s crossing shape). An owned /// `Spawned` reply carries exactly one task-handle grant. `id` remains /// correlation data only (hash-derived, unordered). A `Spawned` reply uses @@ -1615,9 +1720,11 @@ pub struct TaskRepWire { /// Tagged observation payload: ProcessorPage for processor replies, or /// iommu_status::Snapshot encoded by its wire-page methods for IOMMU replies. pub processor: crate::processor_status::ProcessorPage, - /// [`TAG_DEVICE_CENSUS`](Self::TAG_DEVICE_CENSUS)'s page; the zero - /// page on every other tag. - pub census: crate::device::DeviceCensusPage, + /// The device record of [`TAG_DEVICE_CENSUS`](Self::TAG_DEVICE_CENSUS), + /// [`TAG_DEVICE_STATES`](Self::TAG_DEVICE_STATES) or + /// [`TAG_DEVICE_BARS`](Self::TAG_DEVICE_BARS); the zero slot on every + /// other tag. + pub device: crate::device::DevicePage, /// Only `REFUSE_LEDGER_FULL` with `index == 1` may carry this bounded /// failure snapshot. Every other reply requires the all-zero form. pub cleanup: CleanupRowDiagnosticWire, @@ -1750,6 +1857,24 @@ impl TaskRepWire { /// unprovisioned home is a posture (no store, or a stage that failed and /// is retried next boot), never a refusal. pub const TAG_HOME_PROVISIONED: u8 = 25; + /// Reply to [`TaskReqWire::OP_INSTALL_FILE`]: the private generation was + /// registered, and the reply grants its executable endpoint. `index` + /// carries the segment count and `status` the launch profile word, as + /// [`TAG_TREE_ACQUIRED`](Self::TAG_TREE_ACQUIRED) carries them. + pub const TAG_FILE_INSTALLED: u8 = 26; + /// Reply to [`TaskReqWire::OP_INSTALL_FILE`]: the file is longer than + /// the conveyed supply. `id` is the page-rounded size the file needs, + /// which a caller may retry with; nothing was registered, and the supply + /// was retired, so its memory is the caller's again. + pub const TAG_INSTALL_SUPPLY_SHORT: u8 = 27; + /// Reply to [`TaskReqWire::OP_DEVICE_STATES`]: `device` is the page. + pub const TAG_DEVICE_STATES: u8 = 28; + /// Reply to [`TaskReqWire::OP_DEVICE_BARS`]: `device` is the table. + pub const TAG_DEVICE_BARS: u8 = 29; + /// The unit [`TAG_INSTALL_SUPPLY_SHORT`](Self::TAG_INSTALL_SUPPLY_SHORT) + /// counts in: mm grants a base-class supply in whole 4 KiB pages, so a + /// size the caller can ask for is a multiple of it. + pub const INSTALL_SUPPLY_GRANULE: u64 = 4096; /// `TAG_HOME_PROVISIONED.index`: the home is recorded at the current /// skeleton version. pub const HOME_PROVISIONED: u8 = 1 << 0; @@ -1854,6 +1979,12 @@ impl TaskRepWire { pub const REFUSE_PEER_ADMISSION_WEDGED: u8 = 22; pub const REFUSE_MODEL_UNAVAILABLE: u8 = 23; pub const REFUSE_MODEL_MEMORY_UNAVAILABLE: u8 = 24; + /// `TaskRefusal::FileUnavailable` — an install's path named no file the + /// presented view could read whole: no such entry, a directory, or a + /// read the filesystem refused. Distinct from + /// [`REFUSE_IMAGE_UNAVAILABLE`](Self::REFUSE_IMAGE_UNAVAILABLE), which + /// says the bytes were read and are not a dynamic executable. + pub const REFUSE_FILE_UNAVAILABLE: u8 = 25; } // Live task discriminants stay disjoint from the frozen retired band on @@ -3834,6 +3965,16 @@ impl InReqWire { /// `InputStreamRequest::Reprint` — echo the line being edited, cursor /// in place, then apply every key queued since `Hold`. pub const OP_REPRINT: u8 = 6; + /// `InputStreamRequest::Insert` — type text into the **held** line at + /// its cursor, as if typed, without echo (the `Reprint` shows it): + /// `reserved0` = the byte count (1 to [`INSERT_BYTES`](Self::INSERT_BYTES)), + /// `reserved2..reserved4` = the bytes, little-endian, the words past the + /// count zero. Printable ASCII only: the line takes bytes up to the first + /// it cannot hold — any other byte, or one past a full line — so no + /// insert can ever complete a line. + pub const OP_INSERT: u8 = 7; + /// The most one `Insert` carries: its three data words. + pub const INSERT_BYTES: usize = 24; } /// The stream-input-service reply wire form. The legacy stream counters are @@ -3926,6 +4067,9 @@ impl InRepWire { /// `InputStreamResponse::Reprinted` — the line is echoed again and the /// held keys applied. pub const TAG_REPRINTED: u8 = 7; + /// `InputStreamResponse::Inserted` — `produced` = how many of the + /// insert's bytes the line took (zero to its count). + pub const TAG_INSERTED: u8 = 8; /// `InputStreamRefusal::Busy` — a queue is already attached (the v1 /// single-peer bound, `res/USERLAND.md` §5.9). @@ -3935,6 +4079,9 @@ impl InRepWire { /// `InputStreamRefusal::BadQueue` — the delivered slot was absent or /// not a well-formed queue window. pub const REFUSE_BAD_QUEUE: u8 = 3; + /// `InputStreamRefusal::NotHeld` — an insert into a line that is not + /// held, or not being edited (a completed line still in flight). + pub const REFUSE_NOT_HELD: u8 = 4; } /// One data-plane chunk — the `ring::spsc` record the backend @@ -3963,6 +4110,42 @@ impl InChunk { mod tests { use super::*; + /// An install path round-trips through a supply's head, and every + /// malformed head refuses: empty, past the bound, a reserved byte set, + /// a length past the bytes present, and bytes that are not UTF-8. + #[test] + fn install_paths_ride_a_supply_head() { + let mut head = [0u8; InstallPathHeader::MAX_BYTES + 1]; + let path = b"build/hello"; + let header = InstallPathHeader::for_path(path).unwrap(); + head[..InstallPathHeader::BYTES].copy_from_slice(header.as_bytes()); + head[InstallPathHeader::BYTES..][..path.len()].copy_from_slice(path); + assert_eq!(InstallPathHeader::path(&head), Some("build/hello")); + + assert_eq!(InstallPathHeader::for_path(b""), None); + let longest = [b'a'; InstallPathHeader::MAX_PATH]; + assert!(InstallPathHeader::for_path(&longest).is_some()); + let too_long = [b'a'; InstallPathHeader::MAX_PATH + 1]; + assert_eq!(InstallPathHeader::for_path(&too_long), None); + + let mut reserved = head; + reserved[7] = 1; + assert_eq!(InstallPathHeader::path(&reserved), None); + let mut empty = head; + empty[..2].copy_from_slice(&0u16.to_le_bytes()); + assert_eq!(InstallPathHeader::path(&empty), None); + let mut past = head; + past[..2].copy_from_slice(&((InstallPathHeader::MAX_PATH + 1) as u16).to_le_bytes()); + assert_eq!(InstallPathHeader::path(&past), None); + assert_eq!( + InstallPathHeader::path(&head[..InstallPathHeader::BYTES + path.len() - 1]), + None + ); + let mut invalid = head; + invalid[InstallPathHeader::BYTES] = 0xff; + assert_eq!(InstallPathHeader::path(&invalid), None); + } + /// Every wire form reinterprets in place from arbitrary bytes and /// round-trips through its own bytes — the shared-ring contract /// (any bit pattern is *readable*; the codec's job is refusal). @@ -4335,6 +4518,32 @@ impl SurReqWire { /// compositor snapshots the name and ends the pick. No subject /// ([`SurRepWire::TAG_PICK_ANSWERED`]). pub const OP_PICKED: u8 = 13; + /// `SurfaceRequest::CopyChunk` — one piece of the text this window puts + /// on its seat's clipboard (`res/CLIPBOARD.md`), written at the head of + /// its pair lease's request plane (`kore_window::menu`'s page): `a` = + /// the piece's offset in the text `| len << 32` (1 to the plane's size), + /// `b` = the whole text's byte length (1 to + /// `kore_window::clipboard::CLIPBOARD_BYTES`). The piece at offset zero + /// begins a copy, and only the seat's focused window may begin one, on + /// a gesture of the user's it has not spent; later pieces continue it + /// in order, and the piece that reaches the length ends it. The + /// compositor copies each piece out before replying, so the plane may be + /// reused once answered. No subject ([`SurRepWire::TAG_COPIED`]). + pub const OP_COPY_CHUNK: u8 = 14; + /// `SurfaceRequest::Paste` — read the seat's clipboard into this + /// window's request plane: `a` = the offset to read from. Offset zero + /// begins a read, and only the seat's focused window may begin one, on + /// a gesture of the user's it has not spent; later offsets continue it + /// until it reaches the end or the clipboard changes. No subject + /// ([`SurRepWire::TAG_CLIPBOARD`], [`SurRepWire::TAG_CLIPBOARD_EMPTY`]). + pub const OP_PASTE: u8 = 15; + /// `SurfaceRequest::PasteToShell` — the seat's terminal window asks for + /// the clipboard to be typed into its shell's line: queued as an inert + /// `ShellAction::Paste` for its controller, which reads the text and + /// edits the line itself. Only the seat's focused terminal window may + /// ask, on a gesture it has not spent. No words, no subject + /// ([`SurRepWire::TAG_PASTE_QUEUED`], [`SurRepWire::TAG_CLIPBOARD_EMPTY`]). + pub const OP_PASTE_TO_SHELL: u8 = 16; /// [`OP_SET_CURSOR`](Self::OP_SET_CURSOR) `a`: the standard arrow. pub const CURSOR_DEFAULT: u64 = 0; @@ -4456,6 +4665,20 @@ impl SurRepWire { /// `SurfaceResponse::PickAnswered` — `a` echoes the token answered; the /// pick has ended and the plane may be reused. pub const TAG_PICK_ANSWERED: u8 = 14; + /// `SurfaceResponse::Copied` — `a` = the text copied so far (the + /// piece's offset plus its length, echoed); when it is the whole + /// length, the text is the seat's clipboard. + pub const TAG_COPIED: u8 = 15; + /// `SurfaceResponse::Clipboard` — `a` = the offset read from (echoed) + /// `| len << 32`, the bytes now at the head of the request plane (1 to + /// the plane's size), `b` = the clipboard text's whole byte length. + pub const TAG_CLIPBOARD: u8 = 16; + /// `SurfaceResponse::ClipboardEmpty` — the seat's clipboard holds no + /// text. No words. + pub const TAG_CLIPBOARD_EMPTY: u8 = 17; + /// `SurfaceResponse::PasteQueued` — `a` = the byte length of the text + /// the controller was asked to type. + pub const TAG_PASTE_QUEUED: u8 = 18; /// A v1 vocabulary gate: an unknown op, a non-XRGB8888 format, a /// non-CPU reach, a non-LINEAR modifier, a non-empty flag set. @@ -4472,6 +4695,10 @@ impl SurRepWire { /// A capability transfer or teardown did not settle. A retiring surface /// keeps its backing reserved until revocation succeeds; retry is safe. pub const REFUSE_BUSY: u8 = 6; + /// The user did not ask: a clipboard verb from a window that is not its + /// seat's focused window, or with no unspent gesture of the user's + /// behind it. + pub const REFUSE_DENIED: u8 = 7; } /// One damage rectangle of a [`CommitWire`] record — pixel-addressed, @@ -4815,7 +5042,17 @@ impl FsReqWire { /// route no longer validates. The view must belong to the caller's task. /// `window_offset` is return-post correlation. pub const OP_MATCH_VIEW: u8 = 65; - pub const OP_MAX: u8 = Self::OP_MATCH_VIEW; + /// Arm watch slot `kind` on the directory `handle`, tagged `length` + /// (non-zero, the client's own name for this arming): from now on the + /// pair's watch plane — the event channel after the rings in this pair's + /// own window — counts every change to what a listing of that directory + /// shows. Re-arming a slot replaces its watch. Observation, not + /// authority: the handle's route is revalidated before every count, and + /// a route that stops proving ends the watch. Answered `Done`. + pub const OP_WATCH: u8 = 66; + /// Disarm watch slot `kind`. Answered `Done`, armed or not. + pub const OP_UNWATCH: u8 = 67; + pub const OP_MAX: u8 = Self::OP_UNWATCH; /// [`OP_STOR_MAINTAIN`](Self::OP_STOR_MAINTAIN): report only. pub const MAINTAIN_STATUS: u8 = 0; @@ -4978,6 +5215,59 @@ impl FsDirEntryWire { pub const BYTES: usize = 48; } +/// Watch slots one filesystem pair may arm ([`FsReqWire::OP_WATCH`]). +/// +/// Sized by the client that follows the most folders at once: the Files +/// explorer watches the folder it shows and the sidebar's live folders (its +/// root and the expanded folders whose lines are on screen), fifteen of them +/// at most. Every armed watch holds a handle, so this stays under half the +/// service's per-session handle bound (32, `kore_dagfs::bounds`) — a client +/// watching every slot keeps the handles a listing needs. The plane's +/// `ended` word is one bit per slot, so 32 is the layout's own ceiling. +pub const FS_WATCH_SLOTS: usize = 16; +const _: () = assert!(FS_WATCH_SLOTS <= u32::BITS as usize); + +/// The level state of a filesystem pair's **watch plane**: the event channel +/// (`res/EVENT.md`) the service formats after the request/reply rings in the +/// pair's own window and republishes whole whenever a watched directory +/// changes. Slot `k` belongs to the watch [`FsReqWire::OP_WATCH`] armed with +/// slot `k`. 136 bytes. +/// +/// Counts, not contents: a change is "what a listing of this directory, or +/// of one of its child folders, shows may differ" — an entry made, removed +/// or renamed in it, a child's length, revision or folder count moved, or an +/// entry made or removed in a child folder (its line's count and whether it +/// holds folders) — and the client lists again through its own handle, whose +/// route the service revalidates. Nothing here names an entry, so the plane +/// carries no observation the handle could not make itself. +#[repr(C)] +#[derive( + Clone, Copy, Debug, Default, PartialEq, Eq, FromBytes, IntoBytes, KnownLayout, Immutable, +)] +pub struct FsWatchState { + /// Per slot: the tag the slot was armed with; zero while disarmed. A + /// client reads a slot's counts only under the tag it armed, so a count + /// left by the slot's previous watch is never mistaken for its own. + pub tag: [u32; FS_WATCH_SLOTS], + /// Per slot: changes seen since the slot was armed, wrapping. Coalesced: + /// several changes between two service passes advance it once. + pub changes: [u32; FS_WATCH_SLOTS], + /// Bit `k`: slot `k`'s watch ended — its handle closed, its route stopped + /// proving, or its directory was reclaimed. Cleared by re-arming. + pub ended: u32, + /// Reserved; zero. + pub reserved: u32, +} + +/// The watch plane's record ring: one slot of a reserved edge kind, never +/// pushed. A later per-entry change record lands in it without moving the +/// plane's state or its offset. +pub type FsWatchRecord = u64; +/// Records the watch plane's ring holds. +pub const FS_WATCH_RECORDS: u32 = 1; + +const _: () = assert!(core::mem::size_of::() == 136); + /// Maximum bytes in one view-relative search path. /// /// The filesystem admits thirty-two 32-byte components diff --git a/native/crates/abi/src/task.rs b/native/crates/abi/src/task.rs index baed7a9..3cd22f4 100644 --- a/native/crates/abi/src/task.rs +++ b/native/crates/abi/src/task.rs @@ -280,9 +280,12 @@ pub mod role { /// the face table's slot arithmetic must never interleave with a /// heap tier's. /// - /// Geometry contract: [`FACE_SUPPLY_BYTES`](super::FACE_SUPPLY_BYTES) - /// = [`FACE_RING_STRIDE`](super::FACE_RING_STRIDE) rings — the - /// founder mints exactly that, the face table carves exactly that. + /// Geometry contract, one per carrying row: the image service's + /// [`FACE_SUPPLY_BYTES`](super::FACE_SUPPLY_BYTES) and a broker's + /// [`CLAIMANT_FACE_SUPPLY_BYTES`](super::CLAIMANT_FACE_SUPPLY_BYTES), + /// each a whole number of + /// [`FACE_RING_STRIDE`](super::FACE_RING_STRIDE) rings — the founder + /// mints exactly that, the face table carves exactly that. pub const FACE_SUPPLY: u16 = 29; /// Read-only machine telemetry authority, endowed only to the monitor /// service. The capability carries `ACCESS | DERIVE`; observation is a @@ -462,35 +465,61 @@ pub const INTERACTIVE_CHILD_LOADER_FACES: usize = DESKTOP_BOOTSTRAP_WINDOW_SLOTS + 2 * DESKTOP_CONTROLLER_CAPACITY; /// Live immutable image-generation peers the boot catalogue can hold at once: -/// the 45-row bootstrap catalogue — the desktop's `files`, its documents +/// the 47-row bootstrap catalogue — the desktop's `files`, its documents /// `editor`, `photos`, `settings` and `power-ui`, the screenshot writer `capture`, beside /// the native filesystem verbs `append`, `sync` and `pack`, the two /// WebAssembly rows, `wasm-witness` and `wasm`, `jsapp`, the JS app host, -/// `inference` / `model-data-witness`, and `shell`, the guest shell `term` -/// opens — plus the founded shell, the graphical terminal, and three -/// extraction parsers. +/// `inference` / `model-data-witness`, `shell`, the guest shell `term` +/// opens, `tcc`, the C compiler, and `ioctl`, the device tool — plus the +/// founded shell, the graphical terminal, and three extraction parsers. /// Exec asserts this covers its catalogue (`kore-svc-exec`), the one place /// that sees both. Named /// separately from the interactive reservation below because the two grow for /// unrelated reasons: this one when a row joins the catalogue, that one when /// the desktop admits another window or controller. -pub const BOOTSTRAP_IMAGE_PEERS: usize = 50; +pub const BOOTSTRAP_IMAGE_PEERS: usize = 52; -/// Bootstrap face-supply sizing. Image and runtime grow their face tables and -/// rings on demand; this reserves enough initial memory for the boot catalogue -/// and for a full desktop that can still run one command per controller. -pub const MAX_MINTED_FACE_PAIRS: usize = BOOTSTRAP_IMAGE_PEERS + INTERACTIVE_CHILD_LOADER_FACES; +/// Private image generations installed at runtime that can live at once, +/// across every session: exec's `InstallFile` rows, each a file a task +/// compiled or copied into its own tree and made executable from its own +/// memory supply. +/// +/// Each one costs a ring stride of face supply here (8 × 16 KiB, 128 KiB in +/// all) and one fixed landing in exec's convey table, which takes it from +/// exec's low convey band: eight leave that band thirty slots, above the +/// twelve the founder claims in it (exec asserts the band still holds +/// them). Exec lets one client face hold at most half of these rows, so no +/// session can exhaust the rest. Raising it costs exactly those two things; +/// lowering it below two would let one session's install starve another's. +pub const INSTALLED_IMAGE_PEERS: usize = 8; -/// Total byte length of a [`role::FACE_SUPPLY`] endowment — exactly -/// [`MAX_MINTED_FACE_PAIRS`] ring strides. +/// Bootstrap face-supply sizing. Image and runtime grow their face tables and +/// rings on demand; this reserves enough initial memory for the boot +/// catalogue, the installed generations, and a full desktop that can still +/// run one command per controller. +pub const MAX_MINTED_FACE_PAIRS: usize = + BOOTSTRAP_IMAGE_PEERS + INSTALLED_IMAGE_PEERS + INTERACTIVE_CHILD_LOADER_FACES; + +/// Total byte length of the image service's [`role::FACE_SUPPLY`] +/// endowment — exactly [`MAX_MINTED_FACE_PAIRS`] ring strides. /// The **cross-boundary geometry contract**: the kernel founder mints a -/// lease (or, for the device broker, a full-span sub of the shelf-held -/// claimant band) of exactly this length; the carrying service's face -/// table carves its slots against the same number (its claim checks the -/// span). One constant, cited by both sides, so a drift refuses at -/// bind instead of carving a phantom ring. +/// lease of exactly this length; the image face table carves its slots +/// against the same number (its claim checks the span). One constant, +/// cited by both sides, so a drift refuses at bind instead of carving a +/// phantom ring. The device broker carries the same role under its own +/// contract, [`CLAIMANT_FACE_SUPPLY_BYTES`]. pub const FACE_SUPPLY_BYTES: u64 = MAX_MINTED_FACE_PAIRS as u64 * FACE_RING_STRIDE; +/// Total byte length of a device broker's [`role::FACE_SUPPLY`] +/// endowment, its claimant band — exactly +/// [`MAX_CLAIMANT_FACES`](crate::device::MAX_CLAIMANT_FACES) ring +/// strides, one per driver row. The same contract as +/// [`FACE_SUPPLY_BYTES`]: the kernel founder mints each broker's band at +/// this length and carves the broker's full-span sub off it, and the +/// broker refuses readiness on any other span. +pub const CLAIMANT_FACE_SUPPLY_BYTES: u64 = + crate::device::MAX_CLAIMANT_FACES as u64 * FACE_RING_STRIDE; + /// **Geometry roles** — the window kinds a task learns its own VAs for from the /// [`GeomBlock`] (`res/USERLAND.md` §4; the fixed-VA elimination rung). Unlike /// [`role`] handle entries (which name an arena *slot*), a geometry entry names diff --git a/native/crates/c-shim/Cargo.toml b/native/crates/c-shim/Cargo.toml new file mode 100644 index 0000000..1dde6da --- /dev/null +++ b/native/crates/c-shim/Cargo.toml @@ -0,0 +1,44 @@ +[package] +name = "kore-c-shim" +version = "0.1.0" +edition = "2024" +publish = false +description = "The C symbols a vendored C library imports on a kore user target" + +[features] +default = ["runtime", "malloc", "abort", "wide-classes"] +# The process half: the `` writes over the standard sink the process +# installs, `errno` and the parsers that set it. An executable takes it; a +# shared library leaves it out and imports those symbols from the executable +# it links into, so one process never holds two heaps. +runtime = ["dep:kore-rt"] +# `malloc`, `calloc`, `realloc` and `free` over the task's global heap. A C +# runtime that exports its own allocator under those names (libkore.so) +# leaves this out, so one process never holds two `malloc`s whose blocks +# the other's `free` would misread. +malloc = ["runtime"] +# `abort`, ending the calling task with the aborted status. A C runtime that +# ends a whole program its own way (libkore.so's host) leaves this out and +# exports its own. +abort = ["runtime"] +# The wide character classes (`iswalnum` … `iswxdigit`, `towlower`, +# `towupper`) a vendored C library calls: tree-sitter's scanners. Pure, so a +# shared library may carry its own. A C runtime whose headers declare no +# `` (libkore.so) leaves them out, so it exports exactly the ABI +# its headers declare. +wide-classes = [] +# `strtold` and `ldexpl` as byte-level helpers a consumer's C layer wraps +# (`long_double`): C's `long double` has no Rust spelling, so the C side owns +# the type and this crate the arithmetic. Off unless a consumer compiles C +# that converts `long double` (TCC does: its float literals). +long-double = ["runtime", "dep:rustc_apfloat"] + +[dependencies] +kore-rt = { path = "../rt", optional = true } +# LLVM's APFloat (the code clang folds float literals with): correctly +# rounded decimal conversion and scaling for x87 double-extended and IEEE +# quad. `no_std`, `forbid(unsafe_code)`; it allocates only to convert decimal. +rustc_apfloat = { version = "=0.2.3", optional = true } + +[dev-dependencies] +rustc_apfloat = "=0.2.3" diff --git a/native/crates/c-shim/src/alloc_symbols.rs b/native/crates/c-shim/src/alloc_symbols.rs new file mode 100644 index 0000000..6e97947 --- /dev/null +++ b/native/crates/c-shim/src/alloc_symbols.rs @@ -0,0 +1,107 @@ +//! `malloc`, `calloc`, `realloc` and `free` over the task's global heap. +//! +//! C frees without a size and Rust deallocates with one, so every block +//! carries its payload size in a header just below the pointer C sees. The +//! header is one `max_align_t` wide, so the payload keeps C's alignment +//! guarantee. Overflow and exhaustion return null, as C expects; a size of +//! zero returns null too, which `free` and `realloc` accept. + +extern crate alloc; + +use core::alloc::Layout; +use core::ffi::c_void; +use core::ptr; + +/// C's `max_align_t` on every kore target: the payload alignment `malloc` +/// promises, and the header's width. +const ALIGN: usize = 16; + +fn layout(payload: usize) -> Option { + Layout::from_size_align(payload.checked_add(ALIGN)?, ALIGN).ok() +} + +/// The payload size recorded below `payload`. +/// +/// # Safety +/// `payload` came from this module and is live. +unsafe fn payload_size(payload: *mut c_void) -> usize { + // SAFETY: the caller's contract; the header sits `ALIGN` bytes below. + unsafe { payload.cast::().sub(ALIGN).cast::().read() } +} + +/// Write `size` into the block's header and return its payload. +/// +/// # Safety +/// `block` is a live allocation of `layout(size)`. +unsafe fn finish(block: *mut u8, size: usize) -> *mut c_void { + if block.is_null() { + return ptr::null_mut(); + } + // SAFETY: the block is at least `ALIGN` bytes and `ALIGN`-aligned. + unsafe { + block.cast::().write(size); + block.add(ALIGN).cast() + } +} + +#[unsafe(no_mangle)] +extern "C" fn malloc(size: usize) -> *mut c_void { + if size == 0 { + return ptr::null_mut(); + } + let Some(layout) = layout(size) else { + return ptr::null_mut(); + }; + // SAFETY: the layout is non-zero; the header is written before return. + unsafe { finish(alloc::alloc::alloc(layout), size) } +} + +#[unsafe(no_mangle)] +extern "C" fn calloc(count: usize, size: usize) -> *mut c_void { + let Some(total) = count.checked_mul(size).filter(|&total| total != 0) else { + return ptr::null_mut(); + }; + let Some(layout) = layout(total) else { + return ptr::null_mut(); + }; + // SAFETY: as `malloc`; the zeroed allocation covers the payload. + unsafe { finish(alloc::alloc::alloc_zeroed(layout), total) } +} + +#[unsafe(no_mangle)] +unsafe extern "C" fn realloc(payload: *mut c_void, size: usize) -> *mut c_void { + if payload.is_null() { + return malloc(size); + } + if size == 0 { + // SAFETY: the caller's pointer came from this module. + unsafe { free(payload) }; + return ptr::null_mut(); + } + // SAFETY: a non-null pointer C passes back came from this module. + let old = unsafe { payload_size(payload) }; + let (Some(old_layout), Some(_)) = (layout(old), layout(size)) else { + return ptr::null_mut(); + }; + // SAFETY: the block and its layout are the live allocation; the new size + // fits a layout (checked above). + unsafe { + let block = payload.cast::().sub(ALIGN); + finish(alloc::alloc::realloc(block, old_layout, size + ALIGN), size) + } +} + +#[unsafe(no_mangle)] +unsafe extern "C" fn free(payload: *mut c_void) { + if payload.is_null() { + return; + } + // SAFETY: a non-null pointer C passes back came from this module, so + // its header names the layout it was allocated with. + unsafe { + let size = payload_size(payload); + if let Some(layout) = layout(size) { + alloc::alloc::dealloc(payload.cast::().sub(ALIGN), layout); + } + } +} diff --git a/native/crates/c-shim/src/errno.rs b/native/crates/c-shim/src/errno.rs new file mode 100644 index 0000000..7a5fbf8 --- /dev/null +++ b/native/crates/c-shim/src/errno.rs @@ -0,0 +1,96 @@ +//! `errno`: one per thread, as C requires, and the codes this crate and its +//! consumers set. +//! +//! The word is the calling thread's runtime-block slot +//! ([`kore_rt::thread::c_errno`]), not an ELF thread-local: a TLS segment in +//! every image that carries C would buy each of them a TCB migration, a +//! template every spawn must place, and `kore-late`'s refusal of a host with +//! TLS — all for one `int`. +//! +//! The numbers are musl's `` — the headers every kore C build +//! compiles against (`zig cc -target -linux-musl`), which use the +//! same values on x86_64, aarch64 and riscv64. A consumer's C code compares +//! `errno` against those headers' macros, so these must agree with them. + +use core::ffi::{CStr, c_int}; + +pub const EPERM: c_int = 1; +pub const ENOENT: c_int = 2; +pub const EIO: c_int = 5; +pub const ENXIO: c_int = 6; +pub const EBADF: c_int = 9; +pub const EAGAIN: c_int = 11; +pub const ENOMEM: c_int = 12; +pub const EACCES: c_int = 13; +pub const EFAULT: c_int = 14; +pub const EBUSY: c_int = 16; +pub const EEXIST: c_int = 17; +pub const ENOTDIR: c_int = 20; +pub const EISDIR: c_int = 21; +pub const EINVAL: c_int = 22; +pub const ENFILE: c_int = 23; +pub const EMFILE: c_int = 24; +pub const EFBIG: c_int = 27; +pub const ENOSPC: c_int = 28; +pub const ESPIPE: c_int = 29; +pub const EROFS: c_int = 30; +pub const ERANGE: c_int = 34; +pub const ENAMETOOLONG: c_int = 36; +pub const ENOSYS: c_int = 38; +pub const ENOTEMPTY: c_int = 39; +pub const ELOOP: c_int = 40; +pub const EOVERFLOW: c_int = 75; +pub const ENOTSUP: c_int = 95; + +/// Set this thread's `errno`. +pub fn set(code: c_int) { + // SAFETY: the calling thread's own word, live for its life and touched + // by no other thread (`kore_rt::thread::c_errno`'s contract). + unsafe { kore_rt::thread::c_errno().write(code) }; +} + +/// This thread's `errno`. +pub fn get() -> c_int { + // SAFETY: as `set`. + unsafe { kore_rt::thread::c_errno().read() } +} + +#[unsafe(no_mangle)] +extern "C" fn __errno_location() -> *mut c_int { + kore_rt::thread::c_errno() +} + +/// The message `strerror` answers for `code`. +pub fn describe(code: c_int) -> &'static CStr { + match code { + 0 => c"No error information", + EPERM => c"Operation not permitted", + ENOENT => c"No such file or directory", + EIO => c"I/O error", + ENXIO => c"No such device or address", + EBADF => c"Bad file descriptor", + EAGAIN => c"Resource temporarily unavailable", + ENOMEM => c"Out of memory", + EACCES => c"Permission denied", + EFAULT => c"Bad address", + EBUSY => c"Resource busy", + EEXIST => c"File exists", + ENOTDIR => c"Not a directory", + EISDIR => c"Is a directory", + EINVAL => c"Invalid argument", + ENFILE => c"Too many open files in system", + EMFILE => c"No file descriptors available", + EFBIG => c"File too large", + ENOSPC => c"No space left on device", + ESPIPE => c"Invalid seek", + EROFS => c"Read-only file system", + ERANGE => c"Result not representable", + ENAMETOOLONG => c"Filename too long", + ENOSYS => c"Function not implemented", + ENOTEMPTY => c"Directory not empty", + ELOOP => c"Symbolic link loop", + EOVERFLOW => c"Value too large for data type", + ENOTSUP => c"Not supported", + _ => c"Unknown error", + } +} diff --git a/native/crates/c-shim/src/format.rs b/native/crates/c-shim/src/format.rs new file mode 100644 index 0000000..0702d76 --- /dev/null +++ b/native/crates/c-shim/src/format.rs @@ -0,0 +1,724 @@ +//! The `printf` conversion engine behind every formatted write in this crate. +//! +//! One engine serves `snprintf` into a caller's buffer and `fprintf` into a +//! stream, so the two cannot disagree about what a conversion means. It is +//! generic over where arguments come from ([`Args`]: a C `va_list` on the +//! target, a scripted list in host tests) and where bytes go ([`Sink`]), and +//! it is pure: the host suite drives exactly the code the target runs. +//! +//! ## What it formats +//! +//! ISO C's integer, character, string and pointer conversions in full — +//! `d i o u x X c s p %`, the `- + space # 0` flags, field width and +//! precision (both also as `*`), and the `hh h l ll j z t` length modifiers. +//! A precision bounds a `%s` read, so `%.*s` over a counted, unterminated +//! string never reads past its count. `%p` is spelled `0x` and the address +//! in lowercase hexadecimal (the spelling is implementation-defined; this is +//! the one this crate has always used). +//! +//! The float conversions `e E f F g G a A` too, of a `double` and (with `L`) +//! of a `long double` in either of kore's formats, every flag, width and +//! precision honoured and every digit exact ([`float`]): the C programs +//! libkore.so runs print through this engine. +//! +//! ## What it deliberately does not +//! +//! - **`%n`** stores nothing: its pointer is consumed and ignored. A format +//! that writes through an argument is the classic format-string attack, +//! and no consumer needs one. +//! - **Wide characters**: an ASCII `%lc` is written as its byte and anything +//! else as `?`; `%ls` consumes its pointer and is spelled back. +//! +//! An unknown conversion is spelled back and consumes nothing — its type +//! cannot be known, and guessing would misalign every argument after it. + +use core::ffi::c_int; +use core::marker::PhantomData; + +mod float; + +pub use float::Float; + +/// The C type a length modifier names. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Length { + /// `hh`: `char`, promoted to `int` in a variadic call. + Char, + /// `h`: `short`, promoted to `int`. + Short, + /// No modifier: `int`. + Int, + /// `l`: `long`. + Long, + /// `ll` (and BSD's `q`): `long long`. + LongLong, + /// `j`: `intmax_t`. + Max, + /// `z`: `size_t`. + Size, + /// `t`: `ptrdiff_t`. + Diff, + /// `L`: `long double` beside a float conversion; `long long` beside an + /// integer one, as glibc reads it. + LongDouble, +} + +/// Where each conversion's argument comes from. +/// +/// Every method consumes exactly one argument of the C type it names, as a +/// variadic callee must. +pub trait Args { + /// A signed integer of `length`'s type, sign-extended. + fn signed(&mut self, length: Length) -> i64; + /// An unsigned integer of `length`'s type, zero-extended. + fn unsigned(&mut self, length: Length) -> u64; + /// An `int`: a `%c` argument, or a `*` width or precision. + fn int(&mut self) -> c_int; + /// A pointer, as an address: `%p`, `%n`, `%ls`, and `%s`'s string. + fn pointer(&mut self) -> usize; + /// A `double` (a `float` arrives promoted to one). + fn double(&mut self) -> f64; + /// A `long double`, read in the C ABI's own format. Rust has no type for + /// it, so the source decodes it. + fn long_double(&mut self) -> Float; +} + +/// Where formatted bytes go. The engine counts them; a sink only stores. +pub trait Sink { + /// Accept the next run of output bytes. + fn put(&mut self, bytes: &[u8]); +} + +/// Format `format` into `sink`, drawing arguments from `args`, and return +/// how many bytes the complete output has — the count `snprintf` returns +/// even when its buffer is shorter. +/// +/// # Safety +/// `format` is NUL-terminated, and its conversions match the arguments +/// `args` yields. +pub unsafe fn format(sink: &mut impl Sink, format: *const u8, args: &mut impl Args) -> usize { + let format = Format { + base: format, + _bytes: PhantomData, + }; + let mut out = Out { sink, count: 0 }; + let mut at = 0usize; + loop { + // The literal run up to the next conversion, written whole. + let start = at; + // SAFETY (every `format.at` / `format.slice` below): reads stop at + // the terminator, and `at` only advances past bytes already read as + // non-NUL. + while !matches!(unsafe { format.at(at) }, 0 | b'%') { + at += 1; + } + out.put(unsafe { format.slice(start, at) }); + if unsafe { format.at(at) } == 0 { + return out.count; + } + let spec_start = at; + at += 1; + + let mut spec = Spec::default(); + loop { + match unsafe { format.at(at) } { + b'-' => spec.left = true, + b'+' => spec.plus = true, + b' ' => spec.space = true, + b'#' => spec.alternate = true, + b'0' => spec.zero = true, + _ => break, + } + at += 1; + } + if unsafe { format.at(at) } == b'*' { + at += 1; + let width = args.int(); + // A negative `*` width is a `-` flag and a positive width. + spec.left |= width < 0; + spec.width = (width.unsigned_abs() as usize).min(MAX_FIELD); + } else { + spec.width = unsafe { format.decimal(&mut at) }; + } + if unsafe { format.at(at) } == b'.' { + at += 1; + if unsafe { format.at(at) } == b'*' { + at += 1; + // A negative precision is taken as if it were omitted. + spec.precision = usize::try_from(args.int()).ok(); + } else { + spec.precision = Some(unsafe { format.decimal(&mut at) }); + } + } + let (length, skip) = match unsafe { [format.at(at), format.at(at + 1)] } { + [b'h', b'h'] => (Length::Char, 2), + [b'h', _] => (Length::Short, 1), + [b'l', b'l'] => (Length::LongLong, 2), + [b'l', _] => (Length::Long, 1), + [b'q', _] => (Length::LongLong, 1), + [b'j', _] => (Length::Max, 1), + [b'z', _] => (Length::Size, 1), + [b't', _] => (Length::Diff, 1), + [b'L', _] => (Length::LongDouble, 1), + _ => (Length::Int, 0), + }; + at += skip; + let conversion = unsafe { format.at(at) }; + if conversion == 0 { + // A specification cut off by the terminator: spell it back. + out.put(unsafe { format.slice(spec_start, at) }); + return out.count; + } + at += 1; + // The specification's own bytes, for the conversions spelled back. + let spelled = unsafe { format.slice(spec_start, at) }; + + match conversion { + b'%' => out.put(b"%"), + b'd' | b'i' => { + let value = args.signed(length); + integer( + &mut out, + spec, + value.unsigned_abs(), + value < 0, + Radix::Signed, + ); + } + b'u' => integer(&mut out, spec, args.unsigned(length), false, Radix::Decimal), + b'o' => integer(&mut out, spec, args.unsigned(length), false, Radix::Octal), + b'x' => integer(&mut out, spec, args.unsigned(length), false, Radix::Hex), + b'X' => integer( + &mut out, + spec, + args.unsigned(length), + false, + Radix::HexUpper, + ), + b'c' => { + let value = args.int(); + let byte = if length == Length::Long { + // `wint_t`: only an ASCII code point has a byte here. + u8::try_from(value) + .ok() + .filter(u8::is_ascii) + .unwrap_or(b'?') + } else { + value as u8 + }; + text(&mut out, spec, &[byte]); + } + b's' if length == Length::Long => { + args.pointer(); + out.put(spelled); + } + b's' => { + let address = args.pointer() as *const u8; + if address.is_null() { + text(&mut out, spec, b"(null)"); + } else { + // SAFETY: C promises a string readable to its NUL, or to + // the precision when one bounds the read. + let bytes = unsafe { c_string(address, spec.precision) }; + text(&mut out, spec, bytes); + } + } + b'p' => { + let mut digits = [0u8; 22]; + let digits = render(args.pointer() as u64, Radix::Hex, &mut digits); + let mut buffer = [0u8; 18]; + buffer[..2].copy_from_slice(b"0x"); + buffer[2..2 + digits.len()].copy_from_slice(digits); + text(&mut out, spec, &buffer[..2 + digits.len()]); + } + b'n' => { + args.pointer(); + } + b'e' | b'E' | b'f' | b'F' | b'g' | b'G' | b'a' | b'A' => { + if length == Length::LongDouble { + let value = args.long_double(); + float::convert::<{ float::LONG_DOUBLE_LIMBS }, _>( + &mut out, spec, conversion, value, + ); + } else { + let value = Float::from_f64(args.double()); + float::convert::<{ float::DOUBLE_LIMBS }, _>(&mut out, spec, conversion, value); + } + } + _ => out.put(spelled), + } + } +} + +/// A NUL-terminated C format string. +struct Format<'f> { + base: *const u8, + _bytes: PhantomData<&'f [u8]>, +} + +impl<'f> Format<'f> { + /// # Safety + /// `at` lies at or before the terminator. + unsafe fn at(&self, at: usize) -> u8 { + // SAFETY: the caller's contract. + unsafe { *self.base.add(at) } + } + + /// The bytes in `start..end`. + /// + /// # Safety + /// The range was read and lies before the terminator. + unsafe fn slice(&self, start: usize, end: usize) -> &'f [u8] { + // SAFETY: the caller's contract. + unsafe { core::slice::from_raw_parts(self.base.add(start), end - start) } + } + + /// A run of decimal digits as a field width or precision, saturating at + /// [`MAX_FIELD`]. + /// + /// # Safety + /// `at` lies at or before the terminator. + unsafe fn decimal(&self, at: &mut usize) -> usize { + let mut value = 0usize; + // SAFETY: the caller's contract; a digit is never the terminator. + while let digit @ b'0'..=b'9' = unsafe { self.at(*at) } { + value = value + .saturating_mul(10) + .saturating_add(usize::from(digit - b'0')) + .min(MAX_FIELD); + *at += 1; + } + value + } +} + +/// A `*` or decimal width or precision is an `int`; this is the largest. +const MAX_FIELD: usize = c_int::MAX as usize; + +/// Output padding comes from these, a run at a time. +const SPACES: [u8; 32] = [b' '; 32]; +const ZEROS: [u8; 32] = [b'0'; 32]; + +/// The engine's counting front: every byte a sink sees passes through here. +struct Out<'s, S: Sink> { + sink: &'s mut S, + count: usize, +} + +impl Out<'_, S> { + fn put(&mut self, bytes: &[u8]) { + if !bytes.is_empty() { + self.sink.put(bytes); + self.count = self.count.saturating_add(bytes.len()); + } + } + + fn pad(&mut self, fill: &[u8; 32], mut count: usize) { + while count > 0 { + let run = count.min(fill.len()); + self.put(&fill[..run]); + count -= run; + } + } +} + +/// One parsed conversion specification. +#[derive(Clone, Copy, Debug, Default)] +struct Spec { + left: bool, + plus: bool, + space: bool, + alternate: bool, + zero: bool, + width: usize, + precision: Option, +} + +#[derive(Clone, Copy, PartialEq, Eq)] +enum Radix { + Octal, + /// `%d`/`%i`: decimal, and the only radix the `+` and space flags mark. + Signed, + /// `%u`. + Decimal, + Hex, + HexUpper, +} + +/// The bytes of the C string at `address`, stopping at its NUL or, when +/// `limit` is given, after `limit` bytes — whichever comes first. +/// +/// # Safety +/// `address` is readable up to its NUL or `limit`. +unsafe fn c_string<'s>(address: *const u8, limit: Option) -> &'s [u8] { + let limit = limit.unwrap_or(usize::MAX); + let mut len = 0usize; + // SAFETY: the caller's contract bounds every read. + while len < limit && unsafe { *address.add(len) } != 0 { + len += 1; + } + unsafe { core::slice::from_raw_parts(address, len) } +} + +/// `value`'s digits, written into the tail of `buffer` (22 bytes hold a u64 +/// in octal). +fn render(mut value: u64, radix: Radix, buffer: &mut [u8; 22]) -> &[u8] { + let (base, digits): (u64, &[u8; 16]) = match radix { + Radix::Octal => (8, b"0123456789abcdef"), + Radix::Signed | Radix::Decimal => (10, b"0123456789abcdef"), + Radix::Hex => (16, b"0123456789abcdef"), + Radix::HexUpper => (16, b"0123456789ABCDEF"), + }; + let mut at = buffer.len(); + loop { + at -= 1; + buffer[at] = digits[(value % base) as usize]; + value /= base; + if value == 0 { + break; + } + } + &buffer[at..] +} + +/// An integer conversion: sign or prefix, the precision's leading zeros, +/// digits, and the field's padding, placed as ISO C 7.21.6.1 places them. +fn integer(out: &mut Out<'_, impl Sink>, spec: Spec, magnitude: u64, negative: bool, radix: Radix) { + let mut buffer = [0u8; 22]; + let mut digits = render(magnitude, radix, &mut buffer); + // A zero precision prints no digits for a zero value. + if magnitude == 0 && spec.precision == Some(0) { + digits = &[]; + } + let prefix: &[u8] = match radix { + _ if negative => b"-", + Radix::Signed if spec.plus => b"+", + Radix::Signed if spec.space => b" ", + Radix::Hex if spec.alternate && magnitude != 0 => b"0x", + Radix::HexUpper if spec.alternate && magnitude != 0 => b"0X", + _ => b"", + }; + let mut leading = spec + .precision + .map_or(0, |precision| precision.saturating_sub(digits.len())); + // `#` with `o` raises the precision just enough for a leading zero. + if spec.alternate && radix == Radix::Octal && leading == 0 && digits.first() != Some(&b'0') { + leading = 1; + } + let fill = spec + .width + .saturating_sub(prefix.len() + leading + digits.len()); + // `0` pads between the prefix and the digits, unless a precision was + // given or the field is left-justified. + let zero_fill = spec.zero && !spec.left && spec.precision.is_none(); + if !spec.left && !zero_fill { + out.pad(&SPACES, fill); + } + out.put(prefix); + if zero_fill { + out.pad(&ZEROS, fill); + } + out.pad(&ZEROS, leading); + out.put(digits); + if spec.left { + out.pad(&SPACES, fill); + } +} + +/// A text conversion (`%c`, `%s`, `%p`): the bytes and the field's padding. +/// The `0` flag means nothing here, as in C. +fn text(out: &mut Out<'_, impl Sink>, spec: Spec, bytes: &[u8]) { + let fill = spec.width.saturating_sub(bytes.len()); + if !spec.left { + out.pad(&SPACES, fill); + } + out.put(bytes); + if spec.left { + out.pad(&SPACES, fill); + } +} + +#[cfg(test)] +mod tests { + use super::*; + extern crate std; + use std::{collections::VecDeque, string::String, vec::Vec}; + + /// One scripted argument. + #[derive(Clone, Copy, Debug)] + enum Arg { + Int(i64), + Uint(u64), + Ptr(usize), + Double(f64), + LongDouble(Float), + } + + /// A NUL-terminated (or, for `%.*s`, counted) string argument. + fn text(bytes: &'static [u8]) -> Arg { + Arg::Ptr(bytes.as_ptr() as usize) + } + + struct Script { + args: VecDeque, + /// Each consumption, so a test can prove alignment. + taken: Vec<&'static str>, + } + + impl Script { + fn take(&mut self, kind: &'static str) -> Arg { + self.taken.push(kind); + self.args + .pop_front() + .expect("the format consumed more arguments than given") + } + } + + impl Args for Script { + fn signed(&mut self, length: Length) -> i64 { + let Arg::Int(value) = self.take("signed") else { + panic!("a signed conversion met another type") + }; + // The truncation a C callee applies to a promoted argument. + match length { + Length::Char => value as i8 as i64, + Length::Short => value as i16 as i64, + Length::Int => value as i32 as i64, + _ => value, + } + } + fn unsigned(&mut self, length: Length) -> u64 { + let value = match self.take("unsigned") { + Arg::Uint(value) => value, + Arg::Int(value) => value as u64, + other => panic!("an unsigned conversion met {other:?}"), + }; + match length { + Length::Char => value as u8 as u64, + Length::Short => value as u16 as u64, + Length::Int => value as u32 as u64, + _ => value, + } + } + fn int(&mut self) -> c_int { + let Arg::Int(value) = self.take("int") else { + panic!("an int met another type") + }; + value as c_int + } + fn pointer(&mut self) -> usize { + let Arg::Ptr(value) = self.take("pointer") else { + panic!("a pointer met another type") + }; + value + } + fn double(&mut self) -> f64 { + let Arg::Double(value) = self.take("double") else { + panic!("a double met another type") + }; + value + } + fn long_double(&mut self) -> Float { + let Arg::LongDouble(value) = self.take("long double") else { + panic!("a long double met another type") + }; + value + } + } + + struct Collect(Vec); + + impl Sink for Collect { + fn put(&mut self, bytes: &[u8]) { + self.0.extend_from_slice(bytes); + } + } + + /// Format a NUL-terminated `format`; return the text, count and script. + fn run(format: &[u8], args: &[Arg]) -> (String, usize, Vec<&'static str>) { + assert_eq!(format.last(), Some(&0), "test formats are NUL-terminated"); + let mut script = Script { + args: args.iter().copied().collect(), + taken: Vec::new(), + }; + let mut sink = Collect(Vec::new()); + // SAFETY: the format is NUL-terminated and the script matches it. + let count = unsafe { super::format(&mut sink, format.as_ptr(), &mut script) }; + assert!( + script.args.is_empty(), + "the format left arguments unconsumed" + ); + (String::from_utf8(sink.0).unwrap(), count, script.taken) + } + + fn check(format: &[u8], args: &[Arg], expected: &str) { + let (out, count, _) = run(format, args); + assert_eq!( + out, + expected, + "format {:?}", + String::from_utf8_lossy(format) + ); + assert_eq!(count, expected.len()); + } + + // Integer, character and string expectations match the host C library's + // printf for the same format and arguments. + + #[test] + fn literal_text_and_percent() { + check(b"plain\0", &[], "plain"); + check(b"100%% sure\0", &[], "100% sure"); + check(b"\0", &[], ""); + } + + #[test] + fn signed_decimal_with_flags_width_and_precision() { + check(b"%d\0", &[Arg::Int(42)], "42"); + check(b"%d\0", &[Arg::Int(-42)], "-42"); + check(b"%i\0", &[Arg::Int(0)], "0"); + check(b"%5d|\0", &[Arg::Int(42)], " 42|"); + check(b"%-5d|\0", &[Arg::Int(42)], "42 |"); + check(b"%05d\0", &[Arg::Int(-42)], "-0042"); + check(b"%+d %+d\0", &[Arg::Int(7), Arg::Int(-7)], "+7 -7"); + check(b"% d % d\0", &[Arg::Int(7), Arg::Int(-7)], " 7 -7"); + check(b"%.3d\0", &[Arg::Int(7)], "007"); + check(b"%6.3d|\0", &[Arg::Int(-7)], " -007|"); + check(b"%06.3d|\0", &[Arg::Int(7)], " 007|"); + check(b"%.0d|\0", &[Arg::Int(0)], "|"); + check(b"%-2d|\0", &[Arg::Int(3)], "3 |"); + check(b"%d\0", &[Arg::Int(i32::MIN as i64)], "-2147483648"); + check(b"%lld\0", &[Arg::Int(i64::MIN)], "-9223372036854775808"); + check(b"%ld\0", &[Arg::Int(1 << 40)], "1099511627776"); + check(b"%hhd\0", &[Arg::Int(300)], "44"); + check(b"%hd\0", &[Arg::Int(70000)], "4464"); + } + + #[test] + fn unsigned_octal_and_hexadecimal() { + check(b"%u\0", &[Arg::Uint(4294967295)], "4294967295"); + check( + b"%x %X\0", + &[Arg::Uint(0xbeef), Arg::Uint(0xbeef)], + "beef BEEF", + ); + check(b"%08x\0", &[Arg::Uint(0x1a)], "0000001a"); + check(b"%#x %#X\0", &[Arg::Uint(255), Arg::Uint(255)], "0xff 0XFF"); + check(b"%#x\0", &[Arg::Uint(0)], "0"); + check(b"%#010x\0", &[Arg::Uint(255)], "0x000000ff"); + check( + b"%o %#o %#o\0", + &[Arg::Uint(8), Arg::Uint(8), Arg::Uint(0)], + "10 010 0", + ); + check(b"%#.0o\0", &[Arg::Uint(0)], "0"); + check(b"%.8x\0", &[Arg::Uint(0x1234)], "00001234"); + check(b"%3X|\0", &[Arg::Uint(0xa)], " A|"); + check(b"%8.2X|\0", &[Arg::Uint(0x5)], " 05|"); + check(b"%lx\0", &[Arg::Uint(0xdead_beef_cafe)], "deadbeefcafe"); + check(b"%08llx\0", &[Arg::Uint(0x12)], "00000012"); + check(b"%llu\0", &[Arg::Uint(u64::MAX)], "18446744073709551615"); + check( + b"%zu %zx\0", + &[Arg::Uint(4096), Arg::Uint(4096)], + "4096 1000", + ); + check(b"%hhx\0", &[Arg::Uint(0x1ff)], "ff"); + // `+` and space belong to signed conversions only. + check(b"%+u % u\0", &[Arg::Uint(5), Arg::Uint(5)], "5 5"); + } + + #[test] + fn characters_strings_and_pointers() { + check(b"[%c]\0", &[Arg::Int(b'A' as i64)], "[A]"); + check( + b"[%3c][%-3c]\0", + &[Arg::Int(b'x' as i64), Arg::Int(b'y' as i64)], + "[ x][y ]", + ); + check(b"%s\0", &[text(b"hello\0")], "hello"); + check( + b"[%8s][%-8s]\0", + &[text(b"ab\0"), text(b"cd\0")], + "[ ab][cd ]", + ); + check(b"%.2s\0", &[text(b"hello\0")], "he"); + check(b"%s\0", &[Arg::Ptr(0)], "(null)"); + check( + b"%lc %lc\0", + &[Arg::Int(b'q' as i64), Arg::Int(0x263a)], + "q ?", + ); + // `%p` is this crate's own spelling, not the host's. + check(b"%p\0", &[Arg::Ptr(0x1000)], "0x1000"); + check(b"%p\0", &[Arg::Ptr(0)], "0x0"); + check(b"[%10p]\0", &[Arg::Ptr(0xabc)], "[ 0xabc]"); + } + + #[test] + fn a_precision_bounds_the_read_of_an_unterminated_string() { + // No NUL in these bytes: `%.*s` must stop at its count. + check(b"[%.*s]\0", &[Arg::Int(3), text(b"abcd")], "[abc]"); + check(b"[%.4s]\0", &[text(b"abcd")], "[abcd]"); + } + + #[test] + fn star_width_and_precision() { + check(b"[%*d]\0", &[Arg::Int(5), Arg::Int(42)], "[ 42]"); + check(b"[%*d]\0", &[Arg::Int(-5), Arg::Int(42)], "[42 ]"); + check(b"[%*s]\0", &[Arg::Int(4), text(b"x\0")], "[ x]"); + check(b"[%.*s]\0", &[Arg::Int(2), text(b"xyz\0")], "[xy]"); + // A negative precision is as if it were absent. + check(b"[%.*s]\0", &[Arg::Int(-1), text(b"xyz\0")], "[xyz]"); + check(b"[%.*d]\0", &[Arg::Int(4), Arg::Int(7)], "[0007]"); + } + + #[test] + fn floats_are_formatted_and_the_arguments_after_them_line_up() { + let (out, _, taken) = run(b"%0.3f s, %u lines\0", &[Arg::Double(1.5), Arg::Uint(9)]); + assert_eq!(out, "1.500 s, 9 lines"); + assert_eq!(taken, ["double", "unsigned"]); + // `L` reads a long double, in the source's own format. + let x87_two_and_a_half = Float::from_x87(0x4000_a000_0000_0000_0000); + let (out, _, taken) = run( + b"a %Lf then %d\0", + &[Arg::LongDouble(x87_two_and_a_half), Arg::Int(7)], + ); + assert_eq!(out, "a 2.500000 then 7"); + assert_eq!(taken, ["long double", "signed"]); + check( + b"[%8.2e|%-8g|%+a]\0", + &[Arg::Double(-0.0), Arg::Double(1e-5), Arg::Double(0.5)], + "[-0.00e+00|1e-05 |+0x1p-1]", + ); + } + + #[test] + fn n_stores_nothing_and_unknown_conversions_consume_nothing() { + let (out, _, taken) = run(b"ab%ncd\0", &[Arg::Ptr(0x10)]); + assert_eq!(out, "abcd"); + assert_eq!(taken, ["pointer"]); + let (out, _, taken) = run(b"(%rip) %d\0", &[Arg::Int(1)]); + assert_eq!(out, "(%rip) 1"); + assert_eq!(taken, ["signed"]); + } + + #[test] + fn a_specification_cut_off_by_the_terminator_is_spelled_back() { + check(b"tail %-05\0", &[], "tail %-05"); + check(b"tail %\0", &[], "tail %"); + } + + #[test] + fn a_field_width_saturates_at_int_max() { + let digits = b"99999999999999999999x\0"; + let format = Format { + base: digits.as_ptr(), + _bytes: PhantomData, + }; + let mut at = 0; + // SAFETY: NUL-terminated; the scan stops at the `x`. + assert_eq!(unsafe { format.decimal(&mut at) }, MAX_FIELD); + assert_eq!(at, 20); + } +} diff --git a/native/crates/c-shim/src/format/float.rs b/native/crates/c-shim/src/format/float.rs new file mode 100644 index 0000000..2766af5 --- /dev/null +++ b/native/crates/c-shim/src/format/float.rs @@ -0,0 +1,866 @@ +//! The float conversions of the `printf` engine ([`super::format`]): `%e %E +//! %f %F %g %G %a %A`, of a `double` and of a `long double` in either format +//! kore's C ABIs give it — x87 double-extended on x86_64, IEEE binary128 on +//! aarch64 and riscv64. +//! +//! ## Every digit is the value's +//! +//! A finite float is `mantissa × 2^exponent` with an integer mantissa, so its +//! decimal expansion is finite, and a conversion prints that expansion +//! rounded once, to nearest with ties to even, at the last digit it keeps: +//! the correctly rounded conversion C asks for in the default rounding mode +//! (C11 7.21.6.1p13, F.5), and what glibc and musl print. A [`Decimal`] holds +//! the expansion in base-10⁹ limbs, built from the mantissa by doubling 29 +//! bits or halving 9 bits at a time (musl's scheme, kept in integers), so no +//! digit is estimated. +//! +//! Halving moves digits only toward the low end: a limb below the last digit +//! a conversion can keep never changes one above it. Such limbs are folded +//! into one sticky bit as they appear, which is all a round to nearest asks +//! of them, so a conversion stores at most the digits it prints and two limbs +//! more, and never more than the expansion has: a `%.1000000f` of a `double` +//! costs what its 1074 binary places give. Doubling carries upward, so an +//! integer part is built whole: at most 4933 digits, for a `long double`. +//! +//! ## Spellings +//! +//! An infinity is `inf` and a NaN `nan`, upper case under `E F G A`, each +//! signed as any value is: a NaN's sign bit is its sign, as glibc and musl +//! print it. `%a` normalizes every nonzero value to a leading `1`, subnormals +//! included, as musl does (glibc spells a subnormal `double` with a leading +//! `0`, and an x87 value with its explicit integer bit); with a precision it +//! rounds to nearest-even, which can carry the leading digit to `2`. + +use core::cmp::Ordering; + +use super::{Out, SPACES, Sink, Spec, ZEROS}; + +/// A float as a conversion reads it: its sign bit, and what it holds. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Float { + negative: bool, + class: Class, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum Class { + /// `mantissa × 2^exponent`; a zero mantissa is a zero of that sign. + Finite { + mantissa: u128, + exponent: i32, + }, + Infinite, + Nan, +} + +/// IEEE binary64: 52 stored fraction bits under an 11-bit exponent, the sign +/// on top. +const F64_FRACTION: u32 = 52; +const F64_BIAS: i32 = 1023; +const F64_EXPONENT_MASK: u64 = 0x7ff; +const F64_SIGN: u32 = u64::BITS - 1; + +/// x87 double-extended: a 64-bit significand whose integer bit is stored, +/// under a 15-bit exponent and the sign, in the low 80 bits. +const X87_SIGNIFICAND: u32 = u64::BITS; +const X87_FRACTION: u32 = X87_SIGNIFICAND - 1; +const X87_BIAS: i32 = 16383; +const X87_SIGN: u32 = X87_SIGNIFICAND + 15; + +/// IEEE binary128: 112 stored fraction bits under a 15-bit exponent, the +/// sign on top. +const QUAD_FRACTION: u32 = 112; +const QUAD_BIAS: i32 = 16383; +const QUAD_SIGN: u32 = u128::BITS - 1; + +/// The 15-bit exponent field both `long double` formats share. +const WIDE_EXPONENT_MASK: u128 = 0x7fff; + +impl Float { + /// A `double`. + pub fn from_f64(value: f64) -> Self { + let bits = value.to_bits(); + let biased = (bits >> F64_FRACTION) & F64_EXPONENT_MASK; + let fraction = u128::from(bits & ((1 << F64_FRACTION) - 1)); + let class = match biased { + F64_EXPONENT_MASK if fraction == 0 => Class::Infinite, + F64_EXPONENT_MASK => Class::Nan, + // Subnormal: the least exponent, and no implicit bit. + 0 => Class::Finite { + mantissa: fraction, + exponent: 1 - F64_BIAS - F64_FRACTION as i32, + }, + _ => Class::Finite { + mantissa: fraction | 1 << F64_FRACTION, + exponent: biased as i32 - F64_BIAS - F64_FRACTION as i32, + }, + }; + Self { + negative: bits >> F64_SIGN != 0, + class, + } + } + + /// An x87 double-extended `long double`: its 80 bits, low-aligned (any + /// padding above them is ignored). + /// + /// The encodings no x87 since the 387 produces are read as musl reads + /// them: an unnormal (a nonzero exponent and a clear integer bit) is a + /// NaN, a pseudo-infinity (the top exponent, a clear integer bit and a + /// zero fraction) an infinity, and a pseudo-denormal (a zero exponent and + /// a set integer bit) the value its bits spell. + pub fn from_x87(bits: u128) -> Self { + let negative = (bits >> X87_SIGN) & 1 != 0; + let biased = (bits >> X87_SIGNIFICAND) & WIDE_EXPONENT_MASK; + let significand = bits as u64; + let integer = significand >> X87_FRACTION != 0; + let class = match biased { + WIDE_EXPONENT_MASK if significand << 1 == 0 => Class::Infinite, + WIDE_EXPONENT_MASK => Class::Nan, + 0 => Class::Finite { + mantissa: u128::from(significand), + exponent: 1 - X87_BIAS - X87_FRACTION as i32, + }, + _ if !integer => Class::Nan, + _ => Class::Finite { + mantissa: u128::from(significand), + exponent: biased as i32 - X87_BIAS - X87_FRACTION as i32, + }, + }; + Self { negative, class } + } + + /// An IEEE binary128 `long double`. + pub fn from_quad(bits: u128) -> Self { + let biased = (bits >> QUAD_FRACTION) & WIDE_EXPONENT_MASK; + let fraction = bits & ((1 << QUAD_FRACTION) - 1); + let class = match biased { + WIDE_EXPONENT_MASK if fraction == 0 => Class::Infinite, + WIDE_EXPONENT_MASK => Class::Nan, + 0 => Class::Finite { + mantissa: fraction, + exponent: 1 - QUAD_BIAS - QUAD_FRACTION as i32, + }, + _ => Class::Finite { + mantissa: fraction | 1 << QUAD_FRACTION, + exponent: biased as i32 - QUAD_BIAS - QUAD_FRACTION as i32, + }, + }; + Self { + negative: bits >> QUAD_SIGN != 0, + class, + } + } +} + +/// The precision `e`, `f` and `g` take when none is given (C11 7.21.6.1p8). +const DEFAULT_PRECISION: i64 = 6; + +/// One base-10⁹ limb's radix. +const BASE: u32 = 1_000_000_000; + +/// Decimal digits per limb. +const LIMB_DIGITS: i64 = 9; + +/// `POW10[k]` is `10^k`, up to a limb's radix. +const POW10: [u32; 10] = [ + 1, + 10, + 100, + 1_000, + 10_000, + 100_000, + 1_000_000, + 10_000_000, + 100_000_000, + 1_000_000_000, +]; + +/// Bits one doubling pass shifts in: a limb below 10⁹ times 2²⁹, plus the +/// carry below 2²⁹ + 1, stays under 2⁶³, and the carry out fits one limb. +const DOUBLE_STEP: u32 = 29; + +/// Bits one halving pass shifts out. 10⁹ = 2⁹·5⁹, so a limb's remainder +/// below 2⁹ passes to the next limb as exactly `remainder × 10⁹ / 2⁹`, and +/// a pass adds at most one limb below the last. +const HALVE_STEP: u32 = 9; + +/// Limbs kept free in front of an expansion: rounding at the digit just +/// above it materializes one limb, and a carry out of the top one more. +const HEADROOM: usize = 2; + +/// Decimal digits an integer of `bits` bits can have: `⌊bits·log₁₀2⌋ + 1`, +/// by 30103/100000, which exceeds log₁₀2. +const fn decimal_digits(bits: u32) -> usize { + bits as usize * 30103 / 100_000 + 1 +} + +/// Limbs a mantissa occupies before it is scaled: any `u128`. +const MANTISSA_LIMBS: usize = decimal_digits(u128::BITS).div_ceil(LIMB_DIGITS as usize); + +/// The limbs a format's widest expansion needs: its largest integer part +/// (`mantissa_bits + max_exponent` bits, which doubling builds whole), or its +/// longest fraction (one digit per binary place below the point, and the +/// window's one limb more) beside the integer part a fraction can have; and +/// the rounding headroom. +const fn capacity(mantissa_bits: u32, max_exponent: i32, min_exponent: i32) -> usize { + let whole = decimal_digits(mantissa_bits + max_exponent as u32).div_ceil(LIMB_DIGITS as usize); + let fraction = (min_exponent.unsigned_abs() as usize).div_ceil(LIMB_DIGITS as usize) + + 1 + + decimal_digits(mantissa_bits).div_ceil(LIMB_DIGITS as usize); + (if whole > fraction { whole } else { fraction }) + HEADROOM +} + +const fn max(a: usize, b: usize) -> usize { + if a > b { a } else { b } +} + +/// A `double`'s expansion: 125 limbs. +pub(super) const DOUBLE_LIMBS: usize = capacity( + F64_FRACTION + 1, + (2 * F64_BIAS) - F64_BIAS - F64_FRACTION as i32, + 1 - F64_BIAS - F64_FRACTION as i32, +); + +/// A `long double`'s expansion, in either format: 1840 limbs. +pub(super) const LONG_DOUBLE_LIMBS: usize = max( + capacity( + X87_FRACTION + 1, + (2 * X87_BIAS) - X87_BIAS - X87_FRACTION as i32, + 1 - X87_BIAS - X87_FRACTION as i32, + ), + capacity( + QUAD_FRACTION + 1, + (2 * QUAD_BIAS) - QUAD_BIAS - QUAD_FRACTION as i32, + 1 - QUAD_BIAS - QUAD_FRACTION as i32, + ), +); + +/// The decimal expansion of a finite nonzero value, most significant limb +/// first: `limbs[start..start + len]`, the first of which is `10^(9·top)`'s. +/// Digits below the last stored limb are zero, or — when `sticky` is set — +/// include a nonzero one the window let go. +struct Decimal { + limbs: [u32; N], + start: usize, + len: usize, + top: i32, + sticky: bool, +} + +impl Decimal { + /// `mantissa × 2^exponent` (`mantissa` nonzero), exact in every limb whose + /// base-10⁹ exponent is at least `cut`. + fn new(mantissa: u128, exponent: i32, cut: i64) -> Self { + let mut low_first = [0u32; MANTISSA_LIMBS]; + let mut count = 0; + let mut rest = mantissa; + while rest != 0 { + low_first[count] = (rest % u128::from(BASE)) as u32; + rest /= u128::from(BASE); + count += 1; + } + // Doubling grows toward the front and halving toward the back. + let start = if exponent > 0 { N - count } else { HEADROOM }; + let mut decimal = Self { + limbs: [0; N], + start, + len: count, + top: count as i32 - 1, + sticky: false, + }; + for (slot, &limb) in decimal.limbs[start..start + count] + .iter_mut() + .zip(low_first[..count].iter().rev()) + { + *slot = limb; + } + match exponent.cmp(&0) { + Ordering::Greater => decimal.double(exponent.unsigned_abs()), + Ordering::Less => decimal.halve(exponent.unsigned_abs(), cut), + Ordering::Equal => {} + } + decimal + } + + fn double(&mut self, mut shift: u32) { + while shift > 0 { + let step = shift.min(DOUBLE_STEP); + let mut carry = 0u64; + for limb in self.limbs[self.start..self.start + self.len] + .iter_mut() + .rev() + { + let wide = (u64::from(*limb) << step) + carry; + *limb = (wide % u64::from(BASE)) as u32; + carry = wide / u64::from(BASE); + } + if carry != 0 { + self.start -= 1; + self.len += 1; + self.top += 1; + self.limbs[self.start] = carry as u32; + } + shift -= step; + } + } + + fn halve(&mut self, mut shift: u32, cut: i64) { + self.fold_below(cut); + while shift > 0 && self.len > 0 { + let step = shift.min(HALVE_STEP); + let mask = (1u32 << step) - 1; + let share = BASE >> step; + let mut carry = 0u32; + for limb in &mut self.limbs[self.start..self.start + self.len] { + let rest = *limb & mask; + *limb = (*limb >> step) + carry; + carry = share * rest; + } + // Only the leading limb can empty in a pass: the one after it + // gains the leading limb's whole remainder. + if self.limbs[self.start] == 0 { + self.start += 1; + self.len -= 1; + self.top -= 1; + } + if carry != 0 { + if i64::from(self.top) - self.len as i64 >= cut { + self.limbs[self.start + self.len] = carry; + self.len += 1; + } else { + self.sticky = true; + } + } + shift -= step; + } + } + + /// Fold every limb below the base-10⁹ exponent `cut` into `sticky`. + fn fold_below(&mut self, cut: i64) { + let keep = (i64::from(self.top) - cut + 1).clamp(0, self.len as i64) as usize; + let end = self.start + self.len; + if self.limbs[self.start + keep..end] + .iter() + .any(|&limb| limb != 0) + { + self.sticky = true; + } + self.len = keep; + } + + fn is_zero(&self) -> bool { + self.len == 0 + } + + /// The position of the leading digit: `p` for the digit worth `10^p`. + /// The expansion is nonzero. + fn leading(&self) -> i64 { + let first = self.limbs[self.start]; + let digits = POW10 + .iter() + .position(|&power| first < power) + .unwrap_or(POW10.len()); + LIMB_DIGITS * i64::from(self.top) + digits as i64 - 1 + } + + /// The digit at `position`. + #[cfg(test)] + fn digit(&self, position: i64) -> u32 { + let back = i64::from(self.top) - position.div_euclid(LIMB_DIGITS); + if !(0..self.len as i64).contains(&back) { + return 0; + } + let limb = self.limbs[self.start + back as usize]; + limb / POW10[position.rem_euclid(LIMB_DIGITS) as usize] % 10 + } + + /// Keep the digits at `position` and above, rounded to nearest, ties to + /// even, on everything below. + fn round(&mut self, position: i64) { + let below_everything = self.is_zero() || position - 1 > self.leading(); + let limb = position.div_euclid(LIMB_DIGITS); + let lowest = i64::from(self.top) - self.len as i64 + 1; + if below_everything { + // Under a tenth of the last kept digit's unit: zero. An empty + // expansion here had every digit folded below its window, which + // lies under the digit after `position`. + self.len = 0; + self.sticky = false; + return; + } + if limb < lowest { + // Every stored digit is kept, and what lies below them is under a + // tenth of the last kept digit's unit (the window reaches below + // `position`'s limb whenever anything was folded). + self.sticky = false; + return; + } + if limb > i64::from(self.top) { + // `position - 1` is the top digit of the leading limb, so the + // kept digit is the first of the limb above: materialize it. + self.start -= 1; + self.len += 1; + self.top += 1; + self.limbs[self.start] = 0; + } + let at = self.start + (i64::from(self.top) - limb) as usize; + let end = self.start + self.len; + let k = position.rem_euclid(LIMB_DIGITS) as usize; + let (round_digit, rest_nonzero) = if k > 0 { + let under = self.limbs[at] % POW10[k]; + ( + under / POW10[k - 1], + under % POW10[k - 1] != 0 || self.limbs[at + 1..end].iter().any(|&l| l != 0), + ) + } else if at + 1 < end { + let next = self.limbs[at + 1]; + let top_digit = POW10[LIMB_DIGITS as usize - 1]; + ( + next / top_digit, + next % top_digit != 0 || self.limbs[at + 2..end].iter().any(|&l| l != 0), + ) + } else { + (0, false) + }; + let rest_nonzero = rest_nonzero || self.sticky; + self.limbs[at] -= self.limbs[at] % POW10[k]; + self.len = at + 1 - self.start; + self.sticky = false; + let odd = self.limbs[at] / POW10[k] % 2 == 1; + if round_digit > 5 || (round_digit == 5 && (rest_nonzero || odd)) { + let mut carry_at = at; + self.limbs[carry_at] += POW10[k]; + while self.limbs[carry_at] >= BASE { + self.limbs[carry_at] -= BASE; + if carry_at == self.start { + self.start -= 1; + self.len += 1; + self.top += 1; + self.limbs[self.start] = 0; + } + carry_at -= 1; + self.limbs[carry_at] += 1; + } + } + while self.len > 0 && self.limbs[self.start] == 0 { + self.start += 1; + self.len -= 1; + self.top -= 1; + } + } + + /// The position of the lowest nonzero stored digit. + fn lowest(&self) -> Option { + let stored = &self.limbs[self.start..self.start + self.len]; + let back = stored.iter().rposition(|&limb| limb != 0)?; + let limb = stored[back]; + let zeros = POW10[1..] + .iter() + .take_while(|&&power| limb % power == 0) + .count(); + Some(LIMB_DIGITS * (i64::from(self.top) - back as i64) + zeros as i64) + } + + /// Write the digits at positions `from` down to `to`, inclusive. + fn write(&self, out: &mut Out<'_, S>, from: i64, to: i64) { + if from < to { + return; + } + let mut position = from; + let (high, low) = if self.is_zero() { + (i64::MIN, i64::MAX) + } else { + ( + LIMB_DIGITS * i64::from(self.top) + LIMB_DIGITS - 1, + LIMB_DIGITS * (i64::from(self.top) - self.len as i64 + 1), + ) + }; + if position > high { + let zeros = position - high.max(to - 1); + out.pad(&ZEROS, zeros as usize); + position -= zeros; + } + while position >= to && position >= low { + let limb = position.div_euclid(LIMB_DIGITS); + let at = self.start + (i64::from(self.top) - limb) as usize; + let mut text = [b'0'; LIMB_DIGITS as usize]; + let mut value = self.limbs[at]; + for byte in text.iter_mut().rev() { + *byte = b'0' + (value % 10) as u8; + value /= 10; + } + let hi = position.rem_euclid(LIMB_DIGITS) as usize; + let lo = if to.div_euclid(LIMB_DIGITS) == limb { + to.rem_euclid(LIMB_DIGITS) as usize + } else { + 0 + }; + let last = LIMB_DIGITS as usize - 1; + out.put(&text[last - hi..=last - lo]); + position -= (hi - lo + 1) as i64; + } + if position >= to { + out.pad(&ZEROS, (position - to + 1) as usize); + } + } +} + +/// Write the digits of `digits` (all zeros for a zero) from `from` down to +/// `to`. +fn write_digits( + out: &mut Out<'_, S>, + digits: Option<&Decimal>, + from: i64, + to: i64, +) { + match digits { + Some(digits) => digits.write(out, from, to), + None if from >= to => out.pad(&ZEROS, (from - to + 1) as usize), + None => {} + } +} + +/// The lowest base-10⁹ limb a rounding at `position` must read exactly: the +/// one holding the digit after it, and one more. Nothing below the +/// expansion's last digit is ever needed: a fraction of `-exponent` binary +/// places has exactly that many decimal ones. +fn window(position: i64, exponent: i32) -> i64 { + let cut = (position - 1).div_euclid(LIMB_DIGITS) - 1; + cut.max(i64::from(exponent).div_euclid(LIMB_DIGITS) - 1) +} + +/// log₁₀2 = 0.30102999566398…, truncated to eleven places: below the truth, +/// by less than 10⁻¹¹. +const LOG10_2_NUMERATOR: i64 = 30_102_999_566; +const LOG10_2_DENOMINATOR: i64 = 100_000_000_000; + +/// A position at or below that of `mantissa × 2^exponent`'s leading digit. +/// The value is at least `2^power` for `power = bits − 1 + exponent`, whose +/// leading digit sits at `⌊power·log₁₀2⌋`; the truncated logarithm moves +/// that floor by at most one over every exponent a `long double` has, and +/// one digit of slack absorbs it. +fn leading_floor(mantissa: u128, exponent: i32) -> i64 { + let power = i64::from(u128::BITS - mantissa.leading_zeros()) - 1 + i64::from(exponent); + (power * LOG10_2_NUMERATOR).div_euclid(LOG10_2_DENOMINATOR) - 1 +} + +/// `spec`'s precision for `e`, `f` and `g`. +fn precision(spec: Spec) -> i64 { + spec.precision + .map_or(DEFAULT_PRECISION, |precision| precision as i64) +} + +/// Format `value` by `conversion` (`e E f F g G a A`) under `spec`, its +/// expansion held in at most `N` limbs. Never inlined into the engine: a +/// `long double`'s limbs are 7 KiB of stack, which only its conversions pay. +#[inline(never)] +pub(super) fn convert( + out: &mut Out<'_, S>, + spec: Spec, + conversion: u8, + value: Float, +) { + let upper = conversion.is_ascii_uppercase(); + let sign: &[u8] = if value.negative { + b"-" + } else if spec.plus { + b"+" + } else if spec.space { + b" " + } else { + b"" + }; + let (mantissa, exponent) = match value.class { + Class::Infinite => return special(out, spec, sign, if upper { b"INF" } else { b"inf" }), + Class::Nan => return special(out, spec, sign, if upper { b"NAN" } else { b"nan" }), + Class::Finite { mantissa, exponent } => (mantissa, exponent), + }; + match conversion.to_ascii_lowercase() { + b'a' => hexadecimal(out, spec, sign, upper, mantissa, exponent), + b'f' => fixed::(out, spec, sign, mantissa, exponent), + b'e' => scientific::(out, spec, sign, upper, mantissa, exponent), + _ => general::(out, spec, sign, upper, mantissa, exponent), + } +} + +/// Lay out a number of `body` bytes after `sign` and `prefix` in `spec`'s +/// field: spaces before it, or after it under `-`, or zeros between the +/// prefix and the body under `0` (which `-` overrides). +fn field( + out: &mut Out<'_, S>, + spec: Spec, + sign: &[u8], + prefix: &[u8], + body: usize, + write: impl FnOnce(&mut Out<'_, S>), +) { + let fill = spec.width.saturating_sub(sign.len() + prefix.len() + body); + let zeros = spec.zero && !spec.left; + if !spec.left && !zeros { + out.pad(&SPACES, fill); + } + out.put(sign); + out.put(prefix); + if zeros { + out.pad(&ZEROS, fill); + } + write(out); + if spec.left { + out.pad(&SPACES, fill); + } +} + +/// An infinity or a NaN: a word, which the `0` flag never pads. +fn special(out: &mut Out<'_, S>, spec: Spec, sign: &[u8], word: &[u8; 3]) { + let spec = Spec { + zero: false, + ..spec + }; + field(out, spec, sign, b"", word.len(), |out| out.put(word)); +} + +/// `%f`: the digits down to the precision's last. +fn fixed( + out: &mut Out<'_, S>, + spec: Spec, + sign: &[u8], + mantissa: u128, + exponent: i32, +) { + let precision = precision(spec); + let digits = (mantissa != 0).then(|| { + let mut digits = Decimal::::new(mantissa, exponent, window(-precision, exponent)); + digits.round(-precision); + digits + }); + fixed_layout(out, spec, sign, digits.as_ref(), precision); +} + +/// `%e`: one digit, the precision's after the point, and the exponent. +fn scientific( + out: &mut Out<'_, S>, + spec: Spec, + sign: &[u8], + upper: bool, + mantissa: u128, + exponent: i32, +) { + let precision = precision(spec); + let (digits, leading) = significant::(mantissa, exponent, precision); + exponent_layout(out, spec, sign, upper, digits.as_ref(), leading, precision); +} + +/// `%g`: `%e`'s rounding to the precision's significant digits, laid out as +/// `%f` when the exponent is at least −4 and below the precision, without +/// trailing fraction zeros unless `#` keeps them (C11 7.21.6.1p8). +fn general( + out: &mut Out<'_, S>, + spec: Spec, + sign: &[u8], + upper: bool, + mantissa: u128, + exponent: i32, +) { + let significant_digits = match spec.precision { + Some(0) => 1, + Some(precision) => precision as i64, + None => DEFAULT_PRECISION, + }; + let (digits, leading) = significant::(mantissa, exponent, significant_digits - 1); + let lowest = digits.as_ref().and_then(Decimal::lowest); + if significant_digits > leading && leading >= -4 { + let precision = significant_digits - 1 - leading; + let fraction = if spec.alternate { + precision + } else { + lowest.map_or(0, |low| (-low).clamp(0, precision)) + }; + fixed_layout(out, spec, sign, digits.as_ref(), fraction); + } else { + let precision = significant_digits - 1; + let fraction = if spec.alternate { + precision + } else { + lowest.map_or(0, |low| (leading - low).clamp(0, precision)) + }; + exponent_layout(out, spec, sign, upper, digits.as_ref(), leading, fraction); + } +} + +/// The expansion rounded to `precision` digits after its leading one, and +/// that leading digit's position (`0` for a zero). The window reaches from a +/// bound below the leading digit, so the expansion is never empty. +fn significant( + mantissa: u128, + exponent: i32, + precision: i64, +) -> (Option>, i64) { + if mantissa == 0 { + return (None, 0); + } + let floor = leading_floor(mantissa, exponent); + let mut digits = Decimal::::new(mantissa, exponent, window(floor - precision, exponent)); + let leading = digits.leading(); + digits.round(leading - precision); + // A carry can open a new leading digit: 9.96 at one place is 1.0e1. + let leading = digits.leading(); + (Some(digits), leading) +} + +/// `[-]d…d[.d…d]`: the integer digits (one `0` when there are none), then the +/// point and `fraction` digits when there are any or `#` asks for the point. +fn fixed_layout( + out: &mut Out<'_, S>, + spec: Spec, + sign: &[u8], + digits: Option<&Decimal>, + fraction: i64, +) { + let leading = digits + .filter(|digits| !digits.is_zero()) + .map(Decimal::leading); + let whole = leading.map_or(1, |leading| leading.max(0) + 1); + let point = fraction > 0 || spec.alternate; + let body = whole as usize + usize::from(point) + fraction as usize; + field(out, spec, sign, b"", body, |out| { + write_digits(out, digits, whole - 1, 0); + if point { + out.put(b"."); + } + write_digits(out, digits, -1, -fraction); + }); +} + +/// `[-]d[.d…d]e±dd`: the leading digit, then the point and `fraction` digits +/// when there are any or `#` asks for the point, then the exponent in at +/// least two digits. +fn exponent_layout( + out: &mut Out<'_, S>, + spec: Spec, + sign: &[u8], + upper: bool, + digits: Option<&Decimal>, + leading: i64, + fraction: i64, +) { + let point = fraction > 0 || spec.alternate; + let mut text = [0u8; EXPONENT_TEXT]; + let power = exponent_text(&mut text, if upper { b'E' } else { b'e' }, leading, 2); + let body = 1 + usize::from(point) + fraction as usize + power.len(); + field(out, spec, sign, b"", body, |out| { + write_digits(out, digits, leading, leading); + if point { + out.put(b"."); + } + write_digits(out, digits, leading - 1, leading - fraction); + out.put(power); + }); +} + +/// The letter, the sign and up to six digits: a binary exponent of a +/// `long double` subnormal (−16494) is the widest. +const EXPONENT_TEXT: usize = 8; + +/// `letter`, a sign, and `value`'s decimal digits, at least `min_digits`. +fn exponent_text( + text: &mut [u8; EXPONENT_TEXT], + letter: u8, + value: i64, + min_digits: usize, +) -> &[u8] { + let mut digits = [b'0'; EXPONENT_TEXT - 2]; + let mut at = digits.len(); + let mut rest = value.unsigned_abs(); + while rest != 0 || digits.len() - at < min_digits { + at -= 1; + digits[at] = b'0' + (rest % 10) as u8; + rest /= 10; + } + let count = digits.len() - at; + text[0] = letter; + text[1] = if value < 0 { b'-' } else { b'+' }; + text[2..2 + count].copy_from_slice(&digits[at..]); + &text[..2 + count] +} + +/// `%a`: `0x1.h…hp±d`, the value normalized to a leading `1` and every +/// fraction digit it has — or, with a precision, that many, rounded to +/// nearest-even. +fn hexadecimal( + out: &mut Out<'_, S>, + spec: Spec, + sign: &[u8], + upper: bool, + mantissa: u128, + exponent: i32, +) { + let alphabet: &[u8; 16] = if upper { + b"0123456789ABCDEF" + } else { + b"0123456789abcdef" + }; + let prefix: &[u8] = if upper { b"0X" } else { b"0x" }; + let (mut lead, mut fraction, mut count, power) = if mantissa == 0 { + (0u8, 0u128, 0usize, 0i64) + } else { + // The bits below the leading one, padded out to whole hex digits, + // without the trailing zero digits a precision did not ask for. + let tail = u128::BITS - 1 - mantissa.leading_zeros(); + let digits = tail.div_ceil(4); + let mut fraction = (mantissa ^ (1 << tail)) << (4 * digits - tail); + let mut count = digits as usize; + while count > 0 && fraction & 0xf == 0 { + fraction >>= 4; + count -= 1; + } + (1u8, fraction, count, i64::from(exponent) + i64::from(tail)) + }; + let mut zeros = 0usize; + if let Some(precision) = spec.precision { + if precision < count { + let dropped = 4 * (count - precision) as u32; + let half = 1u128 << (dropped - 1); + let rest = fraction & ((1u128 << dropped) - 1); + fraction >>= dropped; + let odd = if precision == 0 { + lead & 1 == 1 + } else { + fraction & 1 == 1 + }; + if rest > half || (rest == half && odd) { + fraction += 1; + if precision == 0 || fraction >> (4 * precision) != 0 { + lead += 1; + fraction = 0; + } + } + count = precision; + } else { + zeros = precision - count; + } + } + let point = count + zeros > 0 || spec.alternate; + let mut text = [0u8; EXPONENT_TEXT]; + let power = exponent_text(&mut text, if upper { b'P' } else { b'p' }, power, 1); + let body = 1 + usize::from(point) + count + zeros + power.len(); + field(out, spec, sign, prefix, body, |out| { + out.put(&[alphabet[usize::from(lead)]]); + if point { + out.put(b"."); + } + let mut digits = [0u8; HEX_FRACTION_DIGITS]; + for (i, digit) in digits[..count].iter_mut().enumerate() { + *digit = alphabet[((fraction >> (4 * (count - 1 - i))) & 0xf) as usize]; + } + out.put(&digits[..count]); + out.pad(&ZEROS, zeros); + out.put(power); + }); +} + +/// Hex digits below the leading one any mantissa can have: a `u128`'s. +const HEX_FRACTION_DIGITS: usize = (u128::BITS as usize - 1).div_ceil(4); + +#[cfg(test)] +mod tests; diff --git a/native/crates/c-shim/src/format/float/tests.rs b/native/crates/c-shim/src/format/float/tests.rs new file mode 100644 index 0000000..db777af --- /dev/null +++ b/native/crates/c-shim/src/format/float/tests.rs @@ -0,0 +1,620 @@ +//! The float conversions against the host C library's own. glibc's +//! `snprintf` is the reference for `double`, through every flag, width and +//! precision. For the two `long double` formats it is glibc's `strfroml` +//! (x87, on an x86_64 host) and `strfromf128` (binary128), reached through +//! trampolines: Rust has no x87 type to pass, nor a binary128 one that +//! crosses into C. Where musl and glibc disagree — `%a` of a subnormal, of an +//! x87 value, and x87's invalid encodings — the cases are musl's spellings, +//! written out. + +extern crate std; + +use core::ffi::{c_char, c_int}; +use std::ffi::CString; +use std::string::String; +use std::vec::Vec; + +use super::super::{Args, Length, Sink, format}; +use super::{DOUBLE_LIMBS, Decimal, Float, LONG_DOUBLE_LIMBS, window}; + +/// The one float argument a test format converts. +enum Value { + Double(f64), + Long(Float), +} + +struct One(Option); + +impl Args for One { + fn signed(&mut self, _: Length) -> i64 { + unreachable!("a float test formats one float") + } + fn unsigned(&mut self, _: Length) -> u64 { + unreachable!("a float test formats one float") + } + fn int(&mut self) -> c_int { + unreachable!("a float test formats one float") + } + fn pointer(&mut self) -> usize { + unreachable!("a float test formats one float") + } + fn double(&mut self) -> f64 { + match self.0.take() { + Some(Value::Double(value)) => value, + _ => panic!("the format read a double it was not given"), + } + } + fn long_double(&mut self) -> Float { + match self.0.take() { + Some(Value::Long(value)) => value, + _ => panic!("the format read a long double it was not given"), + } + } +} + +struct Collect(Vec); + +impl Sink for Collect { + fn put(&mut self, bytes: &[u8]) { + self.0.extend_from_slice(bytes); + } +} + +fn ours(spec: &str, value: Value) -> String { + let spec = CString::new(spec).unwrap(); + let mut args = One(Some(value)); + let mut sink = Collect(Vec::new()); + // SAFETY: NUL-terminated, and the one conversion matches the argument. + let count = unsafe { format(&mut sink, spec.as_ptr().cast(), &mut args) }; + assert!(args.0.is_none(), "the format did not read its argument"); + assert_eq!(count, sink.0.len(), "the count is the output's length"); + String::from_utf8(sink.0).unwrap() +} + +/// Room for the widest `%f` of a `long double` with a long precision. +const HOST_BUFFER: usize = 1 << 14; + +unsafe extern "C" { + fn snprintf(buffer: *mut c_char, size: usize, format: *const c_char, ...) -> c_int; +} + +fn host_double(spec: &str, value: f64) -> String { + let spec = CString::new(spec).unwrap(); + let mut buffer = std::vec![0u8; HOST_BUFFER]; + // SAFETY: a buffer of its stated size and a matching double. + let written = unsafe { + snprintf( + buffer.as_mut_ptr().cast(), + buffer.len(), + spec.as_ptr(), + value, + ) + }; + let written = usize::try_from(written).unwrap(); + assert!(written < buffer.len()); + String::from_utf8(buffer[..written].to_vec()).unwrap() +} + +/// xorshift64*: fixed seeds, so a failure reproduces. +struct Rng(u64); + +impl Rng { + fn next(&mut self) -> u64 { + self.0 ^= self.0 >> 12; + self.0 ^= self.0 << 25; + self.0 ^= self.0 >> 27; + self.0.wrapping_mul(0x2545_f491_4f6c_dd1d) + } +} + +/// Values the conversions turn on: zeros and ones, exact ties at every +/// place, the powers of ten either side of a rounding carry, the subnormal +/// and normal edges, the largest finite value, and the non-finite ones. +fn edge_doubles() -> Vec { + let mut values = std::vec![ + 0.0, + -0.0, + 1.0, + -1.0, + 0.5, + 1.5, + 2.5, + 3.5, + -2.5, + 0.125, + 0.375, + 0.625, + 0.05, + 0.15, + 0.25, + 0.35, + 9.5, + 10.5, + 1e-7, + 1e-5, + 1e-4, + 0.000099999995, + 9.9999995, + 99999.95, + 999999.5, + 123456.0, + 1234567.0, + 1e15, + 1e16, + 1e17, + 1e21, + 1e22, + 1e23, + 1e100, + 9007199254740992.0, + 9007199254740994.0, + 1.0 / 3.0, + 2.0 / 3.0, + 0.1, + 0.2, + 0.3, + f64::EPSILON, + f64::MIN_POSITIVE, + f64::from_bits(1), + f64::from_bits(3), + f64::from_bits(0x000f_ffff_ffff_ffff), + f64::MAX, + -f64::MAX, + f64::INFINITY, + f64::NEG_INFINITY, + f64::NAN, + -f64::NAN, + ]; + for exponent in -30..30 { + values.push(10f64.powi(exponent)); + values.push(5.0 * 10f64.powi(exponent)); + } + values +} + +/// Random bit patterns (every exponent, so mostly huge and tiny values) +/// and random values of everyday size. +fn random_doubles(count: usize) -> Vec { + let mut rng = Rng(0x9e37_79b9_7f4a_7c15); + let mut values = Vec::with_capacity(2 * count); + for _ in 0..count { + values.push(f64::from_bits(rng.next())); + let everyday = (rng.next() & 0x800f_ffff_ffff_ffff) | ((1023 - 40 + rng.next() % 80) << 52); + values.push(f64::from_bits(everyday)); + } + values +} + +const DOUBLE_SPECS: &[&str] = &[ + "%e", + "%.0e", + "%.1e", + "%.3e", + "%.16e", + "%.17e", + "%.40e", + "%E", + "%#.0e", + "%+e", + "% .2e", + "[%15.3e]", + "[%-15.3e]", + "%015.3e", + "%f", + "%.0f", + "%.1f", + "%.3f", + "%.20f", + "%.1100f", + "%F", + "%#.0f", + "%+08.2f", + "[%-12.4f]", + "%g", + "%.0g", + "%.1g", + "%.2g", + "%.10g", + "%.17g", + "%.40g", + "%G", + "%+g", + "%012g", + "[%-12g]", + "%a", + "%.0a", + "%.1a", + "%.3a", + "%.13a", + "%.20a", + "%A", + "%#.0a", + "%+a", + "%020a", + "[%-20a]", +]; + +/// `%a` is one conversion musl and glibc spell differently for a `double`: +/// glibc keeps a subnormal's leading `0`. +fn hex_differs(spec: &str, value: f64) -> bool { + spec.contains(['a', 'A']) && value != 0.0 && value.abs() < f64::MIN_POSITIVE +} + +/// `%#g` is the other: when the rounding carries the exponent up to the +/// precision, glibc drops the trailing zeros `#` keeps (musl keeps them, as +/// C 7.21.6.1p8 says). The reference is built from glibc's `%e` and `%#f` or +/// `%#e`, as that paragraph defines `%g`. +const ALTERNATE_GENERAL: &[(&str, Option)] = + &[("%#g", None), ("%#.3g", Some(3)), ("%#.0g", Some(0))]; + +fn host_alternate_general(precision: Option, value: f64) -> String { + if !value.is_finite() { + return host_double("%#g", value); + } + let significant = match precision { + Some(0) => 1, + Some(precision) => precision, + None => 6, + }; + let scientific = host_double(&std::format!("%.{}e", significant - 1), value); + let exponent: i64 = scientific.rsplit_once('e').unwrap().1.parse().unwrap(); + if significant > exponent && exponent >= -4 { + host_double(&std::format!("%#.{}f", significant - 1 - exponent), value) + } else { + host_double(&std::format!("%#.{}e", significant - 1), value) + } +} + +#[test] +fn doubles_print_as_the_host_prints_them() { + let mut values = edge_doubles(); + values.extend(random_doubles(1500)); + for &value in &values { + for &spec in DOUBLE_SPECS { + if hex_differs(spec, value) { + continue; + } + assert_eq!( + ours(spec, Value::Double(value)), + host_double(spec, value), + "{spec} of {value:e} ({:#018x})", + value.to_bits() + ); + } + for &(spec, precision) in ALTERNATE_GENERAL { + assert_eq!( + ours(spec, Value::Double(value)), + host_alternate_general(precision, value), + "{spec} of {value:e} ({:#018x})", + value.to_bits() + ); + } + } +} + +#[test] +fn an_alternate_general_keeps_its_zeros_through_a_carry_as_musl_does() { + let cases: &[(&str, f64, &str)] = &[ + ("%#g", 999999.5, "1.00000e+06"), + ("%#.3g", 999.5, "1.00e+03"), + ("%#.2g", 99.5, "1.0e+02"), + ("%#.1g", 9.5, "1.e+01"), + ("%#.0g", 9.5, "1.e+01"), + ("%#.6g", 0.0009999995, "0.00100000"), + ]; + for &(spec, value, expected) in cases { + assert_eq!( + ours(spec, Value::Double(value)), + expected, + "{spec} of {value:e}" + ); + } +} + +#[test] +fn a_subnormal_double_in_hex_is_normalized_as_musl_spells_it() { + let cases: &[(&str, u64, &str)] = &[ + ("%a", 1, "0x1p-1074"), + ("%a", 3, "0x1.8p-1073"), + ("%.0a", 3, "0x2p-1073"), + ("%a", 0x000f_ffff_ffff_ffff, "0x1.ffffffffffffep-1023"), + ("%.1A", 0x0008_0000_0000_0001, "0X1.0P-1023"), + ]; + for &(spec, bits, expected) in cases { + assert_eq!( + ours(spec, Value::Double(f64::from_bits(bits))), + expected, + "{spec} of {bits:#x}" + ); + } +} + +#[test] +fn every_format_holds_its_widest_expansion() { + // The widest integer part, built whole by doubling. + let max = Decimal::::new((1 << 53) - 1, 971, 0); + assert_eq!(max.leading(), 308); + // The longest fraction a window can reach, whatever the precision. + let least = Decimal::::new(1, -1074, window(-1_000_000, -1074)); + assert_eq!(least.leading(), -324); + assert_eq!( + (-339..=-324) + .rev() + .map(|p| least.digit(p)) + .collect::>(), + [4, 9, 4, 0, 6, 5, 6, 4, 5, 8, 4, 1, 2, 4, 6, 5] + ); + assert!(!least.sticky, "nothing of the expansion was let go"); + assert_eq!(least.lowest(), Some(-1074)); + let quad_max = Decimal::::new((1 << 113) - 1, 16271, 0); + assert_eq!(quad_max.leading(), 4932); + let quad_least = Decimal::::new(1, -16494, window(-1_000_000, -16494)); + assert_eq!(quad_least.leading(), -4966); + assert_eq!(quad_least.lowest(), Some(-16494)); + let x87_least = Decimal::::new(1, -16445, window(-1_000_000, -16445)); + assert_eq!(x87_least.lowest(), Some(-16445)); +} + +#[test] +fn a_huge_precision_prints_its_zeros_without_storing_them() { + let text = ours("%.100000f", Value::Double(0.5)); + assert_eq!(text.len(), 2 + 100_000); + assert!(text.starts_with("0.5000")); + assert!(text[3..].bytes().all(|b| b == b'0')); +} + +/// The `(precision, conversion)` pairs `strfrom*` accepts, which name no +/// flag or width: those are the engine's own, which the `double` sweep has +/// covered. +const WIDE_SPECS: &[(Option, char)] = &[ + (None, 'e'), + (Some(0), 'e'), + (Some(1), 'e'), + (Some(5), 'e'), + (Some(18), 'e'), + (Some(25), 'e'), + (Some(36), 'e'), + (Some(60), 'E'), + (None, 'f'), + (Some(0), 'f'), + (Some(3), 'f'), + (Some(30), 'f'), + (Some(60), 'F'), + (None, 'g'), + (Some(0), 'g'), + (Some(1), 'g'), + (Some(5), 'g'), + (Some(18), 'g'), + (Some(21), 'g'), + (Some(36), 'G'), + (Some(50), 'g'), +]; + +fn wide_spec(precision: Option, conversion: char, long: bool) -> String { + let length = if long { "L" } else { "" }; + match precision { + Some(precision) => std::format!("%.{precision}{length}{conversion}"), + None => std::format!("%{length}{conversion}"), + } +} + +/// Random `long double` encodings of `format`: every exponent, every +/// everyday one, and the subnormals; each a valid encoding. +fn random_wide(rng: &mut Rng, x87: bool) -> u128 { + let sign = u128::from(rng.next() >> 63) << if x87 { 79 } else { 127 }; + let roll = rng.next() % 8; + let biased: u128 = match roll { + 0 => 0, + 1..=3 => u128::from(16383 - 200 + rng.next() % 400), + _ => u128::from(1 + rng.next() % 0x7ffd), + }; + let fraction = (u128::from(rng.next()) << 64) | u128::from(rng.next()); + if x87 { + let integer = u128::from(biased != 0) << 63; + sign | biased << 64 | integer | (fraction & ((1 << 63) - 1)) + } else { + sign | biased << 112 | (fraction & ((1 << 112) - 1)) + } +} + +#[cfg(target_arch = "x86_64")] +mod x86_64_host { + use super::*; + + unsafe extern "C" { + fn strfroml(); + fn strfromf128(); + } + + /// glibc's `strfroml(buffer, size, format, value)`: the x87 value rides + /// the stack, as System V passes every `long double`. + fn host_x87(spec: &str, bits: u128) -> String { + let spec = CString::new(spec).unwrap(); + let mut buffer = std::vec![0u8; HOST_BUFFER]; + let value = bits.to_le_bytes(); + let written: i32; + // SAFETY: the stack stays 16-aligned for the call (`asm!` enters + // aligned and the frame is sixteen bytes), the arguments are live, + // and every caller-saved register is declared clobbered. + unsafe { + core::arch::asm!( + "sub rsp, 16", + "mov rax, qword ptr [r10]", + "mov qword ptr [rsp], rax", + "mov rax, qword ptr [r10 + 8]", + "mov qword ptr [rsp + 8], rax", + "call {f}", + "add rsp, 16", + f = sym strfroml, + in("r10") value.as_ptr(), + in("rdi") buffer.as_mut_ptr(), + in("rsi") buffer.len(), + in("rdx") spec.as_ptr(), + lateout("eax") written, + clobber_abi("C"), + ); + } + let written = usize::try_from(written).unwrap(); + assert!(written < buffer.len()); + String::from_utf8(buffer[..written].to_vec()).unwrap() + } + + /// glibc's `strfromf128(buffer, size, format, value)`: `_Float128` + /// rides `xmm0`. + fn host_quad(spec: &str, bits: u128) -> String { + let spec = CString::new(spec).unwrap(); + let mut buffer = std::vec![0u8; HOST_BUFFER]; + let value = bits.to_le_bytes(); + let written: i32; + // SAFETY: as `host_x87`; the value is loaded into `xmm0` from live + // bytes, and `xmm0` is caller-saved. + unsafe { + core::arch::asm!( + "movdqu xmm0, xmmword ptr [r10]", + "call {f}", + f = sym strfromf128, + in("r10") value.as_ptr(), + in("rdi") buffer.as_mut_ptr(), + in("rsi") buffer.len(), + in("rdx") spec.as_ptr(), + lateout("eax") written, + clobber_abi("C"), + ); + } + let written = usize::try_from(written).unwrap(); + assert!(written < buffer.len()); + String::from_utf8(buffer[..written].to_vec()).unwrap() + } + + fn edge_wide(x87: bool) -> Vec { + if x87 { + std::vec![ + 0, + 1 << 79, + 0x3fff_8000_0000_0000_0000, + 0x3ffb_cccc_cccc_cccc_cccd, + 0x4000_a000_0000_0000_0000, + 0x7ffe_ffff_ffff_ffff_ffff, + 0x0001_8000_0000_0000_0000, + 0x0000_0000_0000_0000_0001, + 0x0000_7fff_ffff_ffff_ffff, + 0x7fff_8000_0000_0000_0000, + 0xffff_8000_0000_0000_0000, + 0x7fff_c000_0000_0000_0000, + ] + } else { + std::vec![ + 0, + 1 << 127, + 0x3fff_0000_0000_0000_0000_0000_0000_0000, + 0x3ffb_9999_9999_9999_9999_9999_9999_999a, + 0x7ffe_ffff_ffff_ffff_ffff_ffff_ffff_ffff, + 0x0001_0000_0000_0000_0000_0000_0000_0000, + 0x0000_0000_0000_0000_0000_0000_0000_0001, + 0x7fff_0000_0000_0000_0000_0000_0000_0000, + 0xffff_0000_0000_0000_0000_0000_0000_0000, + 0x7fff_8000_0000_0000_0000_0000_0000_0000, + ] + } + } + + fn sweep(x87: bool) { + let mut rng = Rng(if x87 { + 0x0123_4567_89ab_cdef + } else { + 0xfedc_ba98_7654_3210 + }); + let mut values = edge_wide(x87); + values.extend((0..400).map(|_| random_wide(&mut rng, x87))); + for &bits in &values { + let value = if x87 { + Float::from_x87(bits) + } else { + Float::from_quad(bits) + }; + for &(precision, conversion) in WIDE_SPECS { + let host = if x87 { + host_x87(&wide_spec(precision, conversion, false), bits) + } else { + host_quad(&wide_spec(precision, conversion, false), bits) + }; + let spec = wide_spec(precision, conversion, true); + assert_eq!( + ours(&spec, Value::Long(value)), + host, + "{spec} of {bits:#034x}" + ); + } + // glibc and musl agree on `%a` of a normal binary128 value only. + let subnormal = (bits >> 112) & 0x7fff == 0 && bits << 1 != 0; + if !x87 && !subnormal { + for spec in ["%a", "%.0a", "%.5a", "%.28a", "%.40A"] { + let long = spec.replace('a', "La").replace('A', "LA"); + assert_eq!( + ours(&long, Value::Long(value)), + host_quad(spec, bits), + "{long} of {bits:#034x}" + ); + } + } + } + } + + #[test] + fn x87_long_doubles_print_as_the_host_prints_them() { + sweep(true); + } + + #[test] + fn binary128_long_doubles_print_as_the_host_prints_them() { + sweep(false); + } +} + +#[test] +fn an_x87_value_in_hex_is_normalized_as_musl_spells_it() { + let cases: &[(u128, &str)] = &[ + (0x3fff_8000_0000_0000_0000, "0x1p+0"), + (0x3ffb_cccc_cccc_cccc_cccd, "0x1.999999999999999ap-4"), + (0x7ffe_ffff_ffff_ffff_ffff, "0x1.fffffffffffffffep+16383"), + (0x0000_0000_0000_0000_0001, "0x1p-16445"), + (0xbfff_c000_0000_0000_0000, "-0x1.8p+0"), + ]; + for &(bits, expected) in cases { + assert_eq!( + ours("%La", Value::Long(Float::from_x87(bits))), + expected, + "{bits:#x}" + ); + } +} + +#[test] +fn x87_encodings_no_fpu_produces_are_read_as_musl_reads_them() { + let spell = |bits: u128| ours("%Lg", Value::Long(Float::from_x87(bits))); + // An unnormal: a clear integer bit under a nonzero exponent. + assert_eq!(spell(0x3fff_4000_0000_0000_0000), "nan"); + // A pseudo-NaN and a pseudo-infinity: the top exponent, integer bit clear. + assert_eq!(spell(0x7fff_4000_0000_0000_0000), "nan"); + assert_eq!(spell(0xffff_0000_0000_0000_0000), "-inf"); + // A pseudo-denormal is the value its bits spell: the least normal. + let least_normal = ours( + "%.30Le", + Value::Long(Float::from_x87(0x0001_8000_0000_0000_0000)), + ); + assert_eq!( + ours( + "%.30Le", + Value::Long(Float::from_x87(0x0000_8000_0000_0000_0000)) + ), + least_normal + ); + // The padding above the 80 bits is not the value's. + assert_eq!( + ours( + "%Lg", + Value::Long(Float::from_x87(0xdead_0000_3fff_8000_0000_0000_0000)) + ), + "1" + ); +} diff --git a/native/crates/c-shim/src/format_symbols.rs b/native/crates/c-shim/src/format_symbols.rs new file mode 100644 index 0000000..da703e9 --- /dev/null +++ b/native/crates/c-shim/src/format_symbols.rs @@ -0,0 +1,342 @@ +//! The `` writes: the `printf` family over [`crate::format`], and +//! the character and block writes over [`crate::stdio`]'s streams. +//! +//! `snprintf` returns what ISO says — the length the complete output would +//! have had — writing at most `n - 1` bytes and a NUL. A count past `INT_MAX` +//! returns `-1`, C's `EOVERFLOW` case. The stream writes return `EOF` when +//! their stream's write fails, which on `stdout`/`stderr` means no +//! [`StandardSink`](crate::stdio::StandardSink) was installed. + +use core::ffi::{VaList, c_char, c_int, c_long, c_longlong, c_uint, c_ulong, c_ulonglong, c_void}; + +use crate::format::{Args, Length, Sink, format}; +use crate::stdio::{self, File, stdout}; + +const EOF: c_int = -1; + +/// A C `va_list` as the engine's argument source. +struct VaArgs<'a, 'f>(&'a mut VaList<'f>); + +impl Args for VaArgs<'_, '_> { + fn signed(&mut self, length: Length) -> i64 { + // SAFETY (each `next_arg`): the format's conversion names this C + // type, the variadic caller's half of the contract. + unsafe { + match length { + // `char` and `short` arrive promoted to `int`. + Length::Char => i64::from(self.0.next_arg::() as i8), + Length::Short => i64::from(self.0.next_arg::() as i16), + Length::Int => i64::from(self.0.next_arg::()), + Length::Long => self.0.next_arg::(), + // glibc reads `%Ld` as `long long`; so does this. + Length::LongLong | Length::Max | Length::LongDouble => { + self.0.next_arg::() + } + Length::Size | Length::Diff => self.0.next_arg::() as i64, + } + } + } + + fn unsigned(&mut self, length: Length) -> u64 { + // SAFETY: as `signed`. + unsafe { + match length { + Length::Char => u64::from(self.0.next_arg::() as u8), + Length::Short => u64::from(self.0.next_arg::() as u16), + Length::Int => u64::from(self.0.next_arg::()), + Length::Long => self.0.next_arg::(), + Length::LongLong | Length::Max | Length::LongDouble => { + self.0.next_arg::() + } + Length::Size | Length::Diff => self.0.next_arg::() as u64, + } + } + } + + fn int(&mut self) -> c_int { + // SAFETY: as `signed`. + unsafe { self.0.next_arg::() } + } + + fn pointer(&mut self) -> usize { + // SAFETY: as `signed`. + unsafe { self.0.next_arg::<*const c_void>() as usize } + } + + fn double(&mut self) -> f64 { + // SAFETY: as `signed`. + unsafe { self.0.next_arg::() } + } + + fn long_double(&mut self) -> crate::format::Float { + // SAFETY: as `signed`; the conversion's `L` names a `long double`. + unsafe { crate::va_long_double::next(self.0) } + } +} + +/// A C buffer that keeps counting past its end, as `snprintf` must. +struct Bounded { + target: *mut u8, + /// Bytes that fit before the terminator. + room: usize, + stored: usize, +} + +impl Sink for Bounded { + fn put(&mut self, bytes: &[u8]) { + let take = bytes.len().min(self.room - self.stored); + // SAFETY: `stored + take <= room`, inside the caller's buffer. + unsafe { + core::ptr::copy_nonoverlapping(bytes.as_ptr(), self.target.add(self.stored), take) + }; + self.stored += take; + } +} + +/// `vsnprintf` with an explicit room: `None` is `vsprintf`'s unbounded +/// buffer, whose size the C caller vouches for. +/// +/// # Safety +/// C's `vsnprintf` contract. +unsafe fn bounded( + target: *mut c_char, + len: Option, + format_: *const c_char, + args: &mut VaList, +) -> c_int { + let room = match len { + Some(0) => 0, + Some(len) => len - 1, + None => usize::MAX, + }; + let mut out = Bounded { + target: target.cast(), + room, + stored: 0, + }; + // SAFETY: the caller's contract. + let count = unsafe { format(&mut out, format_.cast(), &mut VaArgs(args)) }; + if len != Some(0) { + // SAFETY: `stored <= room`, which left one byte for the NUL. + unsafe { *out.target.add(out.stored) = 0 }; + } + c_int::try_from(count).unwrap_or(-1) +} + +#[unsafe(no_mangle)] +unsafe extern "C" fn vsnprintf( + target: *mut c_char, + len: usize, + format: *const c_char, + mut args: VaList, +) -> c_int { + // SAFETY: the caller's contract is vsnprintf's. + unsafe { bounded(target, Some(len), format, &mut args) } +} + +#[unsafe(no_mangle)] +unsafe extern "C" fn snprintf( + target: *mut c_char, + len: usize, + format: *const c_char, + args: ... +) -> c_int { + // SAFETY: the caller's contract is vsnprintf's. + unsafe { vsnprintf(target, len, format, args) } +} + +#[unsafe(no_mangle)] +unsafe extern "C" fn vsprintf( + target: *mut c_char, + format: *const c_char, + mut args: VaList, +) -> c_int { + // SAFETY: the caller vouches that the buffer holds the output. + unsafe { bounded(target, None, format, &mut args) } +} + +#[unsafe(no_mangle)] +unsafe extern "C" fn sprintf(target: *mut c_char, format: *const c_char, args: ...) -> c_int { + // SAFETY: as `vsprintf`. + unsafe { vsprintf(target, format, args) } +} + +/// A stream write, batched: the engine's runs collect here and reach the +/// stream a buffer at a time. +struct Streamed { + file: *mut File, + buffer: [u8; 256], + held: usize, + failed: bool, +} + +impl Streamed { + fn drain(&mut self) { + // SAFETY: the stream is live for the whole call. + if self.held > 0 && !unsafe { stdio::write(self.file, &self.buffer[..self.held]) } { + self.failed = true; + } + self.held = 0; + } +} + +impl Sink for Streamed { + fn put(&mut self, bytes: &[u8]) { + if self.held + bytes.len() > self.buffer.len() { + self.drain(); + } + if bytes.len() > self.buffer.len() { + // A run bigger than the buffer goes straight through. + // SAFETY: as `drain`. + if !unsafe { stdio::write(self.file, bytes) } { + self.failed = true; + } + return; + } + self.buffer[self.held..self.held + bytes.len()].copy_from_slice(bytes); + self.held += bytes.len(); + } +} + +#[unsafe(no_mangle)] +unsafe extern "C" fn vfprintf(file: *mut File, format_: *const c_char, mut args: VaList) -> c_int { + // One call, one lock: every batch reaches the stream before another + // thread's call does. + // SAFETY: C's vfprintf contract: a live stream and a matching format. + unsafe { + stdio::locked(file, || { + let mut out = Streamed { + file, + buffer: [0; 256], + held: 0, + failed: false, + }; + let count = format(&mut out, format_.cast(), &mut VaArgs(&mut args)); + out.drain(); + if out.failed { + return EOF; + } + c_int::try_from(count).unwrap_or(-1) + }) + } +} + +#[unsafe(no_mangle)] +unsafe extern "C" fn fprintf(file: *mut File, format: *const c_char, args: ...) -> c_int { + // SAFETY: as `vfprintf`. + unsafe { vfprintf(file, format, args) } +} + +#[unsafe(no_mangle)] +unsafe extern "C" fn vprintf(format: *const c_char, args: VaList) -> c_int { + // SAFETY: as `vfprintf`, on `stdout`. + unsafe { vfprintf(stdout.file(), format, args) } +} + +#[unsafe(no_mangle)] +unsafe extern "C" fn printf(format: *const c_char, args: ...) -> c_int { + // SAFETY: as `vfprintf`, on `stdout`. + unsafe { vfprintf(stdout.file(), format, args) } +} + +#[unsafe(no_mangle)] +unsafe extern "C" fn fputc(byte: c_int, file: *mut File) -> c_int { + let byte = byte as u8; + // SAFETY: C's contract: a live stream. + if unsafe { stdio::write(file, &[byte]) } { + c_int::from(byte) + } else { + EOF + } +} + +#[unsafe(no_mangle)] +unsafe extern "C" fn putc(byte: c_int, file: *mut File) -> c_int { + // SAFETY: as `fputc`. + unsafe { fputc(byte, file) } +} + +#[unsafe(no_mangle)] +unsafe extern "C" fn putchar(byte: c_int) -> c_int { + // SAFETY: `stdout` is live. + unsafe { fputc(byte, stdout.file()) } +} + +/// The bytes of a C string, without its NUL. +/// +/// # Safety +/// `text` is NUL-terminated. +unsafe fn c_str<'a>(text: *const c_char) -> &'a [u8] { + // SAFETY: the caller's contract. + unsafe { core::ffi::CStr::from_ptr(text) }.to_bytes() +} + +#[unsafe(no_mangle)] +unsafe extern "C" fn fputs(text: *const c_char, file: *mut File) -> c_int { + // SAFETY: C's contract: a NUL-terminated string and a live stream. + if unsafe { stdio::write(file, c_str(text)) } { + 0 + } else { + EOF + } +} + +#[unsafe(no_mangle)] +unsafe extern "C" fn puts(text: *const c_char) -> c_int { + let file = stdout.file(); + // The line and its newline under one lock, as one call. + // SAFETY: as `fputs`, on `stdout`. + let written = unsafe { + stdio::locked(file, || { + stdio::write(file, c_str(text)) && stdio::write(file, b"\n") + }) + }; + if written { 0 } else { EOF } +} + +#[unsafe(no_mangle)] +unsafe extern "C" fn fwrite( + source: *const c_void, + size: usize, + count: usize, + file: *mut File, +) -> usize { + let Some(total) = size.checked_mul(count) else { + return 0; + }; + if total == 0 { + return 0; + } + // SAFETY: C promises `size * count` readable bytes and a live stream. + let bytes = unsafe { core::slice::from_raw_parts(source.cast::(), total) }; + if unsafe { stdio::write(file, bytes) } { + count + } else { + 0 + } +} + +#[unsafe(no_mangle)] +unsafe extern "C" fn fflush(file: *mut File) -> c_int { + if file.is_null() { + // "Every output stream": the standard pair holds nothing back, and a + // consumer's streams flush on their own close. + return 0; + } + // SAFETY: C's contract: a live stream. + if unsafe { stdio::flush(file) } { + 0 + } else { + EOF + } +} + +#[unsafe(no_mangle)] +unsafe extern "C" fn fclose(file: *mut File) -> c_int { + // SAFETY: C's contract: a live stream, not used again. + if unsafe { stdio::close(file) } { + 0 + } else { + EOF + } +} diff --git a/native/crates/c-shim/src/lib.rs b/native/crates/c-shim/src/lib.rs new file mode 100644 index 0000000..68bb647 --- /dev/null +++ b/native/crates/c-shim/src/lib.rs @@ -0,0 +1,83 @@ +//! The C symbols a vendored C library imports on a kore user target. +//! +//! A C library built by `tools/kore-cc` is freestanding: Zig's musl triple +//! supplies ISO C headers and nothing links musl's runtime. What such a +//! library still calls is defined here, over the task's own heap and Rust's +//! Unicode tables, so each consumer does not grow its own copy: +//! +//! - allocation (`malloc`, `calloc`, `realloc`, `free`) and `abort`, each +//! behind its own feature for a C runtime that exports its own; +//! - the `` routines beyond `compiler_builtins`' `memcpy`, +//! `memmove`, `memset`, `memcmp` and `strlen` (its `mem` feature, which +//! every kore user build enables); +//! - with `wide-classes` (a default), the wide character classes; +//! - the `` writes: the `printf` family over one conversion engine +//! ([`format`]), every float conversion exact for `double` and both +//! `long double` formats, whose `va_arg` each ABI's own rule reads +//! (`va_long_double`); and `stdout`/`stderr` plus consumer-built streams +//! ([`stdio`]); +//! - the `` parsers (`strtol` … `strtod`), `qsort`, `strerror`, +//! and a per-thread `errno`; +//! - with `long-double`, the byte-level halves of `strtold` and `ldexpl` +//! ([`long_double`]) a consumer's C layer wraps, since only C can spell +//! the type. +//! +//! The set began as tree-sitter's (its `wasm-stdlib/imports.txt`) and grew +//! with TCC, the second C consumer (`res/TCC.md` §2.2), and with the C +//! programs libkore.so runs, the third: their `printf` is this crate's. +//! +//! Linking this crate is what supplies the symbols: a consumer names it +//! (`use kore_c_shim as _;`) so the linker keeps it. On the host the crate's +//! symbols are absent — the host's libc serves host tests, and a second +//! `malloc` there would replace the process allocator — while the pure +//! engines behind them ([`format`], [`number`], [`sort`], [`long_double`]) +//! are compiled for the host suite, so it tests the code the target runs. +//! +//! The string routines and wide character classes are pure, so a shared +//! library may carry its own (`libkore_grammars.so` takes the classes alone). Everything that touches the process — the +//! standard streams, `errno` (and with it the parsers that set it), and with +//! the default `malloc` and `abort` features the heap's C names and `abort` — +//! sits behind the default `runtime` feature, and a shared library built +//! without it imports those from the executable it links into +//! (`libkore_grammars.so` does), so the heap stays the task's one heap. A C +//! runtime that is itself the process's C library (libkore.so) takes +//! `runtime` alone and exports its own allocator and `abort`. +#![no_std] +#![cfg_attr(all(target_os = "none", feature = "runtime"), feature(c_variadic))] + +#[cfg(test)] +extern crate std; + +#[cfg(any(test, all(target_os = "none", feature = "runtime")))] +pub mod format; +#[cfg(any(test, all(target_os = "none", feature = "long-double")))] +pub mod long_double; +#[cfg(any(test, all(target_os = "none", feature = "runtime")))] +pub mod number; +#[cfg(any(test, all(target_os = "none", feature = "runtime")))] +pub mod sort; + +#[cfg(all(target_os = "none", feature = "malloc"))] +mod alloc_symbols; +#[cfg(all(target_os = "none", feature = "runtime"))] +pub mod errno; +#[cfg(all(target_os = "none", feature = "runtime"))] +mod format_symbols; +#[cfg(all(target_os = "none", feature = "runtime"))] +pub mod stdio; +#[cfg(all(target_os = "none", feature = "runtime"))] +mod stdlib_symbols; +#[cfg(target_os = "none")] +mod string_symbols; +#[cfg(all(target_os = "none", feature = "runtime"))] +mod va_long_double; +#[cfg(all(target_os = "none", feature = "wide-classes"))] +mod wide_symbols; + +/// The C `abort`: the library hit a state it cannot leave. The task ends +/// with the aborted status, which its controller reports. +#[cfg(all(target_os = "none", feature = "abort"))] +#[unsafe(no_mangle)] +extern "C" fn abort() -> ! { + kore_rt::thread::exit(kore_rt::ExitCode::ABORTED) +} diff --git a/native/crates/c-shim/src/long_double.rs b/native/crates/c-shim/src/long_double.rs new file mode 100644 index 0000000..70601b1 --- /dev/null +++ b/native/crates/c-shim/src/long_double.rs @@ -0,0 +1,341 @@ +//! `long double`'s two conversions, `strtold` and `ldexpl`, as bytes. +//! +//! The type is the C compiler's, never Rust's: x87 double-extended (a 64-bit +//! significand with an explicit integer bit) on x86_64, IEEE binary128 on +//! aarch64 and riscv64. Rust has no type for the first and no ABI promise +//! that its `f128` crosses `extern "C"` as C's `long double` for the second, +//! so the consumer's C layer owns the type: it passes its value's bytes and +//! `LDBL_MANT_DIG`, and this side computes on them. Nothing here names an +//! architecture — the format is the C side's word, checked. +//! +//! The arithmetic is LLVM's APFloat ([`rustc_apfloat`]), the code clang +//! folds `long double` literals with: decimal conversion is correctly +//! rounded for any length of input, and scaling rounds once, subnormals and +//! overflow included. The subject grammar is [`crate::number`]'s, so +//! `strtold` reads exactly the sequence `strtod` does. + +use rustc_apfloat::ieee::{Quad, X87DoubleExtended}; +use rustc_apfloat::{Float, Round, Status}; + +use crate::number::{FloatSubject, float_subject}; + +/// `sizeof(long double)` on every kore C ABI: x87's ten bytes padded to +/// sixteen, or binary128's sixteen. The C side asserts its own type agrees. +pub const BYTES: usize = 16; + +/// A `long double` format, by the `LDBL_MANT_DIG` that names it. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Format { + /// x87 double-extended (`LDBL_MANT_DIG == 64`). + X87, + /// IEEE binary128 (`LDBL_MANT_DIG == 113`). + Quad, +} + +impl Format { + /// The format `LDBL_MANT_DIG` names, or `None` for one kore has no ABI + /// with. + pub const fn from_mant_dig(mant_dig: i32) -> Option { + match mant_dig { + 64 => Some(Self::X87), + 113 => Some(Self::Quad), + _ => None, + } + } +} + +/// A `long double` value as its little-endian bytes: the significant bits +/// low (x87's ten, binary128's sixteen), any padding zero. +pub const fn to_bytes(bits: u128) -> [u8; BYTES] { + bits.to_le_bytes() +} + +/// The bits of a `long double`'s bytes. x87's six padding bytes are +/// whatever the C side's storage held, so they are cleared here. +pub const fn from_bytes(format: Format, bytes: [u8; BYTES]) -> u128 { + let bits = u128::from_le_bytes(bytes); + match format { + Format::X87 => bits & ((1u128 << 80) - 1), + Format::Quad => bits, + } +} + +/// What a `strtold` parse found. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Parsed { + /// The value, as [`to_bytes`] lays it out. + pub bits: u128, + /// Overflow to infinity, or a nonzero number rounded to zero: `ERANGE`. + pub range: bool, + /// Bytes consumed; zero when there was no number. + pub end: usize, +} + +/// Parse a `long double` from the start of `text`. +pub fn parse(format: Format, text: &[u8]) -> Parsed { + match format { + Format::X87 => parse_as::(text), + Format::Quad => parse_as::(text), + } +} + +fn parse_as(text: &[u8]) -> Parsed { + let (value, range, end) = match float_subject(text) { + FloatSubject::None => (F::ZERO, false, 0), + FloatSubject::Special { nan, negative, end } => { + let value = if nan { F::NAN } else { F::INFINITY }; + (if negative { -value } else { value }, false, end) + } + FloatSubject::Decimal { + negative, + digits, + nonzero, + end, + } => { + // The subject is ASCII digits, a `.` and an exponent — always + // UTF-8, and always inside APFloat's decimal grammar (a subject + // has at least one digit, and an exponent only with digits). + let converted = core::str::from_utf8(digits) + .ok() + .and_then(|digits| F::from_str_r(digits, Round::NearestTiesToEven).ok()); + let Some(converted) = converted else { + // Unreachable by the grammar; refuse the conversion rather + // than invent a value. + return Parsed { + bits: F::ZERO.to_bits(), + range: false, + end: 0, + }; + }; + let value = if negative { + -converted.value + } else { + converted.value + }; + let range = + converted.status.intersects(Status::OVERFLOW) || (value.is_zero() && nonzero); + (value, range, end) + } + }; + Parsed { + bits: value.to_bits(), + range, + end, + } +} + +/// `ldexpl`: `bits × 2^exponent`, rounded once to nearest-even. +pub fn scale(format: Format, bits: u128, exponent: i32) -> u128 { + match format { + Format::X87 => scale_as::(bits, exponent), + Format::Quad => scale_as::(bits, exponent), + } +} + +fn scale_as(bits: u128, exponent: i32) -> u128 { + F::from_bits(bits) + .scalbn_r(exponent, Round::NearestTiesToEven) + .to_bits() +} + +#[cfg(test)] +mod tests { + use super::*; + use std::ffi::CString; + use std::vec::Vec; + + // The host's own C library is the reference: on an x86_64 host its + // `long double` is x87 and glibc's `strtold` rounds correctly, and + // `strtof128` is binary128. + unsafe extern "C" { + fn strtold(text: *const core::ffi::c_char, end: *mut *mut core::ffi::c_char) -> X87Bytes; + fn ldexpl(value: X87Bytes, exponent: i32) -> X87Bytes; + } + + /// `long double` as the host ABI moves it: Rust has no x87 type, so + /// these tests read the host's answer through its bytes. + #[repr(C, align(16))] + #[derive(Clone, Copy)] + struct X87Bytes([u8; 16]); + + fn host(text: &str) -> (u128, usize) { + let c = CString::new(text).unwrap(); + let mut end = core::ptr::null_mut(); + // SAFETY: a NUL-terminated string and a valid end pointer. + let value = unsafe { host_strtold(c.as_ptr(), &mut end) }; + let consumed = end as usize - c.as_ptr() as usize; + (from_bytes(Format::X87, value), consumed) + } + + /// glibc's `strtold` returns in `st(0)`; this trampoline stores it. + /// The destination rides `r12`, callee-saved, so the call keeps it. + unsafe fn host_strtold( + text: *const core::ffi::c_char, + end: *mut *mut core::ffi::c_char, + ) -> [u8; 16] { + let mut out = [0u8; 16]; + // SAFETY: the caller's pointers; `fstp` stores the ten bytes and + // leaves the x87 stack empty, as the ABI requires. + unsafe { + core::arch::asm!( + "call {f}", + "fstp tbyte ptr [r12]", + f = sym strtold, + in("r12") out.as_mut_ptr(), + in("rdi") text, + in("rsi") end, + clobber_abi("C"), + ); + } + out + } + + fn ours(text: &str) -> (u128, usize) { + let parsed = parse(Format::X87, text.as_bytes()); + (parsed.bits, parsed.end) + } + + #[test] + #[cfg(target_arch = "x86_64")] + fn x87_parses_as_the_host_does() { + let mut corpus: Vec = [ + "0", + "1", + "0.1", + "1e4932", + "1e4933", + "1e-4951", + "3.6e-4951", + "3.7e-4951", + "1.18973149535723176502e4932", + "1.18973149535723176508e4932", + "3.36210314311209350626e-4932", + "6.95e-4950", + "0.30000000000000000001", + "123456789012345678901234567890", + " -2.5e3x", + ".5", + "5.", + "1e+", + "inf", + "-INFINITY", + "nan", + "18446744073709551615", + "18446744073709551616.5", + "9.999999999999999999999e999", + "2.2250738585072011e-308", + "1.00000000000000000005421010862427522170037264004349708557128906250", + "1.0000000000000000000542101086242752217003726400434970855712890625", + "1.00000000000000000016263032587282567", + "4.9406564584124654e-324", + ] + .iter() + .map(|s| std::string::String::from(*s)) + .collect(); + // A deterministic spread of digit strings and exponents. + let mut seed = 0x9e37_79b9_7f4a_7c15u64; + for _ in 0..4000 { + seed ^= seed << 13; + seed ^= seed >> 7; + seed ^= seed << 17; + let digits = 1 + (seed % 40) as usize; + let mut s = std::string::String::new(); + let mut x = seed; + for i in 0..digits { + if i == digits / 2 && seed & 1 == 0 { + s.push('.'); + } + s.push((b'0' + (x % 10) as u8) as char); + x = x / 10 ^ x.rotate_left(29); + } + let exp = ((seed >> 20) % 9900) as i64 - 4950; + s.push_str(&std::format!("e{exp}")); + corpus.push(s); + } + for text in &corpus { + assert_eq!(ours(text), host(text), "{text}"); + } + } + + #[test] + #[cfg(target_arch = "x86_64")] + fn x87_scales_as_the_host_does() { + let values = [ + "1", + "0.1", + "3.0e-4950", + "1.18973149535723176502e4932", + "-7.25", + "inf", + "0", + ]; + for value in values { + let (bits, _) = host(value); + for exponent in [ + -20000, -16445, -16444, -16383, -100, -1, 0, 1, 100, 16383, 16384, 20000, + ] { + let x = X87Bytes(to_bytes(bits)); + // SAFETY: glibc's `ldexpl` on an x87 value passed in memory. + let expected = unsafe { host_ldexpl(x, exponent) }; + assert_eq!( + scale(Format::X87, bits, exponent), + from_bytes(Format::X87, expected), + "{value} {exponent}" + ); + } + } + } + + /// `ldexpl` takes its `long double` on the stack and returns in `st(0)`. + unsafe fn host_ldexpl(value: X87Bytes, exponent: i32) -> [u8; 16] { + let mut out = [0u8; 16]; + // SAFETY: SysV passes an x87 argument in memory, in order, aligned + // 16 (an asm block without `nostack` enters call-aligned); the + // destination rides callee-saved `r12` across the call. + unsafe { + core::arch::asm!( + "sub rsp, 16", + "movdqu xmm0, [{value}]", + "movdqu [rsp], xmm0", + "call {f}", + "add rsp, 16", + "fstp tbyte ptr [r12]", + f = sym ldexpl, + value = in(reg) value.0.as_ptr(), + in("r12") out.as_mut_ptr(), + in("edi") exponent, + clobber_abi("C"), + ); + } + out + } + + #[test] + fn quad_parses_round_numbers_exactly() { + // binary128 of 1.0: biased exponent 0x3fff, zero fraction. + assert_eq!(parse(Format::Quad, b"1").bits, 0x3fff_u128 << 112); + assert_eq!(parse(Format::Quad, b"-2").bits, (0xc000_u128) << 112); + // 0.1 is the nearest binary128, rounded up in its last bit. + assert_eq!( + parse(Format::Quad, b"0.1").bits, + 0x3ffb_9999_9999_9999_9999_9999_9999_999a + ); + let huge = parse(Format::Quad, b"1e5000"); + assert!(huge.range && huge.bits == 0x7fff_u128 << 112); + let tiny = parse(Format::Quad, b"1e-5000"); + assert!(tiny.range && tiny.bits == 0); + assert_eq!( + scale(Format::Quad, parse(Format::Quad, b"1").bits, 3), + parse(Format::Quad, b"8").bits + ); + } + + #[test] + fn formats_are_named_by_mant_dig() { + assert_eq!(Format::from_mant_dig(64), Some(Format::X87)); + assert_eq!(Format::from_mant_dig(113), Some(Format::Quad)); + assert_eq!(Format::from_mant_dig(53), None); + // x87's padding never leaks into the value. + assert_eq!(from_bytes(Format::X87, [0xff; 16]), (1u128 << 80) - 1); + } +} diff --git a/native/crates/c-shim/src/number.rs b/native/crates/c-shim/src/number.rs new file mode 100644 index 0000000..85e3f8f --- /dev/null +++ b/native/crates/c-shim/src/number.rs @@ -0,0 +1,474 @@ +//! The parsing behind `strtol`, `strtoul`, `strtod` and their siblings: ISO C +//! 7.22.1's grammar over the bytes before a string's NUL, pure so the host +//! suite drives the code the target runs. +//! +//! Integers follow the standard exactly: leading C-locale whitespace, an +//! optional sign, base `0` choosing 16 for `0x`, 8 for a leading `0` and 10 +//! otherwise, overflow reported rather than wrapped, and an end offset that +//! is zero when no digit was found. +//! +//! Decimal floats are rounded correctly — the digits go to core's parser, +//! which rounds any length of input exactly — and `inf`, `infinity` and +//! `nan[(chars)]` are recognized in any case. **Hexadecimal floats are not +//! parsed**: `0x1p3` reads as `0` and ends at the `x`, which is C89's +//! reading. No consumer needs more; TCC, the one that compiles C, parses +//! hexadecimal float literals itself. +//! +//! The subject grammar is public ([`float_subject`]) so a parser for a type +//! core has no parser for — TCC's `long double` — reads exactly the sequence +//! `strtod` reads. + +/// ISO C locale whitespace (`isspace`): space, `\t \n \v \f \r`. +pub const fn c_space(byte: u8) -> bool { + matches!(byte, b' ' | b'\t' | b'\n' | 0x0b | 0x0c | b'\r') +} + +/// What an integer parse found. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Integer { + /// The value's magnitude, saturated at `u64::MAX` when `overflow`. + pub magnitude: u64, + /// A `-` preceded the digits. + pub negative: bool, + /// The magnitude did not fit 64 bits. + pub overflow: bool, + /// Bytes consumed through the last digit; zero when there was no number. + pub end: usize, +} + +/// Parse an integer in `base` (0 or 2..=36) from the start of `text`. +/// `None` for any other base — C's `EINVAL`, with no conversion. +pub fn integer(text: &[u8], base: u32) -> Option { + if base == 1 || base > 36 { + return None; + } + let mut at = 0; + while text.get(at).copied().is_some_and(c_space) { + at += 1; + } + let negative = text.get(at) == Some(&b'-'); + if matches!(text.get(at), Some(b'+' | b'-')) { + at += 1; + } + let digit = |byte: u8, base: u32| char::from(byte).to_digit(base); + let hex_prefix = matches!(text.get(at..at + 2), Some([b'0', b'x' | b'X'])) + && text + .get(at + 2) + .is_some_and(|&byte| digit(byte, 16).is_some()); + let base = match base { + 0 if hex_prefix => 16, + 0 if text.get(at) == Some(&b'0') => 8, + 0 => 10, + base => base, + }; + // `0x` is skipped only when a hexadecimal digit follows it; otherwise + // the `0` is the number and parsing ends at the `x`. + if base == 16 && hex_prefix { + at += 2; + } + let first = at; + let mut magnitude = 0u64; + let mut overflow = false; + while let Some(value) = text.get(at).and_then(|&byte| digit(byte, base)) { + match magnitude + .checked_mul(u64::from(base)) + .and_then(|scaled| scaled.checked_add(u64::from(value))) + { + Some(next) => magnitude = next, + None => { + overflow = true; + magnitude = u64::MAX; + } + } + at += 1; + } + let end = if at == first { 0 } else { at }; + Some(Integer { + magnitude, + negative, + overflow, + end, + }) +} + +impl Integer { + /// The value as a signed integer no wider than `bits`, and whether it + /// was clamped (C's `ERANGE`). + pub fn signed(self, bits: u32) -> (i64, bool) { + let max = (1u64 << (bits - 1)) - 1; + if self.negative { + if self.overflow || self.magnitude > max + 1 { + (-(max as i64) - 1, true) + } else { + (0i64.wrapping_sub(self.magnitude as i64), false) + } + } else if self.overflow || self.magnitude > max { + (max as i64, true) + } else { + (self.magnitude as i64, false) + } + } + + /// The value as an unsigned integer no wider than `bits`, and whether it + /// was clamped. A `-` negates in the unsigned type, as C says. + pub fn unsigned(self, bits: u32) -> (u64, bool) { + let max = if bits == 64 { + u64::MAX + } else { + (1u64 << bits) - 1 + }; + if self.overflow || self.magnitude > max { + (max, true) + } else if self.negative { + (self.magnitude.wrapping_neg() & max, false) + } else { + (self.magnitude, false) + } + } +} + +/// What a float parse found. +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct Float { + pub value: T, + /// Overflow to infinity, or a nonzero number rounded to zero: `ERANGE`. + pub range: bool, + /// Bytes consumed; zero when there was no number. + pub end: usize, +} + +/// The floating subject sequence at the start of `text` (ISO C 7.22.1.3), +/// leading whitespace skipped: what `strtod` and its siblings convert. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum FloatSubject<'t> { + /// No subject: nothing is converted, and the end pointer is the input. + None, + /// `inf`, `infinity` or `nan[(n-char-sequence)]`, in any case. + Special { + nan: bool, + negative: bool, + end: usize, + }, + /// A decimal number. `digits` is the sign-free subject — digits, an + /// optional `.`, an optional exponent — in the grammar a + /// correctly-rounding parser takes; `nonzero` says a mantissa digit was + /// not `0`, so a zero result is an underflow rather than a zero. + Decimal { + negative: bool, + digits: &'t [u8], + nonzero: bool, + end: usize, + }, +} + +/// Scan the floating subject at the start of `text`. +pub fn float_subject(text: &[u8]) -> FloatSubject<'_> { + if let Some((nan, negative, end)) = special(text) { + return FloatSubject::Special { nan, negative, end }; + } + match decimal_subject(text) { + Some((negative, digits, nonzero, end)) => FloatSubject::Decimal { + negative, + digits, + nonzero, + end, + }, + None => FloatSubject::None, + } +} + +/// The decimal subject sequence at the start of `text`: its sign, the slice +/// core's parser takes, whether any mantissa digit was nonzero, and the end. +fn decimal_subject(text: &[u8]) -> Option<(bool, &[u8], bool, usize)> { + let mut at = 0; + while text.get(at).copied().is_some_and(c_space) { + at += 1; + } + let negative = text.get(at) == Some(&b'-'); + if matches!(text.get(at), Some(b'+' | b'-')) { + at += 1; + } + let start = at; + let mut digits = 0usize; + let mut nonzero = false; + while let Some(&byte) = text.get(at).filter(|byte| byte.is_ascii_digit()) { + nonzero |= byte != b'0'; + digits += 1; + at += 1; + } + if text.get(at) == Some(&b'.') { + at += 1; + while let Some(&byte) = text.get(at).filter(|byte| byte.is_ascii_digit()) { + nonzero |= byte != b'0'; + digits += 1; + at += 1; + } + } + if digits == 0 { + return None; + } + // An exponent counts only when at least one digit follows it. + if matches!(text.get(at), Some(b'e' | b'E')) { + let mut exponent = at + 1; + if matches!(text.get(exponent), Some(b'+' | b'-')) { + exponent += 1; + } + if text.get(exponent).is_some_and(u8::is_ascii_digit) { + at = exponent; + while text.get(at).is_some_and(u8::is_ascii_digit) { + at += 1; + } + } + } + Some((negative, &text[start..at], nonzero, at)) +} + +/// `inf`, `infinity` or `nan[(n-char-sequence)]`, case-insensitively, at the +/// start of `text`: `(is_nan, negative, end)`. +fn special(text: &[u8]) -> Option<(bool, bool, usize)> { + let mut at = 0; + while text.get(at).copied().is_some_and(c_space) { + at += 1; + } + let negative = text.get(at) == Some(&b'-'); + if matches!(text.get(at), Some(b'+' | b'-')) { + at += 1; + } + let word = |at: usize, word: &[u8]| { + text.get(at..at + word.len()) + .is_some_and(|got| got.eq_ignore_ascii_case(word)) + }; + if word(at, b"infinity") { + return Some((false, negative, at + 8)); + } + if word(at, b"inf") { + return Some((false, negative, at + 3)); + } + if word(at, b"nan") { + let mut end = at + 3; + // `nan(` followed by alphanumerics and `_`, then `)`. + if text.get(end) == Some(&b'(') { + let mut close = end + 1; + while text + .get(close) + .is_some_and(|&byte| byte.is_ascii_alphanumeric() || byte == b'_') + { + close += 1; + } + if text.get(close) == Some(&b')') { + end = close + 1; + } + } + return Some((true, negative, end)); + } + None +} + +macro_rules! float_parser { + ($name:ident, $ty:ty) => { + /// Parse a float from the start of `text`. + pub fn $name(text: &[u8]) -> Float<$ty> { + let (negative, subject, nonzero, end) = match float_subject(text) { + FloatSubject::Special { nan, negative, end } => { + let value = if nan { <$ty>::NAN } else { <$ty>::INFINITY }; + return Float { + value: if negative { -value } else { value }, + range: false, + end, + }; + } + FloatSubject::None => { + return Float { + value: 0.0, + range: false, + end: 0, + }; + } + FloatSubject::Decimal { + negative, + digits, + nonzero, + end, + } => (negative, digits, nonzero, end), + }; + // The subject is ASCII digits, `.` and an exponent: always UTF-8, + // and always in core's grammar (a lone `.` was refused above). + let value: $ty = core::str::from_utf8(subject) + .ok() + .and_then(|digits| digits.parse().ok()) + .unwrap_or(0.0); + let range = value.is_infinite() || (value == 0.0 && nonzero); + Float { + value: if negative { -value } else { value }, + range, + end, + } + } + }; +} + +float_parser!(double, f64); +float_parser!(single, f32); + +#[cfg(test)] +mod tests { + use super::*; + + fn int(text: &str, base: u32) -> Integer { + integer(text.as_bytes(), base).expect("a valid base") + } + + #[test] + fn integers_follow_the_c_grammar() { + assert_eq!( + int(" 42abc", 10), + Integer { + magnitude: 42, + negative: false, + overflow: false, + end: 4 + } + ); + assert_eq!(int("-17", 10).signed(64), (-17, false)); + assert_eq!(int("+9", 0).signed(64), (9, false)); + assert_eq!(int("0x1F", 0).magnitude, 31); + assert_eq!(int("0x1F", 16).magnitude, 31); + assert_eq!(int("017", 0).magnitude, 15); + assert_eq!(int("z", 36).magnitude, 35); + // `0x` without a hex digit after it: the `0` is the number. + assert_eq!( + int("0xg", 0), + Integer { + magnitude: 0, + negative: false, + overflow: false, + end: 1 + } + ); + assert_eq!(int("0x", 16).end, 1); + // No digits: no conversion, end zero. + assert_eq!(int(" -", 10).end, 0); + assert_eq!(int("", 10).end, 0); + assert!(integer(b"1", 1).is_none()); + assert!(integer(b"1", 37).is_none()); + } + + #[test] + fn integer_overflow_clamps_and_reports() { + assert_eq!(int("9223372036854775807", 10).signed(64), (i64::MAX, false)); + assert_eq!(int("9223372036854775808", 10).signed(64), (i64::MAX, true)); + assert_eq!( + int("-9223372036854775808", 10).signed(64), + (i64::MIN, false) + ); + assert_eq!(int("-9223372036854775809", 10).signed(64), (i64::MIN, true)); + assert_eq!( + int("99999999999999999999999", 10).signed(64), + (i64::MAX, true) + ); + assert_eq!(int("2147483648", 10).signed(32), (i32::MAX as i64, true)); + assert_eq!( + int("18446744073709551615", 10).unsigned(64), + (u64::MAX, false) + ); + assert_eq!( + int("18446744073709551616", 10).unsigned(64), + (u64::MAX, true) + ); + // A minus sign negates in the unsigned type. + assert_eq!(int("-1", 10).unsigned(64), (u64::MAX, false)); + assert_eq!(int("-1", 10).unsigned(32), (u32::MAX as u64, false)); + // Overflowed digits are still consumed. + assert_eq!(int("99999999999999999999999x", 10).end, 23); + } + + #[test] + fn decimal_floats_round_correctly() { + assert_eq!( + double(b"1.5"), + Float { + value: 1.5, + range: false, + end: 3 + } + ); + assert_eq!(double(b" -2.5e3x").value, -2500.0); + assert_eq!(double(b" -2.5e3x").end, 8); + assert_eq!(double(b".5").value, 0.5); + assert_eq!(double(b"5.").value, 5.0); + assert_eq!(double(b"5.").end, 2); + // An exponent with no digits is not part of the number. + assert_eq!(double(b"1e+").end, 1); + assert_eq!(double(b"0.1").value, 0.1); + assert_eq!(single(b"0.1").value, 0.1f32); + assert_eq!(double(b"").end, 0); + assert_eq!(double(b".").end, 0); + assert_eq!(double(b"-").end, 0); + } + + #[test] + fn float_range_and_specials() { + let huge = double(b"1e400"); + assert!(huge.value.is_infinite() && huge.range); + let tiny = double(b"1e-400"); + assert!(tiny.value == 0.0 && tiny.range); + let zero = double(b"0.000"); + assert!(zero.value == 0.0 && !zero.range); + assert!(single(b"1e39").range); + assert_eq!(double(b"INF").value, f64::INFINITY); + assert_eq!(double(b"-Infinity").value, f64::NEG_INFINITY); + assert_eq!(double(b"-Infinity").end, 9); + assert!(double(b"nan(0x1)").value.is_nan()); + assert_eq!(double(b"nan(abc)").end, 8); + assert_eq!(double(b"nan(").end, 3); + // Hexadecimal floats read as C89 does: the `0`, ending at the `x`. + assert_eq!( + double(b"0x1p3"), + Float { + value: 0.0, + range: false, + end: 1 + } + ); + } + + #[test] + fn the_subject_is_the_parsers_own() { + assert_eq!( + float_subject(b" -1.25e3x"), + FloatSubject::Decimal { + negative: true, + digits: b"1.25e3", + nonzero: true, + end: 8 + } + ); + assert_eq!( + float_subject(b"0.000"), + FloatSubject::Decimal { + negative: false, + digits: b"0.000", + nonzero: false, + end: 5 + } + ); + assert_eq!( + float_subject(b"-INF"), + FloatSubject::Special { + nan: false, + negative: true, + end: 4 + } + ); + assert_eq!( + float_subject(b"nan(7)"), + FloatSubject::Special { + nan: true, + negative: false, + end: 6 + } + ); + assert_eq!(float_subject(b" ."), FloatSubject::None); + assert_eq!(float_subject(b"+"), FloatSubject::None); + } +} diff --git a/native/crates/c-shim/src/sort.rs b/native/crates/c-shim/src/sort.rs new file mode 100644 index 0000000..7b16c60 --- /dev/null +++ b/native/crates/c-shim/src/sort.rs @@ -0,0 +1,115 @@ +//! `qsort`'s algorithm: heapsort over raw elements of a C-supplied size. +//! +//! Heapsort because it is O(n log n) in the worst case, iterative (no +//! recursion to exhaust a 64 KiB worker stack), and in place. C's `qsort` +//! promises no stability, and this keeps none. + +/// Sort `count` elements of `size` bytes at `base` by `before(a, b)`, which +/// answers whether `a` sorts strictly after `b` (a C comparator's `> 0`). +/// +/// # Safety +/// `base` points to `count * size` writable bytes, and `after` reads only +/// the `size` bytes at each pointer it is given. +pub unsafe fn heapsort( + base: *mut u8, + count: usize, + size: usize, + after: &mut impl FnMut(*const u8, *const u8) -> bool, +) { + if count < 2 || size == 0 { + return; + } + let at = |index: usize| -> *mut u8 { + // SAFETY: every index the algorithm forms is below `count`. + unsafe { base.add(index * size) } + }; + // SAFETY (the swaps): distinct indices name disjoint elements. + let sift = + |mut root: usize, end: usize, after: &mut dyn FnMut(*const u8, *const u8) -> bool| { + loop { + let left = 2 * root + 1; + if left >= end { + return; + } + let right = left + 1; + let child = if right < end && after(at(right), at(left)) { + right + } else { + left + }; + if !after(at(child), at(root)) { + return; + } + unsafe { core::ptr::swap_nonoverlapping(at(root), at(child), size) }; + root = child; + } + }; + let mut start = count / 2; + while start > 0 { + start -= 1; + sift(start, count, after); + } + let mut end = count; + while end > 1 { + end -= 1; + unsafe { core::ptr::swap_nonoverlapping(at(0), at(end), size) }; + sift(0, end, after); + } +} + +#[cfg(test)] +mod tests { + use super::*; + extern crate std; + use std::vec::Vec; + + fn sort_u32(values: &mut [u32]) { + let len = values.len(); + // SAFETY: the slice's bytes, elements of four bytes. + unsafe { + heapsort(values.as_mut_ptr().cast(), len, 4, &mut |a, b| { + a.cast::().read_unaligned() > b.cast::().read_unaligned() + }) + }; + } + + #[test] + fn sorts_every_shape() { + for len in 0..40 { + let mut ascending: Vec = (0..len).collect(); + let mut descending: Vec = (0..len).rev().collect(); + let mut repeated: Vec = (0..len).map(|i| i % 3).collect(); + let mut scrambled: Vec = + (0..len).map(|i| i.wrapping_mul(2654435761) % 97).collect(); + for values in [ + &mut ascending, + &mut descending, + &mut repeated, + &mut scrambled, + ] { + let mut expected = values.clone(); + expected.sort_unstable(); + sort_u32(values); + assert_eq!(*values, expected); + } + } + } + + #[test] + fn sorts_wide_elements_by_their_key() { + // 24-byte records keyed on their first word, as C sorts structs. + let mut records: Vec<[u64; 3]> = (0..17u64).map(|i| [(i * 7) % 17, i, !i]).collect(); + let len = records.len(); + // SAFETY: the vector's bytes, elements of 24 bytes. + unsafe { + heapsort(records.as_mut_ptr().cast(), len, 24, &mut |a, b| { + a.cast::().read_unaligned() > b.cast::().read_unaligned() + }) + }; + for (index, record) in records.iter().enumerate() { + assert_eq!(record[0], index as u64); + // Each record moved whole. + assert_eq!(record[2], !record[1]); + } + } +} diff --git a/native/crates/c-shim/src/stdio.rs b/native/crates/c-shim/src/stdio.rs new file mode 100644 index 0000000..f004ee4 --- /dev/null +++ b/native/crates/c-shim/src/stdio.rs @@ -0,0 +1,210 @@ +//! `FILE` streams: the standard pair, and the streams a consumer builds over +//! its own file layer. +//! +//! A [`File`] is a write operation and its state. C only ever holds a +//! `FILE *` to one (musl's `FILE` is incomplete in the headers every kore C +//! build uses), so the layout is this crate's own. +//! +//! - **`stdout` and `stderr`** go to the [`StandardSink`] the process +//! installs once ([`install_standard_sink`]) — its terminal, or a result +//! buffer it reports when the task ends. Until one is installed a write +//! reports `EOF` and goes nowhere, which is what every consumer that never +//! installs one has always seen. +//! - **Any other stream** is built by the consumer that owns a file layer +//! (`fdopen` over its own descriptors) with [`File::new`] and its own +//! [`FileOps`]. This crate never opens a file: kore has no ambient +//! filesystem for it to open one in. + +use core::ffi::c_void; +use core::ptr; +use core::sync::atomic::{AtomicPtr, Ordering}; + +/// A C `FILE`. +#[repr(C)] +pub struct File { + ops: &'static FileOps, + state: *mut c_void, +} + +// SAFETY: the standard streams' state is an inert tag, never dereferenced; +// a consumer that places its own stream in a `static` owns that stream's +// synchronization, as C's own `FILE` locking would. +unsafe impl Sync for File {} + +/// What a stream does with its bytes. +pub struct FileOps { + /// Accept `bytes` whole. `false` is a write error: C's `EOF`. + pub write: unsafe fn(state: *mut c_void, bytes: &[u8]) -> bool, + /// Push anything buffered onward. `false` is an error. + pub flush: unsafe fn(state: *mut c_void) -> bool, + /// `fclose`'s last act: flush and release the stream, the `File` itself + /// included when its owner allocated it. `false` is an error; the stream + /// is closed either way, as C's `fclose` leaves it. + pub close: unsafe fn(file: *mut File) -> bool, +} + +impl File { + /// A stream over `ops`, carrying the consumer's `state`. + pub const fn new(ops: &'static FileOps, state: *mut c_void) -> Self { + Self { ops, state } + } + + /// The consumer's state, for its [`FileOps`]. + pub const fn state(&self) -> *mut c_void { + self.state + } +} + +/// Write `bytes` to the stream at `file`. +/// +/// # Safety +/// `file` is a live `FILE *` this crate or a consumer constructed. +pub(crate) unsafe fn write(file: *mut File, bytes: &[u8]) -> bool { + if bytes.is_empty() { + return true; + } + // SAFETY: the caller's contract; only the fields are read. + unsafe { ((*file).ops.write)((*file).state, bytes) } +} + +/// # Safety +/// As [`write`]. +pub(crate) unsafe fn flush(file: *mut File) -> bool { + // SAFETY: the caller's contract. + unsafe { ((*file).ops.flush)((*file).state) } +} + +/// # Safety +/// As [`write`]; `file` is not used again. +pub(crate) unsafe fn close(file: *mut File) -> bool { + // SAFETY: the caller's contract. + unsafe { ((*file).ops.close)(file) } +} + +/// The two standard output streams. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Standard { + /// `stdout`. + Output, + /// `stderr`. + Error, +} + +/// Where the standard streams' bytes go, and the lock one stream call holds +/// across every write it makes. +/// +/// POSIX has each stdio call lock its stream (`flockfile`), so one call's +/// output reaches the stream together even when the call writes it in +/// pieces, as `vfprintf`'s batches and `puts`' line and newline are. The +/// process decides what the lock is: a sink only one thread writes takes +/// none. +pub struct StandardSink { + /// Write `bytes` to the stream whole. `false` reports a write error. + pub write: fn(Standard, &[u8]) -> bool, + /// Take the standard streams' lock for one call. A sink reached with + /// the lock already held by the calling thread takes it again: the lock + /// is the sink's to make reentrant. + pub lock: fn(), + /// Release one [`lock`](Self::lock). + pub unlock: fn(), +} + +/// The installed sink; null until a process installs one. +static SINK: AtomicPtr = AtomicPtr::new(ptr::null_mut()); + +/// Send `stdout` and `stderr` to `sink` from now on. A later install +/// replaces an earlier one; a call in flight finishes on the sink it read. +pub fn install_standard_sink(sink: &'static StandardSink) { + SINK.store(ptr::from_ref(sink).cast_mut(), Ordering::Release); +} + +fn installed() -> Option<&'static StandardSink> { + // SAFETY: only `install_standard_sink` stores here, and it stores a + // `&'static StandardSink`. + unsafe { SINK.load(Ordering::Acquire).as_ref() } +} + +/// Run one stream call on `file` holding its lock: around every write it +/// makes, the standard streams' sink's lock; nothing for a consumer's +/// stream, whose owner synchronizes it. +/// +/// # Safety +/// `file` is a live `FILE *` this crate or a consumer constructed. +pub(crate) unsafe fn locked(file: *mut File, call: impl FnOnce() -> R) -> R { + // SAFETY: the caller's contract. + let standard = ptr::eq(unsafe { (*file).ops }, &STANDARD_OPS); + let sink = if standard { installed() } else { None }; + if let Some(sink) = sink { + (sink.lock)(); + } + let result = call(); + if let Some(sink) = sink { + (sink.unlock)(); + } + result +} + +/// The standard streams' state: an inert tag naming which one. +const OUTPUT_TAG: *mut c_void = ptr::without_provenance_mut(1); +const ERROR_TAG: *mut c_void = ptr::without_provenance_mut(2); + +unsafe fn standard_write(state: *mut c_void, bytes: &[u8]) -> bool { + let Some(sink) = installed() else { + return false; + }; + let stream = if state == OUTPUT_TAG { + Standard::Output + } else { + Standard::Error + }; + (sink.write)(stream, bytes) +} + +unsafe fn standard_flush(_: *mut c_void) -> bool { + // The sink receives every write whole; there is nothing buffered here. + true +} + +unsafe fn standard_close(_: *mut File) -> bool { + // C lets a program close `stdout`; the stream is static, so closing it + // only ends this program's use of it. + true +} + +static STANDARD_OPS: FileOps = FileOps { + write: standard_write, + flush: standard_flush, + close: standard_close, +}; + +static STDOUT: File = File::new(&STANDARD_OPS, OUTPUT_TAG); +static STDERR: File = File::new(&STANDARD_OPS, ERROR_TAG); + +/// A `FILE *` value: musl declares `extern FILE *const stdout`, so the +/// symbol holds a pointer to the stream, not the stream. +#[repr(transparent)] +pub struct Stream(*mut File); + +// SAFETY: a pointer to an immutable static; nothing writes through it. +unsafe impl Sync for Stream {} + +impl Stream { + pub(crate) fn file(&self) -> *mut File { + self.0 + } +} + +/// The `FILE *` of a standard stream — what a consumer's `fdopen` answers +/// for descriptor 1 or 2. +pub fn standard_stream(which: Standard) -> *mut File { + match which { + Standard::Output => stdout.file(), + Standard::Error => stderr.file(), + } +} + +#[unsafe(no_mangle)] +pub static stdout: Stream = Stream(&STDOUT as *const File as *mut File); + +#[unsafe(no_mangle)] +pub static stderr: Stream = Stream(&STDERR as *const File as *mut File); diff --git a/native/crates/c-shim/src/stdlib_symbols.rs b/native/crates/c-shim/src/stdlib_symbols.rs new file mode 100644 index 0000000..fa0a508 --- /dev/null +++ b/native/crates/c-shim/src/stdlib_symbols.rs @@ -0,0 +1,240 @@ +//! The `` and `` routines that are pure computation: +//! number parsing, `qsort`, `strerror`. +//! +//! The parsers set `errno` as ISO C requires, so they sit with it behind the +//! `runtime` feature; `qsort` and `strerror` are pure but travel with them. + +use core::ffi::{ + c_char, c_double, c_float, c_int, c_long, c_longlong, c_ulong, c_ulonglong, c_void, +}; + +use crate::errno::{self, EINVAL, ERANGE}; +use crate::number; + +/// The bytes before `text`'s NUL. +/// +/// # Safety +/// `text` is NUL-terminated. +unsafe fn bytes<'a>(text: *const c_char) -> &'a [u8] { + // SAFETY: the caller's contract. + unsafe { core::ffi::CStr::from_ptr(text) }.to_bytes() +} + +/// Store `text + end` through `end_out`, as the parsers' `endptr`. +/// +/// # Safety +/// `end_out` is null or writable; `end` lies within `text`'s string. +unsafe fn report_end(text: *const c_char, end: usize, end_out: *mut *mut c_char) { + if !end_out.is_null() { + // SAFETY: the caller's contract. + unsafe { *end_out = text.add(end).cast_mut() }; + } +} + +/// The shared front of the integer parsers: parse, report the end, and set +/// `EINVAL` for a base C does not define. +/// +/// # Safety +/// C's `strtol` contract. +unsafe fn parse( + text: *const c_char, + end_out: *mut *mut c_char, + base: c_int, +) -> Option { + // SAFETY: the caller's contract. + let subject = unsafe { bytes(text) }; + let parsed = u32::try_from(base) + .ok() + .and_then(|base| number::integer(subject, base)); + let end = parsed.map_or(0, |parsed| parsed.end); + // SAFETY: the end lies within the string. + unsafe { report_end(text, end, end_out) }; + if parsed.is_none() { + errno::set(EINVAL); + } + parsed +} + +#[unsafe(no_mangle)] +unsafe extern "C" fn strtoll( + text: *const c_char, + end: *mut *mut c_char, + base: c_int, +) -> c_longlong { + // SAFETY: C's contract. + let Some(parsed) = (unsafe { parse(text, end, base) }) else { + return 0; + }; + let (value, clamped) = parsed.signed(c_longlong::BITS); + if clamped { + errno::set(ERANGE); + } + value +} + +#[unsafe(no_mangle)] +unsafe extern "C" fn strtol(text: *const c_char, end: *mut *mut c_char, base: c_int) -> c_long { + // SAFETY: C's contract. + let Some(parsed) = (unsafe { parse(text, end, base) }) else { + return 0; + }; + let (value, clamped) = parsed.signed(c_long::BITS); + if clamped { + errno::set(ERANGE); + } + value as c_long +} + +#[unsafe(no_mangle)] +unsafe extern "C" fn strtoull( + text: *const c_char, + end: *mut *mut c_char, + base: c_int, +) -> c_ulonglong { + // SAFETY: C's contract. + let Some(parsed) = (unsafe { parse(text, end, base) }) else { + return 0; + }; + let (value, clamped) = parsed.unsigned(c_ulonglong::BITS); + if clamped { + errno::set(ERANGE); + } + value +} + +#[unsafe(no_mangle)] +unsafe extern "C" fn strtoul(text: *const c_char, end: *mut *mut c_char, base: c_int) -> c_ulong { + // SAFETY: C's contract. + let Some(parsed) = (unsafe { parse(text, end, base) }) else { + return 0; + }; + let (value, clamped) = parsed.unsigned(c_ulong::BITS); + if clamped { + errno::set(ERANGE); + } + value as c_ulong +} + +/// `atoi`/`atol`: `strtol` in base 10, as glibc defines them — out-of-range +/// input is undefined in C; here it is `strtol`'s clamp, truncated. +#[unsafe(no_mangle)] +unsafe extern "C" fn atoi(text: *const c_char) -> c_int { + // SAFETY: C's contract. + unsafe { strtol(text, core::ptr::null_mut(), 10) as c_int } +} + +#[unsafe(no_mangle)] +unsafe extern "C" fn atol(text: *const c_char) -> c_long { + // SAFETY: C's contract. + unsafe { strtol(text, core::ptr::null_mut(), 10) } +} + +#[unsafe(no_mangle)] +unsafe extern "C" fn strtod(text: *const c_char, end: *mut *mut c_char) -> c_double { + // SAFETY: C's contract. + let parsed = number::double(unsafe { bytes(text) }); + unsafe { report_end(text, parsed.end, end) }; + if parsed.range { + errno::set(ERANGE); + } + parsed.value +} + +#[unsafe(no_mangle)] +unsafe extern "C" fn strtof(text: *const c_char, end: *mut *mut c_char) -> c_float { + // SAFETY: C's contract. + let parsed = number::single(unsafe { bytes(text) }); + unsafe { report_end(text, parsed.end, end) }; + if parsed.range { + errno::set(ERANGE); + } + parsed.value +} + +/// A C comparator. +type Compare = unsafe extern "C" fn(*const c_void, *const c_void) -> c_int; + +#[unsafe(no_mangle)] +unsafe extern "C" fn qsort(base: *mut c_void, count: usize, size: usize, compare: Compare) { + // SAFETY: C's contract: `count * size` bytes at `base`, and a comparator + // that reads only the elements it is handed. + unsafe { + crate::sort::heapsort(base.cast(), count, size, &mut |a, b| { + compare(a.cast(), b.cast()) > 0 + }) + }; +} + +#[unsafe(no_mangle)] +extern "C" fn strerror(code: c_int) -> *mut c_char { + errno::describe(code).as_ptr().cast_mut() +} + +/// `strtold`'s byte-level half (`long_double`): parse `text` as the +/// `long double` `mant_dig` names, store its [`long_double::BYTES`] bytes at +/// `out`, and the end at `*end`. Answers the `errno` the C wrapper sets — +/// `ERANGE` as `strtod` would, `EINVAL` for a format or buffer this crate +/// has no ABI for (the value is then `+0` and nothing is consumed) — or `0`. +/// +/// # Safety +/// C's `strtold` contract for `text` and `end`; `out` is writable for +/// `out_len` bytes. +#[cfg(feature = "long-double")] +#[unsafe(no_mangle)] +unsafe extern "C" fn __kore_strtold_bytes( + text: *const c_char, + end: *mut *mut c_char, + mant_dig: c_int, + out: *mut u8, + out_len: usize, +) -> c_int { + use crate::long_double; + let Some(format) = + long_double::Format::from_mant_dig(mant_dig).filter(|_| out_len == long_double::BYTES) + else { + // SAFETY: C's contract. + unsafe { report_end(text, 0, end) }; + if !out.is_null() { + // SAFETY: `out_len` writable bytes. + unsafe { core::ptr::write_bytes(out, 0, out_len) }; + } + return EINVAL; + }; + // SAFETY: C's contract. + let parsed = long_double::parse(format, unsafe { bytes(text) }); + unsafe { + report_end(text, parsed.end, end); + out.cast::<[u8; long_double::BYTES]>() + .write_unaligned(long_double::to_bytes(parsed.bits)); + } + if parsed.range { ERANGE } else { 0 } +} + +/// `ldexpl`'s byte-level half: replace the `long double` at `value` with +/// itself times `2^exponent`, rounded once. `EINVAL` (value untouched) for a +/// format or buffer this crate has no ABI for, else `0`. +/// +/// # Safety +/// `value` is readable and writable for `len` bytes. +#[cfg(feature = "long-double")] +#[unsafe(no_mangle)] +unsafe extern "C" fn __kore_ldexpl_bytes( + value: *mut u8, + len: usize, + mant_dig: c_int, + exponent: c_int, +) -> c_int { + use crate::long_double; + let Some(format) = + long_double::Format::from_mant_dig(mant_dig).filter(|_| len == long_double::BYTES) + else { + return EINVAL; + }; + let slot = value.cast::<[u8; long_double::BYTES]>(); + // SAFETY: the caller's `len` bytes. + let bits = long_double::from_bytes(format, unsafe { slot.read_unaligned() }); + let scaled = long_double::scale(format, bits, exponent); + // SAFETY: as above. + unsafe { slot.write_unaligned(long_double::to_bytes(scaled)) }; + 0 +} diff --git a/native/crates/c-shim/src/string_symbols.rs b/native/crates/c-shim/src/string_symbols.rs new file mode 100644 index 0000000..92206b4 --- /dev/null +++ b/native/crates/c-shim/src/string_symbols.rs @@ -0,0 +1,158 @@ +//! The ISO string routines a C library imports beyond `compiler_builtins`' +//! `memcpy`/`memmove`/`memset`/`memcmp`/`strlen`. + +use core::ffi::{CStr, c_char, c_int, c_void}; + +/// The bytes of a C string, without its NUL. +/// +/// # Safety +/// `text` is NUL-terminated. +unsafe fn bytes<'a>(text: *const c_char) -> &'a [u8] { + // SAFETY: the caller's contract. + unsafe { CStr::from_ptr(text) }.to_bytes() +} + +#[unsafe(no_mangle)] +unsafe extern "C" fn strchr(text: *const c_char, value: c_int) -> *mut c_char { + let needle = value as u8; + // SAFETY: C promises a NUL-terminated string. + let body = unsafe { bytes(text) }; + // The terminator itself is part of the searchable string: `strchr(s, 0)` + // answers the NUL's address. + let found = if needle == 0 { + Some(body.len()) + } else { + body.iter().position(|&byte| byte == needle) + }; + match found { + // SAFETY: the offset lies within the string or on its NUL. + Some(offset) => unsafe { text.add(offset) }.cast_mut(), + None => core::ptr::null_mut(), + } +} + +#[unsafe(no_mangle)] +unsafe extern "C" fn strrchr(text: *const c_char, value: c_int) -> *mut c_char { + let needle = value as u8; + // SAFETY: C promises a NUL-terminated string. + let body = unsafe { bytes(text) }; + let found = if needle == 0 { + Some(body.len()) + } else { + body.iter().rposition(|&byte| byte == needle) + }; + match found { + // SAFETY: as `strchr`. + Some(offset) => unsafe { text.add(offset) }.cast_mut(), + None => core::ptr::null_mut(), + } +} + +#[unsafe(no_mangle)] +unsafe extern "C" fn strcpy(target: *mut c_char, source: *const c_char) -> *mut c_char { + // SAFETY: C promises `source` NUL-terminated and `target` large enough + // for it and its NUL, not overlapping. + unsafe { + let len = bytes(source).len(); + core::ptr::copy_nonoverlapping(source, target, len + 1); + } + target +} + +#[unsafe(no_mangle)] +unsafe extern "C" fn strstr(haystack: *const c_char, needle: *const c_char) -> *mut c_char { + // SAFETY: C promises both NUL-terminated. + let (hay, pin) = unsafe { (bytes(haystack), bytes(needle)) }; + let Some((&first, rest)) = pin.split_first() else { + return haystack.cast_mut(); + }; + if pin.len() > hay.len() { + return core::ptr::null_mut(); + } + // Scan for the first byte, then compare the rest in place. + let last_start = hay.len() - pin.len(); + let mut at = 0; + while let Some(skip) = hay[at..=last_start].iter().position(|&byte| byte == first) { + let start = at + skip; + if &hay[start + 1..start + pin.len()] == rest { + // SAFETY: `start` lies within the haystack. + return unsafe { haystack.add(start) }.cast_mut(); + } + at = start + 1; + if at > last_start { + break; + } + } + core::ptr::null_mut() +} + +#[unsafe(no_mangle)] +unsafe extern "C" fn memchr(source: *const c_void, value: c_int, len: usize) -> *mut c_void { + let bytes = source.cast::(); + let needle = value as u8; + for index in 0..len { + // SAFETY: C promises `len` readable bytes at `source`. + if unsafe { *bytes.add(index) } == needle { + return unsafe { bytes.add(index) }.cast_mut().cast(); + } + } + core::ptr::null_mut() +} + +#[unsafe(no_mangle)] +unsafe extern "C" fn strcmp(left: *const c_char, right: *const c_char) -> c_int { + // SAFETY: both are NUL-terminated, so the bounded compare stops at a NUL. + unsafe { strncmp(left, right, usize::MAX) } +} + +#[unsafe(no_mangle)] +unsafe extern "C" fn strncmp(left: *const c_char, right: *const c_char, len: usize) -> c_int { + for index in 0..len { + // SAFETY: C promises both strings readable up to their NUL or `len`. + let (a, b) = unsafe { (*left.add(index) as u8, *right.add(index) as u8) }; + if a != b { + return c_int::from(a) - c_int::from(b); + } + if a == 0 { + break; + } + } + 0 +} + +#[unsafe(no_mangle)] +unsafe extern "C" fn strncpy(target: *mut c_char, source: *const c_char, len: usize) -> *mut c_char { + let mut index = 0; + // SAFETY: C promises `len` writable bytes at `target` and `source` + // readable up to its NUL or `len`; the tail is NUL-padded, as ISO says. + unsafe { + while index < len && *source.add(index) != 0 { + *target.add(index) = *source.add(index); + index += 1; + } + while index < len { + *target.add(index) = 0; + index += 1; + } + } + target +} + +#[unsafe(no_mangle)] +unsafe extern "C" fn strncat(target: *mut c_char, source: *const c_char, len: usize) -> *mut c_char { + // SAFETY: C promises `target` NUL-terminated with room for at most `len` + // more bytes and a NUL, and `source` readable up to its NUL or `len`. + unsafe { + let mut end = 0; + while *target.add(end) != 0 { + end += 1; + } + let mut index = 0; + while index < len && *source.add(index) != 0 { + *target.add(end + index) = *source.add(index); + index += 1; + } + *target.add(end + index) = 0; + } + target +} diff --git a/native/crates/c-shim/src/va_long_double/aarch64.rs b/native/crates/c-shim/src/va_long_double/aarch64.rs new file mode 100644 index 0000000..3aaf12a --- /dev/null +++ b/native/crates/c-shim/src/va_long_double/aarch64.rs @@ -0,0 +1,47 @@ +//! AAPCS64: a `long double` is a quad-precision floating-point type, passed in +//! the next SIMD and floating-point register while any remains, its save slot +//! in the register save area, and on the stack aligned to sixteen after that +//! (AAPCS64 §B.4, `va_arg` for a floating-point type). + +use super::{BYTES, Float, VaList, align_up, read}; + +/// `va_list` as AAPCS64 declares it (`__va_list`). +#[repr(C)] +struct List { + stack: *mut u8, + gr_top: *mut u8, + vr_top: *mut u8, + gr_offs: i32, + vr_offs: i32, +} + +const _: () = assert!(core::mem::size_of::>() == core::mem::size_of::()); + +/// One vector register's save slot. +const VECTOR_SLOT: i32 = 16; + +/// The next argument, a binary128 `long double`. +/// +/// # Safety +/// The next argument `list` holds is a `long double`. +pub(crate) unsafe fn next(list: &mut VaList<'_>) -> Float { + // SAFETY: `VaList` is the platform's `va_list` on this target (the size + // is asserted), and both areas are the caller's. + let list = unsafe { &mut *(list as *mut VaList<'_>).cast::() }; + let offset = list.vr_offs; + let at = if offset < 0 && offset + VECTOR_SLOT <= 0 { + list.vr_offs = offset + VECTOR_SLOT; + list.vr_top.wrapping_offset(offset as isize) + } else { + if offset < 0 { + // The registers ran out under this argument: the standard still + // counts it against them before it reads the stack. + list.vr_offs = offset + VECTOR_SLOT; + } + let at = align_up(list.stack); + list.stack = at.wrapping_add(BYTES); + at + }; + // SAFETY: the caller's contract puts the argument here. + Float::from_quad(unsafe { read(at) }) +} diff --git a/native/crates/c-shim/src/va_long_double/mod.rs b/native/crates/c-shim/src/va_long_double/mod.rs new file mode 100644 index 0000000..5f6922a --- /dev/null +++ b/native/crates/c-shim/src/va_long_double/mod.rs @@ -0,0 +1,54 @@ +//! `va_arg(ap, long double)`, which Rust's [`VaList`] cannot spell: Rust has +//! no type for x87's 80-bit format and no `VaArgSafe` binary128, so each ABI's +//! own rule reads the argument's bytes and the `printf` engine decodes them +//! ([`Float`]). +//! +//! [`VaList`] is laid out as the platform's `va_list`, the struct or pointer +//! its procedure-call standard defines, and the rule for a 16-byte, +//! 16-aligned argument comes from that same standard. One file per ABI, +//! selected here; the engine never names one. + +use core::ffi::VaList; + +use crate::format::Float; + +#[cfg(target_arch = "aarch64")] +mod aarch64; +#[cfg(target_arch = "riscv64")] +mod riscv64; +#[cfg(target_arch = "x86_64")] +mod x86_64; + +#[cfg(target_arch = "aarch64")] +pub(crate) use aarch64::next; +#[cfg(target_arch = "riscv64")] +pub(crate) use riscv64::next; +#[cfg(target_arch = "x86_64")] +pub(crate) use x86_64::next; + +#[cfg(not(any( + target_arch = "aarch64", + target_arch = "riscv64", + target_arch = "x86_64" +)))] +compile_error!("kore's C ABIs are x86_64, aarch64 and riscv64: name this one's long double rule"); + +/// Every kore C ABI's `long double`: sixteen bytes, aligned to sixteen (x87's +/// ten padded out, or binary128's own). +const BYTES: usize = 16; +const ALIGN: usize = 16; + +/// The 16-aligned address at or above `at`. +fn align_up(at: *mut u8) -> *mut u8 { + at.wrapping_add(at.addr().wrapping_neg() & (ALIGN - 1)) +} + +/// The `long double` stored at `at`, as its bits. +/// +/// # Safety +/// `at` is the address of a `long double` argument the caller passed. +unsafe fn read(at: *const u8) -> u128 { + // SAFETY: the caller's contract; unaligned for no reason but uniformity, + // since every rule aligns its slot. + u128::from_le_bytes(unsafe { at.cast::<[u8; BYTES]>().read_unaligned() }) +} diff --git a/native/crates/c-shim/src/va_long_double/riscv64.rs b/native/crates/c-shim/src/va_long_double/riscv64.rs new file mode 100644 index 0000000..d2b872f --- /dev/null +++ b/native/crates/c-shim/src/va_long_double/riscv64.rs @@ -0,0 +1,24 @@ +//! RISC-V LP64D: a `long double` is a 2×XLEN scalar with 2×XLEN alignment, so a +//! variadic one takes an aligned (even, odd) pair of argument registers, or +//! the stack aligned to sixteen (psABI, "Integer Calling Convention" and +//! "va_list, va_start, and va_arg"). The prologue spills the argument +//! registers just below the stack arguments, a pair to a 16-aligned slot, so +//! one pointer walks both and `va_arg` aligns it. + +use super::{BYTES, Float, VaList, align_up, read}; + +const _: () = assert!(core::mem::size_of::>() == core::mem::size_of::<*mut u8>()); + +/// The next argument, a binary128 `long double`. +/// +/// # Safety +/// The next argument `list` holds is a `long double`. +pub(crate) unsafe fn next(list: &mut VaList<'_>) -> Float { + // SAFETY: `VaList` is the platform's `va_list`, one pointer, on this + // target (the size is asserted). + let cursor = unsafe { &mut *(list as *mut VaList<'_>).cast::<*mut u8>() }; + let at = align_up(*cursor); + *cursor = at.wrapping_add(BYTES); + // SAFETY: the caller's contract puts the argument here. + Float::from_quad(unsafe { read(at) }) +} diff --git a/native/crates/c-shim/src/va_long_double/x86_64.rs b/native/crates/c-shim/src/va_long_double/x86_64.rs new file mode 100644 index 0000000..79de2cf --- /dev/null +++ b/native/crates/c-shim/src/va_long_double/x86_64.rs @@ -0,0 +1,32 @@ +//! System V AMD64: a `long double` is class X87, which the caller always +//! passes in memory, in the overflow area, aligned to sixteen (psABI §3.5.7, +//! the `va_arg` algorithm's step for a type that is not passed in +//! registers). + +use super::{BYTES, Float, VaList, align_up, read}; + +/// `va_list`'s one element, as the psABI declares it (`__va_list_tag`). +#[repr(C)] +struct Tag { + gp_offset: u32, + fp_offset: u32, + overflow_arg_area: *mut u8, + reg_save_area: *mut u8, +} + +const _: () = assert!(core::mem::size_of::>() == core::mem::size_of::()); + +/// The next argument, an x87 `long double`. +/// +/// # Safety +/// The next argument `list` holds is a `long double`. +pub(crate) unsafe fn next(list: &mut VaList<'_>) -> Float { + // SAFETY: `VaList` is the platform's `va_list` element on this target + // (the size is asserted), and the overflow area is the caller's frame. + let tag = unsafe { &mut *(list as *mut VaList<'_>).cast::() }; + let at = align_up(tag.overflow_arg_area); + // SAFETY: the caller's contract puts the argument here. + let bits = unsafe { read(at) }; + tag.overflow_arg_area = at.wrapping_add(BYTES); + Float::from_x87(bits) +} diff --git a/native/crates/c-shim/src/wide_symbols.rs b/native/crates/c-shim/src/wide_symbols.rs new file mode 100644 index 0000000..34a3191 --- /dev/null +++ b/native/crates/c-shim/src/wide_symbols.rs @@ -0,0 +1,77 @@ +//! The `` classes and case maps, answered from Rust's Unicode +//! tables: a grammar's scanner asks whether a code point is a letter or a +//! space, and the answer should not depend on a locale kore does not have. + +use core::ffi::c_int; + +/// C's `wint_t`: an unsigned 32-bit code point (or `WEOF`). +type WInt = u32; + +fn class(code: WInt, test: impl Fn(char) -> bool) -> c_int { + c_int::from(char::from_u32(code).is_some_and(test)) +} + +#[unsafe(no_mangle)] +extern "C" fn iswalnum(code: WInt) -> c_int { + class(code, char::is_alphanumeric) +} + +#[unsafe(no_mangle)] +extern "C" fn iswalpha(code: WInt) -> c_int { + class(code, char::is_alphabetic) +} + +#[unsafe(no_mangle)] +extern "C" fn iswblank(code: WInt) -> c_int { + class(code, |c| c == ' ' || c == '\t' || (c.is_whitespace() && !matches!(c, '\n'..='\r' | '\u{85}' | '\u{2028}' | '\u{2029}'))) +} + +#[unsafe(no_mangle)] +extern "C" fn iswdigit(code: WInt) -> c_int { + class(code, |c| c.is_ascii_digit()) +} + +#[unsafe(no_mangle)] +extern "C" fn iswlower(code: WInt) -> c_int { + class(code, char::is_lowercase) +} + +#[unsafe(no_mangle)] +extern "C" fn iswupper(code: WInt) -> c_int { + class(code, char::is_uppercase) +} + +#[unsafe(no_mangle)] +extern "C" fn iswspace(code: WInt) -> c_int { + class(code, char::is_whitespace) +} + +#[unsafe(no_mangle)] +extern "C" fn iswxdigit(code: WInt) -> c_int { + class(code, |c| c.is_ascii_hexdigit()) +} + +/// Printable, not a space, not a letter or digit: ISO's definition. +#[unsafe(no_mangle)] +extern "C" fn iswpunct(code: WInt) -> c_int { + class(code, |c| !c.is_whitespace() && !c.is_alphanumeric() && !c.is_control()) +} + +/// The one-code-point case maps; a mapping that expands to several code +/// points (`ß` → `SS`) leaves the code point as it is, as C's does. +fn single(mut mapped: impl Iterator, code: WInt) -> WInt { + match (mapped.next(), mapped.next()) { + (Some(c), None) => c as WInt, + _ => code, + } +} + +#[unsafe(no_mangle)] +extern "C" fn towlower(code: WInt) -> WInt { + char::from_u32(code).map_or(code, |c| single(c.to_lowercase(), code)) +} + +#[unsafe(no_mangle)] +extern "C" fn towupper(code: WInt) -> WInt { + char::from_u32(code).map_or(code, |c| single(c.to_uppercase(), code)) +} diff --git a/native/crates/dagfs/src/fs.rs b/native/crates/dagfs/src/fs.rs index 55b4b8d..3dd38e4 100644 --- a/native/crates/dagfs/src/fs.rs +++ b/native/crates/dagfs/src/fs.rs @@ -106,6 +106,10 @@ pub struct Fs<'w, 't> { pub(crate) ring_files: u64, /// Why the last whole-file read did or did not take that ring. pub(crate) ring_note: &'static str, + /// This pair's watch plane — the event channel the service formats + /// after the rings in this pair's own window (`crate::watch`) — or + /// `None` on a pair formatted without one. + pub(crate) watch: Option>, } impl<'w, 't> Fs<'w, 't> { @@ -128,11 +132,13 @@ impl<'w, 't> Fs<'w, 't> { // window only through the SPSC protocol. let window: &'static mut [u8] = unsafe { core::slice::from_raw_parts_mut(window.cast::().as_ptr(), window.len()) }; - // An *attach*, not an init: the filesystem row's own `init_window` - // formatted this ring at founding, and `client_rings` fail-closes - // on an unformatted window rather than reading a zeroed page as an - // empty ring. - let rings = client_rings::(window, REQ_CAP, kore_rt::svc::REP_CAP).ok()?; + // An *attach*, not an init: the filesystem row's own formatter + // laid out this window at founding — the rings, then the watch + // plane — and both attaches fail closed on an unformatted window + // rather than reading a zeroed page as an empty ring. + let (pair, tail) = window.split_at_mut(crate::watch::PLANE_AT.min(window.len())); + let rings = client_rings::(pair, REQ_CAP, kore_rt::svc::REP_CAP).ok()?; + let watch = crate::watch::sink(tail).ok(); let kick = rt .bootstrap() .handle(role::SVC_SEND, process_face::REQUEST_KICK)?; @@ -223,9 +229,70 @@ impl<'w, 't> Fs<'w, 't> { ring, ring_files: 0, ring_note: "unused", + watch, }) } + /// This pair's watch plane, if the pair carries one. + pub fn watch_plane(&self) -> Option<&crate::watch::WatchSink<'static>> { + self.watch.as_ref() + } + + /// Arm watch `slot` on the directory `dir`, tagged `tag` (a non-zero + /// name of the client's own for this arming): the plane counts every + /// change to what listing `dir` shows from now on. The view stays the + /// caller's — closing it ends the watch. `Unsupported` on a pair without + /// a plane. + pub async fn watch(&mut self, dir: &crate::View, slot: u8, tag: u32) -> Result<(), Refusal> { + if self.watch.is_none() { + return Err(Refusal::Fs(crate::FsError::Unsupported)); + } + self.checked(&FsRequest::Watch { + handle: dir.handle(), + slot, + tag, + }) + .await + .map(|_| ()) + } + + /// Disarm watch `slot`. + pub async fn unwatch(&mut self, slot: u8) -> Result<(), Refusal> { + self.checked(&FsRequest::Unwatch { slot }).await.map(|_| ()) + } + + /// The plane's wake word, read before a caller drains its sources: a + /// park published against it ([`Self::arm_watch_park`]) is lossless. + /// Zero, and never moving, on a pair without a plane. + pub fn watch_seq(&self) -> u32 { + self.watch.as_ref().map_or(0, crate::watch::WatchSink::seq) + } + + /// The plane's state, `None` on a pair without one (or mid-publish, + /// which the caller treats as nothing new and reads again next pass — + /// the publish ends by moving the wake word). + pub fn watch_state(&self) -> Option { + self.watch.as_ref().and_then(crate::watch::WatchSink::state) + } + + /// Publish this task's intent to wait on the plane, for a caller about to + /// enter one executor wait that every source shares (the multiplexed + /// park, `res/FILES_DESKTOP.md`): the service's next batch then owes + /// this task a notice on its doorbell. `false` when the plane already + /// moved past `seen` — drain again instead of waiting. Always `true` on + /// a pair without a plane, which has nothing to wait for. + pub fn arm_watch_park(&self, seen: u32) -> bool { + self.watch.as_ref().is_none_or(|sink| sink.park(seen)) + } + + /// Withdraw the intent [`Self::arm_watch_park`] published, however the + /// wait ended, so a departed waiter is never charged a notice. + pub fn withdraw_watch_park(&self) { + if let Some(sink) = self.watch.as_ref() { + sink.unpark(); + } + } + /// The bulk data window, read side. Bytes a verb landed here are valid /// until the next request that uses the window. pub fn window(&self) -> &[u8] { diff --git a/native/crates/dagfs/src/lib.rs b/native/crates/dagfs/src/lib.rs index 4800948..0597035 100644 --- a/native/crates/dagfs/src/lib.rs +++ b/native/crates/dagfs/src/lib.rs @@ -67,6 +67,8 @@ mod view; #[cfg(feature = "runtime")] mod walk; #[cfg(feature = "runtime")] +pub mod watch; +#[cfg(feature = "runtime")] pub mod wire; #[cfg(feature = "runtime")] diff --git a/native/crates/dagfs/src/protocol.rs b/native/crates/dagfs/src/protocol.rs index abfbe60..3125092 100644 --- a/native/crates/dagfs/src/protocol.rs +++ b/native/crates/dagfs/src/protocol.rs @@ -193,6 +193,22 @@ pub enum FsRequest { Sync { handle: Handle, }, + /// Arm watch `slot` (below [`FS_WATCH_SLOTS`](kore_abi::svc::FS_WATCH_SLOTS)) + /// on the directory `handle` names, tagged `tag` (non-zero): the pair's + /// watch plane ([`crate::watch`]) counts every change to what a listing + /// of that directory shows, under `tag`, until the slot is re-armed or + /// disarmed or the watch ends. The handle stays the client's; its route + /// is revalidated before every count, and one that stops proving ends the + /// watch rather than reporting through a route the client lost. + Watch { + handle: Handle, + slot: u8, + tag: u32, + }, + /// Disarm watch `slot`, armed or not. + Unwatch { + slot: u8, + }, /// Hold this session's whole-tree commits until the window closes. /// /// The durable contract serializes session zero's entire tree after every @@ -631,6 +647,17 @@ impl SvcProtocol for DagfsProtocol { /// execution pipeline and made boot topology scale with the command set. const SERVE_PAIRS: usize = 2; + /// The rings, then the watch plane ([`crate::watch`]) in the same window: + /// every filesystem pair carries its change channel, so a client never + /// needs a capability beyond the pair to learn that a directory changed. + fn init_pair_window( + window: &mut [u8], + req_cap: u32, + rep_cap: u32, + ) -> Result<(), kore_ring::spsc::RingError> { + crate::watch::init_pair_window(window, req_cap, rep_cap) + } + fn reply_matches(request: &FsReqWire, response: &FsResponse, grants: u8) -> bool { if grants != 0 { return false; diff --git a/native/crates/dagfs/src/text.rs b/native/crates/dagfs/src/text.rs index 49ac01e..3af8d2f 100644 --- a/native/crates/dagfs/src/text.rs +++ b/native/crates/dagfs/src/text.rs @@ -1,6 +1,7 @@ //! The two text helpers every command needs and none should re-spell: a -//! bounded whitespace splitter over one command line, and a bounded line -//! buffer for composing an answer out of pieces. +//! bounded whitespace splitter over one command line (with `"quoted"` words, +//! so a name with spaces can be named), and a bounded line buffer for +//! composing an answer out of pieces. /// A whitespace splitter over one completed command line. /// @@ -18,13 +19,27 @@ impl<'a> Args<'a> { Self { rest: line } } - /// The next space-separated word, or `None` at the end of the line. + /// The next word, or `None` at the end of the line. A word is a run of + /// non-spaces, or — opened by `"` — everything up to the next `"`, + /// spaces included and quotes excluded, so `inspect "Untitled Folder"` + /// names the folder Files makes. A quote left open runs to the end of + /// the line. Borrowed either way: a quoted word is a span of the line. pub fn next(&mut self) -> Option<&'a [u8]> { let start = self.skip_spaces(); if start >= self.rest.len() { self.rest = &self.rest[self.rest.len()..]; return None; } + if self.rest[start] == b'"' { + let open = start + 1; + let close = self.rest[open..] + .iter() + .position(|&byte| byte == b'"') + .map_or(self.rest.len(), |at| open + at); + let word = &self.rest[open..close]; + self.rest = &self.rest[(close + 1).min(self.rest.len())..]; + return Some(word); + } let mut end = start; while end < self.rest.len() && self.rest[end] != b' ' { end += 1; @@ -128,6 +143,21 @@ mod tests { assert_eq!(args.rest(), b""); } + #[test] + fn a_quoted_word_keeps_its_spaces_and_drops_its_quotes() { + let mut args = Args::new(b"remove \"Untitled Folder\" next \"\" \"open to the end"); + assert_eq!(args.next(), Some(&b"remove"[..])); + assert_eq!(args.next(), Some(&b"Untitled Folder"[..])); + assert_eq!(args.next(), Some(&b"next"[..])); + assert_eq!(args.next(), Some(&b""[..]), "an empty quote is an empty word"); + assert_eq!(args.next(), Some(&b"open to the end"[..])); + assert_eq!(args.next(), None); + // A quote inside a word is part of it: only a word's first byte opens. + let mut args = Args::new(b"a\"b c"); + assert_eq!(args.next(), Some(&b"a\"b"[..])); + assert_eq!(args.next(), Some(&b"c"[..])); + } + #[test] fn a_line_buffer_truncates_rather_than_grows() { let mut line = LineBuf::<8>::new(); diff --git a/native/crates/dagfs/src/watch.rs b/native/crates/dagfs/src/watch.rs new file mode 100644 index 0000000..1f9b68a --- /dev/null +++ b/native/crates/dagfs/src/watch.rs @@ -0,0 +1,251 @@ +//! The **watch plane**: a filesystem pair's change channel (`res/EVENT.md`). +//! +//! A client that shows a directory — the Files explorer — needs to learn when +//! what it shows went stale without asking. The plane is the +//! service→client direction of one pair, as the window pair's event channel is +//! the compositor→client one: an event channel formatted after the +//! request/reply rings in the pair's **own** window, so it rides the lease +//! both sides already hold and needs no capability of its own. +//! +//! - **State is level.** [`FsWatchState`] holds, per slot, the tag the slot +//! was armed with and a wrapping count of changes since; the service +//! republishes it whole. Several changes between two service passes advance +//! a count once — a client lists again, it does not replay. +//! - **The wake is conditional.** The service commits after the request (or +//! teardown) that changed a watched directory and owes the client a notice +//! only if it had parked on the plane; a client that is running pays the +//! service nothing. +//! - **No authority, no names.** A watch is armed on a directory handle the +//! client holds; the service revalidates that handle's route before every +//! count and ends the watch the moment it stops proving. The plane says +//! *that* a listing may differ, never *what* it holds — the client lists +//! through its own handle. +//! +//! The record ring is reserved ([`FS_WATCH_RECORDS`], never pushed) so a later +//! per-entry change record lands without moving the state or the offset. + +use kore_abi::svc::{FS_WATCH_RECORDS, FS_WATCH_SLOTS, FsWatchRecord, FsWatchState}; +use kore_ring::event::{EventSink, EventSource}; +use kore_ring::spsc::RingError; +use kore_rt::svc::{REP_CAP, REQ_CAP, SvcProtocol}; + +use crate::DagfsProtocol; + +/// The producer half: the service's. +pub type WatchSource<'a> = EventSource<'a, FsWatchState, FsWatchRecord>; +/// The consumer half: the client's. +pub type WatchSink<'a> = EventSink<'a, FsWatchState, FsWatchRecord>; + +/// Where the plane begins in a pair window: right after the canonical +/// request/reply rings (both 64-byte rounded, so the plane is line-aligned). +pub const PLANE_AT: usize = kore_rt::svc::window_bytes::( + ::REQUEST_CAPACITY, + ::RESPONSE_CAPACITY, +); + +/// Bytes the plane occupies. +pub const PLANE_BYTES: usize = + kore_ring::event::region_bytes::(FS_WATCH_RECORDS); + +/// The smallest pair window that carries the plane. +pub const WINDOW_BYTES: usize = PLANE_AT + PLANE_BYTES; + +// Whoever sizes a filesystem pair window asserts it holds `WINDOW_BYTES` +// against its own constant (exec's `FS_CONTROL_BYTES`); the formatter refuses +// a smaller window at founding. Line alignment is the layout's own invariant. +const _: () = assert!(PLANE_AT % 64 == 0); +const _: () = assert!( + ::REQUEST_CAPACITY == REQ_CAP + && ::RESPONSE_CAPACITY == REP_CAP +); + +/// Format a whole filesystem pair window — the rings, then the plane — before +/// either side maps it. The protocol's [`SvcProtocol::init_pair_window`]. +pub fn init_pair_window(window: &mut [u8], req_cap: u32, rep_cap: u32) -> Result<(), RingError> { + kore_rt::svc::init_window::(window, req_cap, rep_cap)?; + let at = kore_rt::svc::window_bytes::(req_cap, rep_cap); + let plane = window.get_mut(at..).ok_or(RingError::TooSmall)?; + kore_ring::event::init::(plane, FS_WATCH_RECORDS) +} + +/// The service's half over its mapping of the pair window's tail (`aux` is +/// what follows the rings, as `attach_peer_aux` hands it over). +pub fn source(aux: &mut [u8]) -> Result, RingError> { + kore_ring::event::source(aux.get_mut(..PLANE_BYTES).ok_or(RingError::TooSmall)?) +} + +/// The client's half over its mapping of the same tail. +pub fn sink(aux: &mut [u8]) -> Result, RingError> { + kore_ring::event::sink(aux.get_mut(..PLANE_BYTES).ok_or(RingError::TooSmall)?) +} + +/// What one slot's watch did since a [`Cursor`] last looked. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Seen { + /// Nothing the cursor has not seen. + Same, + /// The directory changed: list it again. + Changed, + /// The watch ended (its route stopped proving, or the directory went): + /// what was shown cannot be refreshed through this watch. + Ended, +} + +/// A client's view of one slot: the tag it armed and the last count it acted +/// on. A slot is read only under the cursor's own tag, so the counts the +/// slot carried for an earlier arming — or for nobody — never read as news. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct Cursor { + slot: u8, + tag: u32, + seen: u32, + ended: bool, +} + +impl Cursor { + /// A cursor for `slot` armed with `tag`; `None` for a slot past the plane + /// or the zero tag (the plane's "disarmed"). + pub const fn new(slot: u8, tag: u32) -> Option { + if slot as usize >= FS_WATCH_SLOTS || tag == 0 { + return None; + } + Some(Self { + slot, + tag, + seen: 0, + ended: false, + }) + } + + pub const fn slot(&self) -> u8 { + self.slot + } + + pub const fn tag(&self) -> u32 { + self.tag + } + + /// Compare `state` with what this cursor last acted on, and take it as + /// seen. A state still showing another tag in the slot (the arming has + /// not landed, or the slot was re-armed by someone else's request on + /// this pair) is [`Seen::Same`]: nothing of this watch is in it yet. + pub fn observe(&mut self, state: &FsWatchState) -> Seen { + let slot = usize::from(self.slot); + if state.tag[slot] != self.tag { + return Seen::Same; + } + if state.ended & (1 << slot) != 0 { + if self.ended { + return Seen::Same; + } + self.ended = true; + return Seen::Ended; + } + let changes = state.changes[slot]; + if changes == self.seen { + return Seen::Same; + } + self.seen = changes; + Seen::Changed + } +} + +/// The plane's state as a producer edits it — the service's one batch verb +/// over a slot. Kept beside the layout so both halves read one definition. +pub mod state { + use super::{FS_WATCH_SLOTS, FsWatchState}; + + /// Arm `slot` under `tag`: no changes yet, not ended. + pub fn arm(state: &mut FsWatchState, slot: usize, tag: u32) { + if slot < FS_WATCH_SLOTS { + state.tag[slot] = tag; + state.changes[slot] = 0; + state.ended &= !(1 << slot); + } + } + + /// Disarm `slot`. + pub fn disarm(state: &mut FsWatchState, slot: usize) { + if slot < FS_WATCH_SLOTS { + state.tag[slot] = 0; + state.changes[slot] = 0; + state.ended &= !(1 << slot); + } + } + + /// One more change under `slot`'s arming. + pub fn changed(state: &mut FsWatchState, slot: usize) { + if slot < FS_WATCH_SLOTS { + state.changes[slot] = state.changes[slot].wrapping_add(1); + } + } + + /// `slot`'s watch ended; its tag stays, so the client that armed it + /// learns that it did. + pub fn end(state: &mut FsWatchState, slot: usize) { + if slot < FS_WATCH_SLOTS { + state.ended |= 1 << slot; + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::vec; + + #[repr(align(64))] + struct Window([u8; WINDOW_BYTES]); + + #[test] + fn the_plane_follows_the_rings_and_both_halves_attach() { + let mut window = Window([0; WINDOW_BYTES]); + init_pair_window(&mut window.0, REQ_CAP, REP_CAP).unwrap(); + // The rings are exactly where every existing client expects them. + let (pair, aux) = window.0.split_at_mut(PLANE_AT); + assert!(kore_rt::svc::client_rings::(pair, REQ_CAP, REP_CAP).is_ok()); + let base = aux.as_mut_ptr(); + // SAFETY: the two halves alias the plane only through the channel + // protocol, as the cross-space form does over two mappings. + let alias = unsafe { core::slice::from_raw_parts_mut(base, PLANE_BYTES) }; + let mut source = source(aux).unwrap(); + let sink = sink(alias).unwrap(); + assert_eq!(sink.state(), Some(FsWatchState::default())); + source.state(|s| state::arm(s, 1, 7)); + assert!(!source.commit(), "nobody parked: no notice owed"); + let seen = sink.seq(); + assert!(sink.park(seen)); + source.state(|s| state::changed(s, 1)); + assert!(source.commit(), "a parked client is owed its notice"); + assert_eq!(sink.state().unwrap().changes[1], 1); + } + + #[test] + fn a_window_too_small_for_the_plane_is_refused() { + let mut short = vec![0u8; PLANE_AT + 64 + 64]; + let at = short.as_ptr().align_offset(64); + let window = &mut short[at..at + PLANE_AT]; + assert_eq!(init_pair_window(window, REQ_CAP, REP_CAP), Err(RingError::TooSmall)); + } + + #[test] + fn a_cursor_reads_only_its_own_arming() { + let mut cursor = Cursor::new(2, 9).unwrap(); + let mut s = FsWatchState::default(); + // The slot's previous watch left counts: not this cursor's news. + state::arm(&mut s, 2, 8); + state::changed(&mut s, 2); + assert_eq!(cursor.observe(&s), Seen::Same); + state::arm(&mut s, 2, 9); + assert_eq!(cursor.observe(&s), Seen::Same); + state::changed(&mut s, 2); + state::changed(&mut s, 2); + assert_eq!(cursor.observe(&s), Seen::Changed, "two changes coalesce"); + assert_eq!(cursor.observe(&s), Seen::Same); + state::end(&mut s, 2); + assert_eq!(cursor.observe(&s), Seen::Ended); + assert_eq!(cursor.observe(&s), Seen::Same, "an ending is reported once"); + assert!(Cursor::new(FS_WATCH_SLOTS as u8, 1).is_none()); + assert!(Cursor::new(0, 0).is_none()); + } +} diff --git a/native/crates/dagfs/src/wire.rs b/native/crates/dagfs/src/wire.rs index 638461e..4cdae6f 100644 --- a/native/crates/dagfs/src/wire.rs +++ b/native/crates/dagfs/src/wire.rs @@ -23,6 +23,8 @@ //! | `OpenAt` | parent | — | — | — | `name` | //! | `OpenGrant` | — | grant id | — | — | — | //! | `Close`/`Stat`/`Sync` | object | — | — | — | — | +//! | `Watch` | directory | — | — | tag (slot in `kind`) | — | +//! | `Unwatch` | — | — | — | — (slot in `kind`) | — | //! | `List` | directory | — | — | cursor | — | //! | `Parents` | object | — | — | cursor | — | //! | `Locate` | anchor | stable identity | — | — | — | @@ -448,6 +450,16 @@ pub fn encode_req(req: &FsRequest) -> FsReqWire { w.op = FsReqWire::OP_SYNC; w.handle = U64::new(handle.to_bits()); } + FsRequest::Watch { handle, slot, tag } => { + w.op = FsReqWire::OP_WATCH; + w.handle = U64::new(handle.to_bits()); + w.kind = slot; + w.length = U32::new(tag); + } + FsRequest::Unwatch { slot } => { + w.op = FsReqWire::OP_UNWATCH; + w.kind = slot; + } FsRequest::DeferCommits { hold } => { w.op = FsReqWire::OP_DEFER_COMMITS; w.kind = u8::from(hold); @@ -713,6 +725,8 @@ pub fn decode_req(w: &FsReqWire) -> Option { | FsReqWire::OP_DEFER_COMMITS | FsReqWire::OP_SET_SYSTEM_SEED | FsReqWire::OP_STOR_MAINTAIN + | FsReqWire::OP_WATCH + | FsReqWire::OP_UNWATCH ) && w.kind != 0 { return None; @@ -914,6 +928,35 @@ pub fn decode_req(w: &FsReqWire) -> Option { } FsReqWire::OP_REVOKE => FsRequest::Revoke { grant }, FsReqWire::OP_SYNC => FsRequest::Sync { handle }, + FsReqWire::OP_WATCH => { + // `kind` is the slot and `length` the tag; nothing else rides. A + // slot past the plane or a zero tag (the plane's "disarmed") is a + // framing mismatch, not a request to guess at. + if usize::from(w.kind) >= kore_abi::svc::FS_WATCH_SLOTS + || w.length.get() == 0 + || w.window_offset.get() != 0 + || w.handle2.get() != 0 + || w.offset.get() != 0 + || w.expected_revision.get() != 0 + || !names_are_empty(w) + { + return None; + } + FsRequest::Watch { + handle, + slot: w.kind, + tag: w.length.get(), + } + } + FsReqWire::OP_UNWATCH => { + if usize::from(w.kind) >= kore_abi::svc::FS_WATCH_SLOTS + || !control_payload_is_empty(w) + || w.window_offset.get() != 0 + { + return None; + } + FsRequest::Unwatch { slot: w.kind } + } FsReqWire::OP_DEFER_COMMITS => { // `kind` is the whole payload: a deferral names no object, and a // value outside the two-state vocabulary is a framing mismatch. @@ -1737,6 +1780,8 @@ pub fn decode_rep_for(request: &FsRequest, w: &FsRepWire) -> Option<(FsResponse, | FsRequest::DetachEdge { .. } | FsRequest::Revoke { .. } | FsRequest::Sync { .. } + | FsRequest::Watch { .. } + | FsRequest::Unwatch { .. } | FsRequest::BeginReplace { .. } | FsRequest::WriteReplacement { .. } | FsRequest::AbortReplace { .. } @@ -1876,6 +1921,8 @@ pub fn reply_matches(request: &FsRequest, response: &FsResponse) -> bool { | FsRequest::DetachEdge { .. } | FsRequest::Revoke { .. } | FsRequest::Sync { .. } + | FsRequest::Watch { .. } + | FsRequest::Unwatch { .. } | FsRequest::BeginReplace { .. } | FsRequest::WriteReplacement { .. } | FsRequest::AbortReplace { .. } @@ -2020,6 +2067,66 @@ mod tests { assert_eq!(decode_req(&stray), None); } + /// A watch carries its directory, its slot and its tag and nothing else; + /// an unwatch carries its slot alone. Both answer `Done`, and a slot past + /// the plane, the zero tag, or any stray operand is refused. + #[test] + fn watch_verbs_round_trip_and_refuse_stray_operands() { + let slots = kore_abi::svc::FS_WATCH_SLOTS as u8; + for (slot, tag) in [(0, 1), (slots - 1, u32::MAX)] { + let watch = FsRequest::Watch { + handle: Handle::from_bits(0x1_0000_0003), + slot, + tag, + }; + let unwatch = FsRequest::Unwatch { slot }; + for request in [watch, unwatch] { + assert!(!request.needs_window()); + assert_eq!(decode_req(&encode_req(&request)), Some(request)); + assert!(reply_matches(&request, &FsResponse::Done)); + assert_eq!( + decode_rep_for(&request, &encode_rep(&FsResponse::Done, 0)), + Some((FsResponse::Done, 0)) + ); + } + } + let wire = encode_req(&FsRequest::Watch { + handle: Handle::from_bits(9), + slot: 1, + tag: 5, + }); + let strays: [fn(&mut FsReqWire); 8] = [ + |w| w.kind = kore_abi::svc::FS_WATCH_SLOTS as u8, + |w| w.length = U32::new(0), + |w| w.window_offset = U32::new(4), + |w| w.handle2 = U64::new(1), + |w| w.offset = U64::new(1), + |w| w.expected_revision = U64::new(1), + |w| w.rights = 1, + |w| { + w.name[0] = b'x'; + w.name_len = 1; + }, + ]; + for stray in strays { + let mut bad = wire; + stray(&mut bad); + assert_eq!(decode_req(&bad), None); + } + let wire = encode_req(&FsRequest::Unwatch { slot: 2 }); + let strays: [fn(&mut FsReqWire); 4] = [ + |w| w.kind = kore_abi::svc::FS_WATCH_SLOTS as u8, + |w| w.handle = U64::new(1), + |w| w.length = U32::new(1), + |w| w.window_offset = U32::new(1), + ]; + for stray in strays { + let mut bad = wire; + stray(&mut bad); + assert_eq!(decode_req(&bad), None); + } + } + /// Every maintenance action survives the wire; only a forced refresh /// carries operands, and an operand on any other action — or a range /// past `u32` — is refused. diff --git a/native/crates/ld/tests/built_images.rs b/native/crates/ld/tests/built_images.rs index f196515..0bf03c1 100644 --- a/native/crates/ld/tests/built_images.rs +++ b/native/crates/ld/tests/built_images.rs @@ -339,6 +339,7 @@ fn check_variant(root: &Path, variant: &str, required: bool) -> us ("sleep", ImageKind::Executable), ("status", ImageKind::NeedsLibrary("libpci_names.so")), ("lspci", ImageKind::NeedsLibrary("libpci_names.so")), + ("ioctl", ImageKind::NeedsLibrary("libpci_names.so")), ("help", ImageKind::Executable), ("window", ImageKind::Executable), ("clear", ImageKind::Executable), diff --git a/native/crates/rt/src/cap/mod.rs b/native/crates/rt/src/cap/mod.rs index 2266846..e5e1d39 100644 --- a/native/crates/rt/src/cap/mod.rs +++ b/native/crates/rt/src/cap/mod.rs @@ -1754,19 +1754,20 @@ pub enum QuiesceFault { Unenforced, } -/// Why a writable lease did not become an executable successor. +/// Why a consuming lease transition did not yield its successor: a +/// finalization, the return to writable, or a freeze. /// -/// Finalization has one abort door: a refusal before commit returns the exact -/// affine source. Once the kernel has retired that source and begun clearing +/// Each has one abort door: a refusal before commit returns the exact affine +/// source. Once the kernel has retired that source and begun clearing /// aliases, rollback is impossible; a later containment or publication /// failure is therefore reported as [`Consumed`](Self::Consumed), never as a -/// writable lease that no longer exists. +/// lease that no longer exists. #[must_use] #[derive(Debug)] pub enum FinalizeExecError { - /// The session did not commit; the writable source is unchanged. + /// The session did not commit; the source is unchanged. Refused(T), - /// The source was consumed, but no executable successor was published. + /// The source was consumed, but no successor was published. Consumed, } @@ -2854,17 +2855,6 @@ pub struct StashedLease { owned: OwnedSlot, } -impl StashedLease { - /// Adopt an already-activated owned slot the kernel just published as a - /// writable lease — the executable → writable return's landing - /// (`res/IMAGE.md` §6a). Crate-internal: the caller must have proved - /// the record's shape first (`writable_result_state`). - #[cfg(target_os = "none")] - pub(crate) fn from_owned(owned: OwnedSlot) -> Self { - Self { owned } - } -} - /// Transferable memory custody: an allocation lease or a scoped mapping loan. /// A route never acquires the allocation/release authority of a MemLease. #[derive(Debug)] diff --git a/native/crates/rt/src/cap/verbs.rs b/native/crates/rt/src/cap/verbs.rs index d6603ef..f07b911 100644 --- a/native/crates/rt/src/cap/verbs.rs +++ b/native/crates/rt/src/cap/verbs.rs @@ -4178,6 +4178,102 @@ mod metal { None } + /// Move a capability a client delivered into a fixed landing slot — a + /// service's convey bank, whose post must stand again for the next + /// request — into a fresh runtime-owned slot, so the service can keep it + /// past the request that carried it. + /// + /// The move rides a channel minted from `space` for it alone: the + /// landing is exact-moved through that channel's send into a post over + /// the new owner, and the channel is then retired. An exact move keeps + /// the record's rights and lineage, so retiring the private channel + /// reaches nothing it carried. Only the kinds the kernel moves exactly + /// relocate: memory leases and views, fabric routes, channel endpoints. + #[cfg(target_os = "none")] + pub fn relocate_landed( + space: &ChannelSpaceToken, + landed: SlotRef, + ) -> Result { + use crate::session::SessionBuilder; + use crate::transport::{Transport, TrapTransport}; + use kore_abi::session::Status; + + fn committed(session: &SessionBuilder) -> bool { + TrapTransport::new() + .commit_now(session) + .is_ok_and(|done| done.status() == Status::Ok && done.failing_index() == 0) + } + + let crate::cap::ChannelPair { root, send, recv } = + mint_channel(space).map_err(|_| RelocateError::Unmoved)?; + let mut owner = crate::cap::ReturnedCapability::reserve(); + let mut armed = false; + let mut sent = false; + if let Some(owner) = owner.as_mut() { + let mut post = SessionBuilder::new(); + if post.post_recv(recv.slot(), owner.target()).is_ok() && committed(&post) { + owner.arm_post(); + armed = true; + let mut moving = SessionBuilder::new(); + sent = moving.send_exact(send.slot(), landed).is_ok() && committed(&moving); + } + } + // The root is the private channel's only pivot: revoking it ends + // both endpoints, and with them any post still naming the owner. + let closed = match root.try_retire() { + Ok(outcome) => outcome.is_closed(), + Err(root) => { + core::mem::forget(root); + false + } + }; + let _ = send.try_retire(); + let _ = recv.try_retire(); + let Some(mut owner) = owner else { + return Err(RelocateError::Unmoved); + }; + if sent { + if matches!( + take_delivery(owner.target()), + Some(crate::cap::RawDelivery::Cap) + ) { + owner.activate(); + return Ok(owner); + } + // A committed exact move into a standing post lands, so this is + // a record the runtime cannot classify: keep its name out of the + // allocator for the task's life rather than reopen it. + core::mem::forget(owner); + return Err(RelocateError::Lost); + } + if armed && !closed { + // The post may still stand over this name. + core::mem::forget(owner); + } else if armed { + owner.disarm_post(); + } + Err(RelocateError::Unmoved) + } + + #[cfg(not(target_os = "none"))] + pub fn relocate_landed( + _space: &ChannelSpaceToken, + _landed: SlotRef, + ) -> Result { + Err(RelocateError::Unmoved) + } + + /// Why [`relocate_landed`] produced no owner. + #[derive(Clone, Copy, Debug, PartialEq, Eq)] + pub enum RelocateError { + /// Nothing moved: the delivery is still in its landing, for the + /// caller's own custody to keep or retire. + Unmoved, + /// The move committed but its result could not be read back; the + /// landing is vacant and the would-be owner is quarantined. + Lost, + } + /// **The metal take-view** (`res/USERLAND.md` §5.6; Phase-A checkpoint /// C) — read a delivered `LeasedRegion` view record off the R/O /// cap-slot alias at [`SLOTS`](kore_abi::task::geom::SLOTS) alias and return @@ -5226,7 +5322,7 @@ mod metal { } ExecFinalizeCompletion::Replaced => { code.release_after_proven_move(); - Ok(StashedLease::from_owned(output)) + Ok(StashedLease::new(output)) } } } @@ -5238,6 +5334,78 @@ mod metal { Err(FinalizeExecError::Refused(code)) } + /// Freeze a runtime-owned writable lease (`res/IMAGE.md` §6b): consume + /// it with every alias and descendant capability of it, and return the + /// frozen successor — the same window and rights, which no lessor may + /// write again while it lives. This is a registrar's exclusive intake: + /// what it then validates is what every later reader of the window reads. + /// + /// Affine slot ownership is preserved on either outcome, as + /// [`finalize_stashed_exec`] preserves it: a refusal returns the exact + /// source, and a consumed source reports + /// [`Consumed`](FinalizeExecError::Consumed) with no authority back. + #[cfg(target_os = "none")] + pub fn freeze_stashed( + mut lease: StashedLease, + ) -> Result> { + let mut output = match crate::cap::OwnedSlot::reserve() { + Some(output) => output, + None => return Err(FinalizeExecError::Refused(lease)), + }; + let Some(source) = lease.slot() else { + return Err(FinalizeExecError::Refused(lease)); + }; + match freeze_slot(source, &mut output) { + ExecFinalizeCompletion::Refused => Err(FinalizeExecError::Refused(lease)), + ExecFinalizeCompletion::Consumed => { + lease.owned.release_after_proven_move(); + Err(FinalizeExecError::Consumed) + } + ExecFinalizeCompletion::Replaced => { + // The committed kernel transition cleared `source`; only its + // dynamic allocator reservation remains to release. + lease.owned.release_after_proven_move(); + Ok(StashedLease::new(output)) + } + } + } + + /// Off metal there is no freeze session to issue. + #[cfg(not(target_os = "none"))] + pub fn freeze_stashed( + lease: StashedLease, + ) -> Result> { + Err(FinalizeExecError::Refused(lease)) + } + + /// [`freeze_stashed`] for a lease a caller delivered into a fixed landing + /// slot (a service's convey bank): the frozen successor moves into a + /// fresh runtime-owned slot, and the landing is left vacant for its post + /// to stand again. A registrar that validates bytes a client wrote takes + /// them this way, so no alias the client kept can write them afterwards. + /// + /// `Refused` leaves the delivery in `slot`, untouched, for the caller's + /// own custody to retire; `Consumed` reports a vacant `slot` with no + /// successor. + #[cfg(target_os = "none")] + pub fn freeze_landed(slot: SlotRef) -> Result> { + let mut output = match crate::cap::OwnedSlot::reserve() { + Some(output) => output, + None => return Err(FinalizeExecError::Refused(())), + }; + match freeze_slot(slot, &mut output) { + ExecFinalizeCompletion::Refused => Err(FinalizeExecError::Refused(())), + ExecFinalizeCompletion::Consumed => Err(FinalizeExecError::Consumed), + ExecFinalizeCompletion::Replaced => Ok(StashedLease::new(output)), + } + } + + /// Off metal there is no freeze session to issue. + #[cfg(not(target_os = "none"))] + pub fn freeze_landed(_slot: SlotRef) -> Result> { + Err(FinalizeExecError::Refused(())) + } + #[cfg(target_os = "none")] #[derive(Clone, Copy, Debug, PartialEq, Eq)] enum ExecFinalizeCompletion { @@ -5274,10 +5442,10 @@ mod metal { } } - /// The writable successor's shape proof — [`exec_result_state`]'s twin. - /// The rights differ (`MODIFY` is back), and checking the exact set is - /// what makes a wrong-shaped record `Malformed` rather than silently - /// accepted. + /// The full-lifecycle successor's shape proof — a return's or a + /// freeze's, [`exec_result_state`]'s twin. The rights differ (`MODIFY` + /// is there), and checking the exact set is what makes a wrong-shaped + /// record `Malformed` rather than silently accepted. #[cfg(target_os = "none")] fn writable_result_state(output: SlotRef) -> ExecResultState { use kore_abi::cap::{CapRecord, ResourceKind}; @@ -5331,6 +5499,34 @@ mod metal { } } + /// Freeze `source` into `output`, mirroring [`return_writable_slot`]: + /// the successor carries the full lifecycle grant, so the same shape + /// proof classifies it. + #[cfg(target_os = "none")] + fn freeze_slot(source: SlotRef, output: &mut crate::cap::OwnedSlot) -> ExecFinalizeCompletion { + use crate::session::SessionBuilder; + use crate::transport::{Transport, TrapTransport}; + + let mut b = SessionBuilder::new(); + if b.freeze(source, output.slot()).is_err() { + return ExecFinalizeCompletion::Refused; + } + let Ok(committed) = TrapTransport::new().commit_now(&b) else { + return ExecFinalizeCompletion::Refused; + }; + if committed.status() != kore_abi::session::Status::Ok || committed.failing_index() != 0 { + return ExecFinalizeCompletion::Refused; + } + match writable_result_state(output.slot()) { + ExecResultState::Replaced => ExecFinalizeCompletion::Replaced, + ExecResultState::Vacant => ExecFinalizeCompletion::Consumed, + ExecResultState::Malformed => { + output.quarantine_unknown_publish(); + ExecFinalizeCompletion::Consumed + } + } + } + #[cfg(target_os = "none")] fn finalize_exec_slot( source: SlotRef, diff --git a/native/crates/rt/src/crash.rs b/native/crates/rt/src/crash.rs index b06b58e..f61c060 100644 --- a/native/crates/rt/src/crash.rs +++ b/native/crates/rt/src/crash.rs @@ -112,10 +112,23 @@ pub fn register( /// Returning executables share the same recursion-safe diagnostic path. pub fn panic(info: &core::panic::PanicInfo<'_>) -> ! { - if crate::entry::begin_panic() { + report(info); + crate::thread::exit(crate::ExitCode::PANIC) +} + +/// [`panic`]'s report without its exit: publish this thread's panic record +/// once (a panic raised while reporting reports nothing more), and return. +/// +/// For a panic handler that must act between the report and the thread's +/// end — a host that ends the whole process on a worker's panic tells the +/// host only once the record is published, so the host's barrier cannot cut +/// the report short. `true` when this call published the record. +pub fn report(info: &core::panic::PanicInfo<'_>) -> bool { + let first = crate::entry::begin_panic(); + if first { report_panic_info(info); } - crate::thread::exit(crate::ExitCode::PANIC) + first } /// Best-effort software report. The entry panic handler has claimed recursion diff --git a/native/crates/rt/src/entry.rs b/native/crates/rt/src/entry.rs index aff740d..00b9b80 100644 --- a/native/crates/rt/src/entry.rs +++ b/native/crates/rt/src/entry.rs @@ -176,6 +176,28 @@ const PROCESS_STDIO_ATTACHMENT: usize = 1 << 1; #[cfg(any(test, target_os = "none"))] const PROCESS_STDIO_HOST_ATTACHMENT: usize = 1 << 2; +/// The free part of a task's link window ([`Rt::link_free_span`]). +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct LinkSpan { + /// The task's whole link window. + pub base: u64, + /// Its end below the module page. + pub limit: u64, + /// The first free address: above everything the startup linker placed. + pub free: u64, +} + +/// Why a task has no free link span ([`Rt::link_free_span`]). +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum LinkSpanError { + /// The task has no `geom::LINK` window. + NoWindow, + /// The window has no module page. + NoModulePage, + /// Nothing free above what the startup linker placed. + Full, +} + /// Atomically spend one runtime attachment name. /// /// This is deliberately a bit in the process-root @@ -255,6 +277,48 @@ impl Rt { None } + /// **The free part of this task's link window** (`geom::LINK`): where a + /// program may place mappings of its own after startup — a Wasmtime + /// engine's guest mappings, a C program's `mmap`. + /// + /// The floor is the larger of the module page's two marks: the recorded + /// segments' top ([`ModulePage::high_water`]) and how far the startup + /// placement reached ([`ModulePage::placed_high_water`]: the copied + /// template views and the TLS block, which no record names). Neither + /// alone is the floor. The limit is the module page itself, the + /// window's last page. + /// + /// This names a span and claims nothing. **One placer per window**: a + /// program places everything it maps there from one cursor + /// (`kore_ld::place::SharedPlacer`), and one that also links native + /// modules late (`kore-late`) partitions the span between the two + /// itself, since two cursors over one window hand out each other's + /// addresses. + /// + /// [`ModulePage::high_water`]: kore_abi::image::ModulePage::high_water + /// [`ModulePage::placed_high_water`]: kore_abi::image::ModulePage::placed_high_water + pub fn link_free_span(&self) -> Result { + let (base, len) = self + .geom() + .and_then(|geom| geom.window(kore_abi::task::geom::LINK)) + .ok_or(LinkSpanError::NoWindow)?; + let limit = + kore_abi::image::module_page_va(base, len).ok_or(LinkSpanError::NoModulePage)?; + let page = self.modules(); + let free = page + .and_then(|page| page.high_water()) + .unwrap_or(base) + .max( + page.and_then(|page| page.placed_high_water(limit)) + .unwrap_or(base), + ) + .max(base); + if free >= limit { + return Err(LinkSpanError::Full); + } + Ok(LinkSpan { base, limit, free }) + } + /// The `.kore.geom` window table, if the founder stamped it — where the /// task learns its founder-mapped `(va, len)` windows (scratch, slots, …) /// instead of hardcoded constants (`res/USERLAND.md` §4). `None` for a @@ -504,7 +568,7 @@ const fn report_slot_install(current_p1: usize, requested: usize) -> ReportSlotI #[cfg(target_os = "none")] pub(crate) mod geometry { - use core::sync::atomic::{AtomicU64, AtomicUsize, Ordering}; + use core::sync::atomic::{AtomicI32, AtomicU32, AtomicU64, AtomicUsize, Ordering}; use kore_abi::task::{GeomBlock, geom}; @@ -631,6 +695,31 @@ pub(crate) mod geometry { finalizer_state: AtomicUsize, finalizers: [AtomicUsize; 32], exit_started: AtomicUsize, + /// This thread's C `errno` ([`c_errno`]): ISO C gives `errno` + /// thread storage duration, and this block is the runtime's + /// per-thread home, so the word lives here rather than in an ELF + /// TLS block — which would give every C-carrying image a TLS + /// segment, and with it a TCB migration and a template every spawn + /// must place (`kore-late` refuses such a host outright). Zero on + /// a fresh page, as C requires at thread start; deliberately *not* + /// inherited by a spawned thread, whose block is its own. Only the + /// owning thread reads or writes it. + c_errno: AtomicI32, + /// Sends into peers' doorbells this thread came to hold after its + /// founding ([`futex::adopt_peer`](crate::futex)), indexed by the + /// peer's band, each its arena slot plus one (`0` = none). Arena + /// names are this thread's alone, so they live here and a band a + /// later thread reuses starts with none. Only the owning thread + /// reads or writes them. + late_kicks: [AtomicU32; crate::thread::MAX_THREADS as usize], + /// The [`futex::Hub`](crate::futex) this thread relays an + /// undeliverable wake through, as an address (`0` = none). Set once. + futex_hub: AtomicUsize, + /// What [`thread::exit`](crate::thread::exit) runs before it commits + /// ([`set_exit_hook`]): a `fn(usize)` as an address (`0` = none), + /// and its argument. Taken by the exit, so it runs once. + exit_call: AtomicUsize, + exit_argument: AtomicUsize, } // The block must fit the founder's one-page window. @@ -732,6 +821,75 @@ pub(crate) mod geometry { block().panicking.swap(1, Ordering::AcqRel) == 0 } + /// This thread's C `errno` word (the block's `c_errno` field). + /// + /// Lawful wherever [`block`] is: C code runs only on an anchored + /// thread, since anchoring is crt0's first act and a spawned thread's + /// trampoline's first act. + pub(crate) fn c_errno() -> *mut i32 { + block().c_errno.as_ptr() + } + + /// The send this thread adopted into band `peer`'s doorbell, if any. + pub(crate) fn late_kick(peer: usize) -> Option { + let slot = block().late_kicks.get(peer)?.load(Ordering::Relaxed); + let index = slot.checked_sub(1)?; + Some(kore_abi::session::SlotRef::from_wire(index)) + } + + /// Record (`Some`) or withdraw (`None`) this thread's adopted send into + /// band `peer`'s doorbell. `false` when `peer` is outside the band. + pub(crate) fn set_late_kick(peer: usize, send: Option) -> bool { + let Some(cell) = block().late_kicks.get(peer) else { + return false; + }; + let word = match send { + Some(slot) => match slot.index().checked_add(1) { + Some(word) => word, + None => return false, + }, + None => 0, + }; + cell.store(word, Ordering::Relaxed); + true + } + + /// The futex hub this thread relays through, as an address (`0` = none). + pub(crate) fn futex_hub() -> usize { + block().futex_hub.load(Ordering::Relaxed) + } + + /// Adopt the futex hub at `addr`; the first adoption stands. + pub(crate) fn set_futex_hub(addr: usize) -> bool { + addr != 0 + && block() + .futex_hub + .compare_exchange(0, addr, Ordering::Relaxed, Ordering::Relaxed) + .is_ok() + } + + /// Have [`thread::exit`](crate::thread::exit) run `call(argument)` on + /// this thread before it commits, replacing any hook set before. + pub(crate) fn set_exit_hook(call: fn(usize), argument: usize) { + let b = block(); + b.exit_argument.store(argument, Ordering::Relaxed); + b.exit_call.store(call as usize, Ordering::Relaxed); + } + + /// Take this thread's exit hook, leaving none: a hook that itself ends + /// the thread runs once. + pub(crate) fn take_exit_hook() -> Option<(fn(usize), usize)> { + let b = block(); + let call = b.exit_call.swap(0, Ordering::Relaxed); + if call == 0 { + return None; + } + // SAFETY: only `set_exit_hook` stores a nonzero `exit_call`, and it + // stores a `fn(usize)` cast to its address. + let call = unsafe { core::mem::transmute::(call) }; + Some((call, b.exit_argument.load(Ordering::Relaxed))) + } + /// Where this thread's block lives — the value the thread pointer holds, /// recovered the same way every other accessor recovers it. pub(crate) fn block_va() -> usize { diff --git a/native/crates/rt/src/futex.rs b/native/crates/rt/src/futex.rs index 04fe2d6..bebb188 100644 --- a/native/crates/rt/src/futex.rs +++ b/native/crates/rt/src/futex.rs @@ -70,15 +70,26 @@ //! `i`'s send end is looked up at `role::DOORBELL@(2 + i)`, and index 2 is //! the self-kick a founder already carves — so a thread endowed with it can //! wake **thread 0** with nothing added, which is the endowment the input -//! service's relay thread already takes. Higher indices are the peer mesh, -//! and `thread::spawn_with` does not provision one yet: [`wake`] returns -//! `false` for a peer this thread holds no send end for, rather than -//! reporting a kick it did not deliver. +//! service's relay thread already takes. Higher indices are the peer mesh; +//! [`wake`] returns `false` for a peer this thread holds no send end for, +//! rather than reporting a kick it did not deliver. //! -//! That asymmetry is a provisioning gap, not a design one — filling it is -//! minting `MAX_THREADS` doorbell pairs in the spawning thread and endowing -//! each child with the ends, which needs a `role::CHANNEL_SPACE` grant in the -//! task that spawns. Nothing in the API changes when it lands. +//! The mesh is provisioned by whoever founds the threads, from a +//! `role::CHANNEL_SPACE` grant: the star (`res/SPMD.md` §3) for an SPMD +//! region, and the [`thread_controller`](crate::thread_controller) for the +//! threads it owns. A founding stamps the sends it can name in advance; a +//! send a thread comes to hold *after* its founding — the channel a +//! controller mints for each child it founds, or the one a starter mints for +//! the controller it founds — is **adopted** ([`adopt_peer`]) into the +//! thread's own runtime block, and [`wake`] finds it there exactly as it +//! finds a stamped one. +//! +//! A waiter that no send this thread holds can reach is handed to a +//! **relay**: the star's hub (one notice on `DOORBELL@`[`RELAY`], answered by +//! a kick to every peer), or a [`Hub`] this thread adopted — a hub that is +//! itself a waiter, woken through its own word, which kicks only the threads +//! with waiters registered ([`sweep`]). Nothing in the wait/wake API changes +//! with either. //! //! **For a word in a shared window the gap is closed** by the kernel //! address-wait (`res/EVENT.md` §6). A word that @@ -349,6 +360,15 @@ pub(crate) const PEER_BASE: u16 = 2; /// band peers so it can never collide with one. pub(crate) const RELAY: u16 = PEER_BASE + MAX_THREADS as u16; +/// A thread's **own kick**: a send into its own doorbell that it may derive +/// from — the authority it hands a thread it founds so that thread can wake +/// it ([`thread::own_kick`](crate::thread)). One past [`RELAY`], so neither +/// the peer arithmetic nor the relay can name it: a thread never kicks itself +/// through the mesh. Thread 0's own kick predates this index — it is the +/// founder-carved self-kick at [`PEER_BASE`], its own peer-0 entry. +#[cfg_attr(not(target_os = "none"), allow(dead_code))] +pub(crate) const OWN_KICK: u16 = RELAY + 1; + /// Wait until another thread wakes this one, or `timeout` elapses. /// /// Parks **only if** `word` still holds `expected` at the moment the wait is @@ -825,6 +845,13 @@ pub(crate) fn notify_resident_completion() -> bool { /// parked is still woken correctly — its post already stands, so the notice /// resolves the park it is about to enter. pub fn wake(word: &AtomicU32) -> bool { + wake_one(word, true) +} + +/// [`wake`]'s body. `relay` says whether a waiter this thread cannot reach +/// is handed on ([`relay_missed`]); a [`Hub`]'s own signal is not, since the +/// hub is the relay. +fn wake_one(word: &AtomicU32, relay: bool) -> bool { let addr = address_of(word); let route = crate::cap::mapped::locate(word as *const AtomicU32 as *const u8); let mut unreachable_waiter = false; @@ -852,12 +879,12 @@ pub fn wake(word: &AtomicU32) -> bool { } } // A registered waiter this caller holds no peer send for is the - // hub's to reach (`res/SPMD.md` §3 — the star): one relay notice, - // counted as a wake because a notice is now in flight toward the - // waiter. No hub wired → today's honest miss. + // hub's to reach (`res/SPMD.md` §3 — the star, or this thread's own + // `Hub`): one relay, counted as a wake because a wake is now in + // flight toward the waiter. No hub wired → today's honest miss. if unreachable_waiter { STAT_MISSED.fetch_add(1, Ordering::Relaxed); - if imp::relay() { + if relay && relay_missed() { STAT_RELAYS.fetch_add(1, Ordering::Relaxed); return true; } @@ -865,6 +892,16 @@ pub fn wake(word: &AtomicU32) -> bool { false } +/// Hand the waiters this thread could not reach to the thread that can: +/// its adopted [`Hub`], through the hub's own word, or else the star's hub +/// through one raw notice on `DOORBELL@`[`RELAY`]. `true` when either went. +fn relay_missed() -> bool { + match imp::hub() { + Some(hub) => hub.relay(), + None => imp::relay(), + } +} + /// Wake **every** thread waiting on `word`; returns how many were kicked. /// /// The count is of notices successfully posted, not of threads that will @@ -904,17 +941,204 @@ pub fn wake_all(word: &AtomicU32) -> usize { missed = true; } } - // The relay hop (the star, `res/SPMD.md` §3): every waiter this - // caller could not kick is the hub's job — one notice covers them - // all, counted once. With the intra-chunk kick mesh this is the - // exception path (cross-chunk waiters), not the release path. - if missed && imp::relay() { + // The relay hop (the star, `res/SPMD.md` §3, or this thread's `Hub`): + // every waiter this caller could not kick is the hub's job — one relay + // covers them all, counted once. With the intra-chunk kick mesh this + // is the exception path (cross-chunk waiters), not the release path. + if missed && relay_missed() { STAT_RELAYS.fetch_add(1, Ordering::Relaxed); woken += 1; } woken } +/// A **relay hub that is also a waiter**: the thread that founds a group of +/// threads and holds a send into each of their doorbells — the +/// [`thread_controller`](crate::thread_controller) — parked on its +/// [`word`](Self::word) like any other waiter. +/// +/// A thread that adopted the hub ([`adopt_hub`]) relays a wake it cannot +/// deliver by counting it in [`relays`](Self::relays), then [`signal`]ing +/// the hub: an ordinary store-then-wake of the hub's word, with every +/// property [`wake`] has. The hub reads `relays` after each wake and, when +/// it moved, [`sweep`]s. +/// +/// Why a count and not the star's raw notice: this hub is woken for work of +/// its own, and a notice says nothing about why it came. A hub that took +/// every notice for a relay would kick every parked thread on every +/// unrelated event; one that took none would lose a relay whose notice +/// coalesced with another on its doorbell. The count is bumped before the +/// word, so a hub that sees the word move sees the relay behind it. +/// +/// [`signal`]: Self::signal +#[repr(align(128))] +pub(crate) struct Hub { + word: AtomicU32, + relays: AtomicU32, +} + +impl Hub { + pub(crate) const fn new() -> Self { + Self { + word: AtomicU32::new(0), + relays: AtomicU32::new(0), + } + } + + /// The word the hub thread waits on. + #[cfg_attr(not(target_os = "none"), allow(dead_code))] + pub(crate) fn word(&self) -> &AtomicU32 { + &self.word + } + + /// Relays owed so far, wrapping: the hub sweeps whenever this moved + /// since its last sweep. + #[cfg_attr(not(target_os = "none"), allow(dead_code))] + pub(crate) fn relays(&self) -> u32 { + self.relays.load(Ordering::SeqCst) + } + + /// Tell the hub something changed — store, then wake, the order this + /// module requires. Delivered directly or not at all: the hub has no + /// relay but itself. `true` when a kick reached it. + pub(crate) fn signal(&self) -> bool { + self.word.fetch_add(1, Ordering::SeqCst); + wake_one(&self.word, false) + } + + /// One relay: counted before the word moves. + fn relay(&self) -> bool { + self.relays.fetch_add(1, Ordering::SeqCst); + self.signal() + } +} + +/// Relay through `hub` from now on: a wake this thread cannot deliver goes +/// to it ([`Hub`]). The first adoption stands — `false` when this thread +/// already relays through a hub, or has no runtime block to record it in. +pub(crate) fn adopt_hub(hub: &'static Hub) -> bool { + imp::adopt_hub(hub) +} + +/// Record that this thread holds `send`, a send into band `peer`'s +/// doorbell its founding did not stamp — the channel a controller mints for +/// each child it founds, or a starter for the controller it founds. From +/// here on [`wake`] reaches `peer` directly. `false` when `peer` is outside +/// the band or this thread has no runtime block. +/// +/// The record is this thread's alone — it names a slot in its own arena — so +/// it lives in its runtime block, not beside [`WAITING_ON`]: a band reused +/// by a later thread starts from a zeroed block and inherits nothing. +#[cfg_attr(not(target_os = "none"), allow(dead_code))] +pub(crate) fn adopt_peer(peer: u32, send: kore_abi::session::SlotRef) -> bool { + imp::adopt_peer(peer as usize, send) +} + +/// Withdraw `peer`'s adopted send. Called before the send is retired, so no +/// wake presents a name the arena is about to reissue. +#[cfg_attr(not(target_os = "none"), allow(dead_code))] +pub(crate) fn forget_peer(peer: u32) { + imp::forget_peer(peer as usize); +} + +/// Whether this thread holds a send into band `peer`'s doorbell — stamped +/// at its founding or adopted since. +pub(crate) fn reaches(peer: u32) -> bool { + imp::reaches(peer as usize) +} + +/// The hub's answer to a relay ([`Hub`]): kick every thread this one can +/// reach that has a doorbell waiter registered, itself excepted, and return +/// how many were kicked. +/// +/// A relay does not say which word it was for, so every such thread is a +/// candidate and each kick is a wake its target re-checks — spurious for one +/// whose word did not move, which every waiter tolerates by contract, and +/// paid only when a relay was owed. A word-parked waiter is not kicked: the +/// kernel's address-wait, not its doorbell, holds it, and its slice bounds +/// a relay it misses. +#[cfg_attr(not(target_os = "none"), allow(dead_code))] +pub(crate) fn sweep() -> usize { + let Some(me) = current_thread() else { + return 0; + }; + imp::sweep(me) +} + +/// [`sweep`]'s selection over the registry, with the kick injected so the +/// host suite can hold it to account. +#[cfg_attr(not(target_os = "none"), allow(dead_code))] +fn sweep_with(me: usize, mut kick: impl FnMut(usize) -> bool) -> usize { + let mut kicked = 0; + for thread in 0..MAX_THREADS as usize { + if thread != me && waits_on_bell(thread) && kick(thread) { + STAT_KICKS.fetch_add(1, Ordering::Relaxed); + kicked += 1; + } + } + kicked +} + +/// Whether `thread` has any waiter a kick reaches — a doorbell-mode entry. +#[cfg_attr(not(target_os = "none"), allow(dead_code))] +fn waits_on_bell(thread: usize) -> bool { + WAITING_ON[thread].iter().any(|cell| { + let entry = cell.load(Ordering::SeqCst); + entry != NOT_WAITING && entry & WORD_MODE == 0 + }) +} + +/// [`wait`] that also ends when the task `task` ends: one doorbell park +/// whose join arm (`SessionFlags::WAIT_TASK`) watches that task beside the +/// word, for a thread that must see both a futex wake and a sibling's end +/// without polling either — the [`thread_controller`](crate::thread_controller) +/// reaping a child that has announced its exit. `report` is a landing the +/// caller owns (a `thread::JoinScratch`); an exit leaves its report there. +/// +/// [`Woken`](WaitOutcome::Woken) covers the task's end too: the caller +/// re-checks its word *and* the task. There is no monitored spin, because +/// the edge waited for is a kernel event no store announces, and the park +/// stays on the doorbell even for a word in a noted window — the kernel's +/// word wait does not compose with a join arm. Uncounted in [`stats`], like +/// the star hub's housekeeping wait: the threads that use it are not the +/// ones a bench measures. +#[cfg_attr(not(target_os = "none"), allow(dead_code))] +pub(crate) fn wait_or_end( + word: &AtomicU32, + expected: u32, + timeout: Option, + task: kore_abi::session::SlotRef, + report: kore_abi::session::SlotRef, +) -> WaitOutcome { + let Some(me) = current_thread() else { + return WaitOutcome::Unsupported; + }; + // Arm, register, check — `wait`'s order and for its reasons. + if !imp::arm() { + return WaitOutcome::Unsupported; + } + let Some(slot) = claim_wait(me, address_of(word)) else { + return WaitOutcome::Unsupported; + }; + let _slot = WaitSlot { thread: me, slot }; + if word.load(Ordering::SeqCst) != expected { + return WaitOutcome::Retry; + } + imp::park_or_end(timeout, task, report) +} + +/// How long a park that could not raise its join arm rides the doorbell +/// alone. A join arm the kernel did not install answers `WouldBlock` at +/// once, with nothing parked; riding the doorbell for this long instead +/// keeps a caller that loops on [`wait_or_end`] from trapping in a ring. +/// The joiner's own pacing for an unestablished park +/// (`thread::join_park`, two milliseconds), so the two degrade alike. +#[cfg_attr( + not(target_os = "none"), + expect(dead_code, reason = "the join arm is a metal-only trap") +)] +const END_SLICE: Duration = Duration::from_millis(2); + /// Whether this thread can [`wait`] at all — a doorbell to park on and an /// identity in the thread band. /// @@ -1182,15 +1406,115 @@ mod imp { let Some(rt) = crate::entry::thread_context() else { return false; }; - let Ok(peer) = u16::try_from(index) else { - return false; - }; - let Some(send) = rt.bootstrap().handle(role::DOORBELL, PEER_BASE + peer) else { + kick_with(&rt, index) + } + + /// [`kick`] against a context the caller already resolved — the sweep + /// kicks many peers off one boot-page parse. + fn kick_with(rt: &crate::entry::Rt, index: usize) -> bool { + let Some(send) = send_for(rt, index) else { return false; }; verbs::post_notice(send, COOKIE_FUTEX, index as u64) } + /// The send this thread holds into band `index`'s doorbell: the one its + /// founding stamped at `DOORBELL@(PEER_BASE + index)`, else the one it + /// adopted since ([`super::adopt_peer`]). + fn send_for(rt: &crate::entry::Rt, index: usize) -> Option { + let peer = u16::try_from(index).ok()?; + rt.bootstrap() + .handle(role::DOORBELL, PEER_BASE + peer) + .or_else(|| crate::entry::geometry::late_kick(index)) + } + + pub(super) fn reaches(index: usize) -> bool { + crate::entry::thread_context().is_some_and(|rt| send_for(&rt, index).is_some()) + } + + pub(super) fn adopt_peer(index: usize, send: kore_abi::session::SlotRef) -> bool { + crate::entry::geometry::set_late_kick(index, Some(send)) + } + + pub(super) fn forget_peer(index: usize) { + let _ = crate::entry::geometry::set_late_kick(index, None); + } + + pub(super) fn sweep(me: usize) -> usize { + let Some(rt) = crate::entry::thread_context() else { + return 0; + }; + super::sweep_with(me, |thread| kick_with(&rt, thread)) + } + + /// The hub this thread relays through, if it adopted one. + pub(super) fn hub() -> Option<&'static super::Hub> { + let addr = crate::entry::geometry::futex_hub(); + // SAFETY: `adopt_hub` records only the address of a `&'static Hub`, + // and nothing ever clears or rewrites it to another value. + (addr != 0).then(|| unsafe { &*(addr as *const super::Hub) }) + } + + pub(super) fn adopt_hub(hub: &'static super::Hub) -> bool { + crate::entry::geometry::set_futex_hub(hub as *const super::Hub as usize) + } + + /// Park on the standing post with the join arm watching `task` beside + /// it — the composite `WAIT_DELIVERY | WAIT_TASK` (± `WAIT_SLEEP`) + /// shape the kernel resolves first-wins (`SessionFlags::WAIT_TASK`). + pub(super) fn park_or_end( + timeout: Option, + task: kore_abi::session::SlotRef, + report: kore_abi::session::SlotRef, + ) -> WaitOutcome { + let Some(db) = doorbell() else { + return WaitOutcome::Unsupported; + }; + let deadline = match timeout { + Some(d) => match crate::clock::uptime() { + Ok(now) => Some((now + d).as_nanos()), + Err(_) => return WaitOutcome::Unsupported, + }, + None => None, + }; + let mut p = SessionBuilder::new(); + p.park_standing(db.chan, db.land.index() as u16, db.recv_tag); + if let Some(at) = deadline { + p.wait_sleep(at); + } + if p.join(task, report).is_err() { + return WaitOutcome::Unsupported; + } + p.wait_task(); + let Ok(committed) = TrapTransport::new().commit_join_park(&p) else { + return WaitOutcome::Unsupported; + }; + match committed.status() { + // The doorbell won: consume and re-stand, as `park` does. + Status::Ok => { + let _ = verbs::take_notice(db.land); + post(&db); + WaitOutcome::Woken + } + // The task ended (`Exited`), or the post or the task was reaped + // (`Revoked`) — the caller's re-check reads which; a reaped + // doorbell refuses its next arm. An OCC loss parked nothing and + // is retried by the caller's loop. + Status::Exited | Status::Revoked | Status::Conflict => WaitOutcome::Woken, + Status::Deadline => WaitOutcome::TimedOut, + // Nobody won, or the join arm could not be raised and nothing + // parked: ride the doorbell alone, bounded (`END_SLICE`). + Status::WouldBlock => { + let slice = timeout.map_or(super::END_SLICE, |t| t.min(super::END_SLICE)); + match park(Some(slice)) { + WaitOutcome::TimedOut if timeout.is_none() => WaitOutcome::Woken, + outcome => outcome, + } + } + _ => WaitOutcome::Unsupported, + } + } + // The relay slot sits exactly one past the last band peer — the // module-level constant and this mesh's arithmetic must agree. const _: () = assert!(super::RELAY == PEER_BASE + super::MAX_THREADS as u16); @@ -1261,6 +1585,37 @@ mod imp { false } + pub(super) fn reaches(_index: usize) -> bool { + false + } + + pub(super) fn adopt_peer(_index: usize, _send: kore_abi::session::SlotRef) -> bool { + false + } + + pub(super) fn forget_peer(_index: usize) {} + + pub(super) fn sweep(_me: usize) -> usize { + 0 + } + + /// No runtime block off metal: no hub was ever adopted. + pub(super) fn hub() -> Option<&'static super::Hub> { + None + } + + pub(super) fn adopt_hub(_hub: &'static super::Hub) -> bool { + false + } + + pub(super) fn park_or_end( + _timeout: Option, + _task: kore_abi::session::SlotRef, + _report: kore_abi::session::SlotRef, + ) -> super::WaitOutcome { + super::WaitOutcome::Unsupported + } + pub(super) fn relay() -> bool { false } @@ -1621,4 +1976,62 @@ mod tests { assert!(!wake(&word)); assert_eq!(wake_all(&word), 0); } + + #[test] + fn a_relay_is_counted_before_the_hub_word_moves() { + // The hub's lost-relay argument: it reads `relays` after it sees its + // word move, so the count must already have moved by then. Off metal + // the kick refuses, but the two stores are the shared half. + static HUB: Hub = Hub::new(); + let (word, relays) = (HUB.word().load(Ordering::SeqCst), HUB.relays()); + assert!(!HUB.relay(), "off metal no kick reaches the hub"); + assert_eq!(HUB.relays(), relays.wrapping_add(1)); + assert_eq!(HUB.word().load(Ordering::SeqCst), word.wrapping_add(1)); + // A plain signal moves the word and owes no relay. + assert!(!HUB.signal()); + assert_eq!(HUB.relays(), relays.wrapping_add(1)); + assert_eq!(HUB.word().load(Ordering::SeqCst), word.wrapping_add(2)); + } + + #[test] + fn a_hub_word_owns_its_interference_span() { + // The hub thread's monitored spin watches its word's line; every + // client of the hub stores to it. Sharing the line with neighbours + // would end that spin on their traffic. + assert_eq!(core::mem::align_of::(), 128); + } + + #[test] + fn a_sweep_kicks_only_doorbell_waiters_other_than_itself() { + // Rows 8..=11: the hub (8), a doorbell waiter (9), a word-parked + // waiter (10), and a thread waiting on nothing (11). + for row in 8..=11 { + reset_thread(row); + } + let a = AtomicU32::new(0); + let b = AtomicU32::new(0); + claim_wait(8, address_of(&a)).expect("the hub's own wait"); + claim_wait(9, address_of(&a)).expect("a doorbell waiter"); + claim_wait_on_word(10, address_of(&b)).expect("a word waiter"); + let mut kicked = std::vec::Vec::new(); + let n = sweep_with(8, |thread| { + kicked.push(thread); + true + }); + assert_eq!(kicked, [9], "the sweep kicked a thread no kick can wake"); + assert_eq!(n, 1); + // A kick that refuses is not counted as a wake. + assert_eq!(sweep_with(8, |_| false), 0); + for row in 8..=11 { + reset_thread(row); + } + } + + #[test] + fn the_own_kick_is_named_past_every_peer_and_the_relay() { + // No peer arithmetic (`PEER_BASE + band`) and no relay lookup may + // ever resolve a thread's own kick. + assert_eq!(OWN_KICK, RELAY + 1); + assert!(OWN_KICK > PEER_BASE + MAX_THREADS as u16 - 1); + } } diff --git a/native/crates/rt/src/process.rs b/native/crates/rt/src/process.rs index 4417d58..03733ae 100644 --- a/native/crates/rt/src/process.rs +++ b/native/crates/rt/src/process.rs @@ -53,6 +53,13 @@ pub fn initialize_with_context( page.initialize_with_context(args, context, TelemetrySnapshot::EMPTY) } +/// Publish the name the launcher resolved the program by +/// ([`ProcessIo::command`]), after the page is initialized and before START. +pub fn write_command(window: &mut [u8], name: &[u8]) -> Result<(), ProcessIoError> { + let (page, _) = ProcessIo::mut_from_prefix(window).map_err(|_| ProcessIoError::Invalid)?; + page.write_command(name) +} + /// Publish the launcher's filesystem snapshot, same window as the telemetry /// snapshot and the same lifecycle point: after founding, before START. pub fn write_filesystem( diff --git a/native/crates/rt/src/service/mod.rs b/native/crates/rt/src/service/mod.rs index bf11006..a4cc04a 100644 --- a/native/crates/rt/src/service/mod.rs +++ b/native/crates/rt/src/service/mod.rs @@ -413,6 +413,21 @@ pub trait SystemService: crate::svc::SvcProtocol + Sized { /// retryable domain custody still closes in `peer_closed` afterward. fn peer_detaching(&mut self, _peer: crate::svc::PeerId) {} + /// Recyclable peers this service owes one wake: it published a batch into + /// an event channel it keeps in their pair's tail (`attach_peer_aux`, + /// `res/EVENT.md`) and the batch's commit found the consumer parked. + /// + /// [`ServeHost`](crate::svc::ServeHost) asks after every unit of work — + /// a request, a teardown, an idle recheck — and sends each peer named in + /// `owed[..n]` one notice through the endpoint it owns, so a service never + /// copies an affine endpoint out of its peer bundle to wake a client. Name + /// each peer once per owed wake and clear it as it is handed over; a full + /// batch is asked again. The default owes nothing: the request/reply shape, + /// whose only wake is the reply's. + fn take_owed_wakes(&mut self, _owed: &mut [Option]) -> usize { + 0 + } + /// Observe completion of an admission requested through /// [`Serve::admit_peer`]. The host, not the wire request, chooses the /// generational identity and reports only the authenticated result. diff --git a/native/crates/rt/src/session/builder.rs b/native/crates/rt/src/session/builder.rs index fa60e93..089a589 100644 --- a/native/crates/rt/src/session/builder.rs +++ b/native/crates/rt/src/session/builder.rs @@ -1264,6 +1264,27 @@ impl SessionBuilder { Ok(Staged(result)) } + /// Consume a writable lease and return its **frozen** successor + /// (`Revoke × REVOKE_KIND_FREEZE`) — the exclusive intake of + /// `res/IMAGE.md` §6b. + /// + /// The kernel requires `ACCESS|MODIFY|DERIVE|REVOKE` on `subject`, kills + /// its complete descendant lineage, removes every CPU/DMA/guest alias, + /// and writes an `ACCESS|MODIFY|DERIVE|REVOKE` lease over the same window + /// into `result`. No lessor may write that window again while the + /// successor lives — the successor and its carves included. + pub fn freeze(&mut self, subject: SlotRef, result: SlotRef) -> Result { + self.push(Inv { + op: OpCode::Revoke as u16, + requested: 0, + kind: kore_abi::session::REVOKE_KIND_FREEZE, + subject: subject.index(), + result: result.index(), + arg: Invocation::NO_ARG, + })?; + Ok(Staged(result)) + } + /// `Derive × DERIVE_KIND_ARENA_EXTEND` on the task's own space /// self-view (slice E): grow the caller's cap arena by one chunk. /// An **effect**, not a mint — the request is empty, the arg is the diff --git a/native/crates/rt/src/session/tests.rs b/native/crates/rt/src/session/tests.rs index 8446618..7cfcd4a 100644 --- a/native/crates/rt/src/session/tests.rs +++ b/native/crates/rt/src/session/tests.rs @@ -235,6 +235,36 @@ fn the_executable_return_is_the_finalizations_wire_twin() { ); } +/// The freeze is the third turn and the same wire shape: one no-argument +/// consuming revoke, distinct from both of the others. +#[test] +fn the_freeze_is_a_no_argument_consuming_revoke() { + let source = SlotRef::from_wire(12); + let result = SlotRef::from_wire(13); + let mut builder = SessionBuilder::new(); + let staged = builder.freeze(source, result).expect("the freeze stages"); + assert_eq!(staged.slot(), result); + + let mut out = [0u8; DESCRIPTOR_MAX]; + let len = builder.finish(&mut out).expect("descriptor finishes"); + assert_eq!(len, 8 + 16, "the same one-invocation descriptor"); + let (header, invocations, args) = parse(&out[..len]); + assert_eq!(header.count.get(), 1); + assert_eq!(invocations[0].op.get(), OpCode::Revoke as u16); + assert_eq!(invocations[0].kind, kore_abi::session::REVOKE_KIND_FREEZE); + assert_eq!(invocations[0].requested, 0, "rights come from the lease"); + assert_eq!(invocations[0].subject.get(), source.index()); + assert_eq!(invocations[0].result.get(), result.index()); + assert_eq!(invocations[0].arg.get(), Invocation::NO_ARG); + assert!(args.is_empty(), "the freeze names no second capability"); + for other in [ + kore_abi::session::REVOKE_KIND_EXEC, + kore_abi::session::REVOKE_KIND_EXEC_RETURN, + ] { + assert_ne!(kore_abi::session::REVOKE_KIND_FREEZE, other); + } +} + /// Length selection happens before the transport chooses registers or /// scratch, but both carriers must receive byte-identical output from the one /// encoder. `finish` writes exactly that prefix and leaves spare carrier bytes diff --git a/native/crates/rt/src/spmd.rs b/native/crates/rt/src/spmd.rs index 135b2c9..5c8b23a 100644 --- a/native/crates/rt/src/spmd.rs +++ b/native/crates/rt/src/spmd.rs @@ -847,6 +847,7 @@ fn launch( // ticket whose founding never happened refuses its join and // detaches (the band stays spent — visible, never hidden). ── decide(ctx, ABORT); + let mut detached = false; for staged in workers.iter_mut().filter_map(Option::take) { // A refused join here is a transient observation failure on // a worker that is already exiting; one bounded retry, then @@ -855,7 +856,10 @@ fn launch( Ok(_) => {} Err(e) => match crate::thread::join_reserved(e.into_thread(), &mut join_scratch) { Ok(_) => {} - Err(e) => e.into_thread().detach(), + Err(e) => { + detached = true; + e.into_thread().detach() + } }, } } @@ -866,6 +870,15 @@ fn launch( // invariant changes later. let _ = star.finish(); } + // Every worker this launch founded is joined, so none can read the + // gate again: restage it, and the next launch over `ctx` may run. + // A detached worker may still be parked on it, and a restaged gate + // would release it into the next launch's body: `ctx` then stays + // aborted, refusing every launch over it, rather than hand one + // region's job to another's worker. + if !detached { + ctx.restage(); + } return None; } decide(ctx, RUN); @@ -1539,6 +1552,10 @@ fn dispatch_resident( .map(|m| m.fitting_width()) .unwrap_or(1) .clamp(1, DISPATCH_CAP as u32); + // No wider than this thread's arena can found: a resident gang + // holds its founder's names for the task's life, and a thread + // other than thread 0 founds from a fixed arena. + let fit = affordable(fit, false); let founded = (fit >= 2) .then(|| { found_resident::( @@ -1556,7 +1573,15 @@ fn dispatch_resident( // contract says; an executable takes `PROFILE_CHANNELS` // to found the star. Some(region) if !region.star_active() => { - let _ = region.retire(); + // The retire told every worker to end before its first + // join, so each join it could not complete is one still + // to finish: collect them all, as a dispatch collects its + // own. Dropping the region instead would leave the gang + // running, its bands, stacks and slots spent for the + // task's life. + if let Err(refused) = region.retire() { + join_dispatched(refused.spmd); + } gang.state.store(RESIDENT_REFUSED, Relaxed); return ResidentOutcome::Unavailable; } @@ -1764,6 +1789,104 @@ fn block_split(span: u32, workers: u32, worker: u32) -> (u32, u32) { (start, start + len) } +/// What a [`launch`] founds beside its workers: what its arena budget +/// ([`launch_slots`]) depends on. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +#[cfg_attr(all(not(target_os = "none"), not(test)), allow(dead_code))] +struct LaunchShape { + /// The star a launch here mints: `Some(founder_kick)` — whether each + /// member's mesh carries the founder's kick — or `None` for a starless + /// launch. + star: Option, + /// Whether the launch founds the star's relay hub: a one-shot region does, + /// a resident one does not. + hub: bool, + /// Whether each worker takes a TLS block. + tls: bool, + /// Whether each compute worker is endowed the process heap. + heap: bool, +} + +/// The arena names a [`launch`] of `width` workers holds at its peak. +/// +/// The launch holds its join landing and the star's wires +/// ([`star::wires_slots`](crate::star::wires_slots)) throughout. Each gang +/// session then holds every one of its workers' founding names at its commit +/// ([`thread::gang_founding_slots`](crate::thread::gang_founding_slots)), on +/// top of what the sessions before it kept ([`RETAINED_SLOTS`] a worker), and +/// the hub's session holds its own on top of all of them. A launch whose peak +/// fits the arena cannot run out of names part way through, where a refusal +/// leaks what it claimed. The sessions are [`launch`]'s own split +/// ([`founding_sessions`]): a starred session carries at most +/// [`MESH_CEILING`](crate::star::MESH_CEILING) workers. +/// +/// [`RETAINED_SLOTS`]: crate::thread::RETAINED_SLOTS +#[cfg_attr(all(not(target_os = "none"), not(test)), allow(dead_code))] +fn launch_slots(width: u32, shape: LaunchShape) -> usize { + use crate::thread::{RETAINED_SLOTS, gang_founding_slots}; + let wires = shape + .star + .map_or(0, |_| crate::star::wires_slots(width as usize)); + let mut held = 1 + wires; + let mut peak = held; + let ceiling = match shape.star { + Some(_) => crate::star::MESH_CEILING, + None => crate::session::MAX_SESSION_FOUNDS, + }; + for (start, end) in founding_sessions(width, ceiling as u32) { + let chunk = (end - start) as usize; + let mesh = shape + .star + .map_or(0, |kick| crate::star::mesh_caps(kick, chunk)); + let endowments = chunk * (mesh + usize::from(shape.heap)); + peak = peak.max(held + gang_founding_slots(chunk, shape.tls, endowments)); + held += chunk * (RETAINED_SLOTS + usize::from(shape.tls)); + } + if shape.hub && shape.star.is_some() { + // The hub's endowment: its own recv and a send to every worker. + peak = peak.max(held + gang_founding_slots(1, shape.tls, 1 + width as usize)); + } + peak +} + +/// The widest launch of at most `want` workers whose peak +/// ([`launch_slots`]) fits `available` arena names, or 0 when not one worker +/// fits. +/// +/// Every width is weighed, widest first: the peak is not monotonic in the +/// width, because a wider gang splits into more sessions whose meshes are +/// smaller (seven meshed workers cost one session of seven; eight cost two of +/// four). +#[cfg_attr(all(not(target_os = "none"), not(test)), allow(dead_code))] +fn affordable_width(want: u32, shape: LaunchShape, available: usize) -> u32 { + (1..=want) + .rev() + .find(|&width| launch_slots(width, shape) <= available) + .unwrap_or(0) +} + +/// The shape a launch over this thread would take, read before it mints +/// anything. +#[cfg(target_os = "none")] +fn launch_shape(hub: bool) -> LaunchShape { + LaunchShape { + star: crate::star::shape().map(|star| star.founder_kick), + hub, + tls: crate::entry::geometry::tls_template().is_some(), + heap: worker_heap().is_some(), + } +} + +/// [`affordable_width`] on this thread's arena, now. +#[cfg(target_os = "none")] +fn affordable(want: u32, hub: bool) -> u32 { + affordable_width( + want, + launch_shape(hub), + crate::entry::geometry::available_cap_slots(), + ) +} + /// **The parallel-for** — run `body` over `0..tiles`, sized to the /// machine, with no thread configuration at all. /// @@ -1855,6 +1978,12 @@ fn dispatch_erased( ResidentOutcome::Busy => return None, ResidentOutcome::Unavailable => {} } + // The one-shot region, as wide as asked and no wider than this thread's + // arena can found whole; `None` only when not one worker fits. + let width = affordable(width, true); + if width == 0 { + return None; + } let env = &body as *const &(dyn Fn(&'static SpmdCtx, u32) + Sync) as *mut (); let region = launch::( &DISPATCH_CTX, @@ -2319,6 +2448,100 @@ mod tests { assert_eq!(split(12), [6, 6]); } + const fn shape(star: Option, hub: bool, tls: bool) -> LaunchShape { + LaunchShape { + star, + hub, + tls, + heap: true, + } + } + + /// The budget counts what the founding holds, step by step: the join + /// landing and the star's wires, every worker's founding names and mesh + /// at the session's commit, what each session keeps, then the hub's + /// session on top. Three meshed workers founded off thread 0 peak at 46 + /// names: two more than a 64-slot worker's arena had left, the C + /// `main` that `kore_spmd_dispatch` refused on. + #[test] + fn a_launch_budget_counts_what_its_founding_holds() { + // Resident, starred, no founder kick: 1 + 9 wires, then 3 × 7 + // founding names and 3 × (4 mesh + heap) endowments. + assert_eq!(launch_slots(3, shape(Some(false), false, false)), 46); + // The one-shot's hub founds after the gang returned its plan: 10 + // held + 3 × 4 kept + 7 + (1 + 3) endowments = 33, under the gang's + // peak. + assert_eq!(launch_slots(3, shape(Some(false), true, false)), 46); + // Thread 0's star carries its kick: one more mesh cap a worker. + assert_eq!(launch_slots(3, shape(Some(true), false, false)), 49); + // Starless: the landing, then 3 × (7 + heap). + assert_eq!(launch_slots(3, shape(None, true, false)), 25); + // A TLS block is one more founding name and one more kept name. + assert_eq!(launch_slots(3, shape(None, false, true)), 28); + // Two sessions: the second commits over what the first kept. + // 8 meshed workers: 1 + 19, then 4 × 7 + 4 × 6 = 72, keeping 16; + // then 36 + 52 = 88. + assert_eq!(launch_slots(8, shape(Some(false), false, false)), 88); + } + + /// A wider gang can cost less at its peak: past the mesh ceiling it + /// splits into sessions whose meshes are smaller. The clamp weighs every + /// width for that reason. + #[test] + fn a_launch_peak_is_not_monotonic_in_its_width() { + let meshed = crate::star::MESH_CEILING as u32; + let starred = shape(Some(false), false, false); + assert!(launch_slots(meshed + 1, starred) < launch_slots(meshed, starred)); + let between = launch_slots(meshed + 1, starred); + assert_eq!(affordable_width(meshed + 1, starred, between), meshed + 1); + assert!(affordable_width(meshed, starred, between) < meshed); + } + + /// The clamp takes the widest width that fits and refuses only when one + /// worker does not: the 44 names a 64-slot C `main` had left found two of + /// its three workers, and a founder's arena founds every one. + #[test] + fn a_launch_is_clamped_to_its_arena() { + let resident = shape(Some(false), false, false); + assert_eq!(affordable_width(3, resident, 44), 2); + assert_eq!(affordable_width(3, resident, 46), 3); + assert_eq!( + affordable_width(3, resident, launch_slots(1, resident) - 1), + 0 + ); + assert_eq!(affordable_width(0, resident, usize::MAX), 0); + // A founder's dynamic names: the founding granule below the linker + // band, after a doorbell worker's boot names and runtime layout. + let founder = crate::entry::STARTUP_LINK_SLOT_BASE as usize - (3 + 5 + 1 + 10 + 2); + for tls in [false, true] { + for hub in [false, true] { + let widest = crate::thread::MAX_THREADS - 1; + let width = affordable_width(widest, shape(Some(false), hub, tls), founder); + assert!(width >= 16, "tls {tls} hub {hub}: {width}"); + } + } + } + + /// The starless degrade (a refused mint) never needs more than the + /// starred launch its budget was weighed for, at any width, so a launch + /// sized for its star cannot run out of names when the star fails. + #[test] + fn the_starless_degrade_fits_its_starred_budget() { + for width in 1..crate::thread::MAX_THREADS { + for kick in [false, true] { + for tls in [false, true] { + for hub in [false, true] { + assert!( + launch_slots(width, shape(None, hub, tls)) + <= launch_slots(width, shape(Some(kick), hub, tls)), + "width {width} kick {kick} tls {tls} hub {hub}" + ); + } + } + } + } + } + /// A scripted worker: joins yield the scripted outcomes in order. struct MockWorker { outcomes: &'static [Result, crate::cap::TaskWaitError>], diff --git a/native/crates/rt/src/star.rs b/native/crates/rt/src/star.rs index 7be7730..2476ce3 100644 --- a/native/crates/rt/src/star.rs +++ b/native/crates/rt/src/star.rs @@ -110,6 +110,49 @@ pub(crate) struct StarPlan { wires: Wires, } +/// Arena names a star of `workers` holds in its founder for the region's +/// life: each worker's channel root and the parent's send into it (the mint's +/// third name, the parent's recv, is retired at once), and the hub channel's +/// root, send and recv. [`provision`]'s peak is the same count: the last +/// mint's recv is the hub's, kept. +pub(crate) const fn wires_slots(workers: usize) -> usize { + 2 * workers + 3 +} + +/// The star a thread could mint, before it mints one: whether it holds a +/// channel space to mint from, and whether its members would carry the +/// founder's kick. [`provision`] mints exactly this shape, and a launch +/// budgets its arena by it (`spmd::launch_slots`). +#[cfg(target_os = "none")] +#[derive(Clone, Copy, Debug)] +pub(crate) struct StarShape { + /// The founder's own kick rides every member's mesh: only thread 0 + /// holds a send into its own doorbell (`DOORBELL@PEER_BASE`). + pub(crate) founder_kick: bool, +} + +/// This thread's [`StarShape`], or `None` with no `CHANNEL_SPACE` grant. +#[cfg(target_os = "none")] +pub(crate) fn shape() -> Option { + let rt = crate::entry::thread_context()?; + rt.bootstrap() + .handle(kore_abi::task::role::CHANNEL_SPACE, 0)?; + Some(StarShape { + founder_kick: founder_kick(&rt).is_some(), + }) +} + +/// The founder kick a member's mesh carries (see [`StarShape`]). +#[cfg(target_os = "none")] +fn founder_kick(rt: &crate::entry::Rt) -> Option { + if crate::entry::geometry::shard_index() == Some(0) { + rt.bootstrap() + .handle(kore_abi::task::role::DOORBELL, crate::futex::PEER_BASE) + } else { + None + } +} + /// Mint the region's whole forest, all-or-nothing: `workers` channels /// plus the hub's. `None` — no `CHANNEL_SPACE` boot grant, dynamic /// arena slots exhausted, a refused mint — releases everything minted @@ -164,12 +207,7 @@ pub(crate) fn provision(workers: usize) -> Option { // doorbell — a spawned thread's `DOORBELL@PEER_BASE` names thread 0's — // so a region founded elsewhere carries no founder kick and its // launcher rides the park's slice (`res/SPMD.md` §14.2). - let founder_kick = if crate::entry::geometry::shard_index() == Some(0) { - rt.bootstrap() - .handle(kore_abi::task::role::DOORBELL, crate::futex::PEER_BASE) - } else { - None - }; + let founder_kick = founder_kick(&rt); Some(StarPlan { wires: Wires { @@ -186,6 +224,11 @@ pub(crate) fn provision(workers: usize) -> Option { }) } +/// [`StarPlan::mesh_caps`] before a plan exists, for a launch's budget. +pub(crate) const fn mesh_caps(founder_kick: bool, chunk: usize) -> usize { + 2 + founder_kick as usize + chunk.saturating_sub(1) +} + /// Descriptor invocations one meshed worker costs its gang session in a /// chunk of `chunk`: two views, the boot delegate and the Found, plus its /// endowment of `chunk + 3` caps (own recv, hub send, the founder's send, @@ -219,6 +262,14 @@ pub(crate) const MESH_CEILING: usize = { #[cfg(target_os = "none")] impl StarPlan { + /// Capabilities one member of a chunk of `chunk` carries in its mesh: + /// its own recv, the hub's send, the founder's kick when the plan has + /// one, and a send to every chunk sibling — the count + /// [`worker_endow_mesh`](Self::worker_endow_mesh) fills. + pub(crate) const fn mesh_caps(&self, chunk: usize) -> usize { + mesh_caps(self.wires.founder_kick.is_some(), chunk) + } + /// Worker `w`'s endowments — **the intra-chunk kick mesh**: its own /// recv (→ `DOORBELL@0`, which the stamped landing name at /// `DOORBELL@1` completes into a working park), the hub send diff --git a/native/crates/rt/src/startup.rs b/native/crates/rt/src/startup.rs index 800d2be..186a3b0 100644 --- a/native/crates/rt/src/startup.rs +++ b/native/crates/rt/src/startup.rs @@ -29,12 +29,23 @@ pub fn drain() -> Result ! { +pub fn barrier(finalize: bool) { use kore_abi::session::Status; let report = loop { match drain() { @@ -46,9 +57,19 @@ pub fn finish(code: ExitCode) -> ! { Err(_) => crate::thread::exit(ExitCode::INTERNAL), } }; - if crate::entry::geometry::begin_process_exit() && !report.forced { + // Marked whether or not this end finalizes, so no later end can run them. + let first = crate::entry::geometry::begin_process_exit(); + if finalize && first && !report.forced { crate::entry::geometry::run_finalizers(); } +} + +/// End a returning program. Live workers are stopped without their cooperation; +/// their stacks and shared storage are reclaimed by kernel task teardown. +/// User finalizers run only after a clean worker barrier ([`barrier`]). +#[cfg(target_os = "none")] +pub fn finish(code: ExitCode) -> ! { + barrier(true); crate::thread::exit(code) } diff --git a/native/crates/rt/src/svc/client.rs b/native/crates/rt/src/svc/client.rs index 8d187dc..994d5c8 100644 --- a/native/crates/rt/src/svc/client.rs +++ b/native/crates/rt/src/svc/client.rs @@ -487,6 +487,21 @@ pub enum AffineBundleCallError { AfterTransfer(CallError), } +/// Failure of a context-bundle call +/// ([`call_with_context_bundle`](SvcClient::call_with_context_bundle)): the +/// [`AffineBundleCallError`] discipline for a bundle with no executable +/// member, so no witness and no first input. +#[derive(Debug)] +pub enum ContextBundleCallError { + NotMoved { + error: CallError, + second: B, + third: C, + auxiliaries: [Option; MAX_INVOCATION_AUXILIARIES], + }, + AfterTransfer(CallError), +} + /// Failure of a return call whose transferred capability is freshly staged /// inside the transfer session. /// @@ -2264,6 +2279,170 @@ impl<'w, 't, T: Transport, S: SvcProtocol> SvcClient<'w, 't, T, S> { } } + /// Move a service request and its context bundle in one atomic session + /// — [`call_with_invocation_bundle`](Self::call_with_invocation_bundle) + /// without the executable member. Such a request names no generation to + /// invoke, so it carries no witness and no input sent through an + /// executable endpoint; the receiver correlates the bundle with the + /// request by the data the request itself carries. Every other step is + /// the invocation's, with its helpers: the same preflights before the + /// request record is published, the same single transfer session, and + /// the same fail-closed poisoning once the record is visible. + pub async fn call_with_context_bundle( + &mut self, + req: &S::Request, + second: B, + second_rights: Rights, + process_tx: SlotRef, + third: C, + third_rights: Rights, + third_tx: SlotRef, + auxiliaries: [Option; MAX_INVOCATION_AUXILIARIES], + auxiliary_tx: [SlotRef; MAX_INVOCATION_AUXILIARIES], + ) -> Result, ContextBundleCallError> { + macro_rules! not_moved { + ($error:expr) => { + ContextBundleCallError::NotMoved { + error: $error, + second, + third, + auxiliaries, + } + }; + } + + if let Err(error) = self.require_idle() { + return Err(not_moved!(error)); + } + // Check every destination before the record is published: see the + // invocation bundle's note on why a post-publication rollback cannot + // be undone. + if ![process_tx, third_tx] + .into_iter() + .all(live_send_destination) + || auxiliaries + .iter() + .zip(auxiliary_tx) + .any(|(auxiliary, destination)| { + auxiliary.is_some() && !live_send_destination(destination) + }) + { + return Err(not_moved!(CallError::Refused(Status::Revoked))); + } + let request = S::encode_req(req); + let grants = match self.prepare_call(&request) { + Ok(grants) => grants, + Err(error) => return Err(not_moved!(error)), + }; + if self.rings.req.capacity_left() == 0 { + return Err(not_moved!(CallError::RingFull)); + } + if let Err(error) = self.ensure_grant_posts(grants).await { + return Err(not_moved!(error)); + } + let second_slot = match preflight_return_send(&second, second_rights) { + Ok(slot) => slot, + Err(error) => return Err(not_moved!(error)), + }; + let third_slot = match preflight_return_send(&third, third_rights) { + Ok(slot) => slot, + Err(error) => return Err(not_moved!(error)), + }; + let mut source_slots = [SlotRef::from_wire(0); 2 + MAX_INVOCATION_AUXILIARIES]; + source_slots[0] = second_slot; + source_slots[1] = third_slot; + let mut source_count = 2; + for auxiliary in auxiliaries.iter().flatten() { + let checked = match auxiliary.transfer { + AuxiliaryTransfer::Derived => { + preflight_return_send(&auxiliary.capability, auxiliary.rights) + } + AuxiliaryTransfer::Exact => preflight_exact_return(&auxiliary.capability), + }; + let slot = match checked { + Ok(slot) => slot, + Err(error) => return Err(not_moved!(error)), + }; + source_slots[source_count] = slot; + source_count += 1; + } + if !invocation_sources_are_distinct(&source_slots[..source_count]) { + return Err(not_moved!(CallError::Refused(Status::Malformed))); + } + + let mut transfer = SessionBuilder::new(); + let mut malformed = transfer + .send(process_tx, second_slot, second_rights) + .and_then(|_| transfer.send(third_tx, third_slot, third_rights)) + .is_err(); + for (auxiliary, destination) in auxiliaries.iter().zip(auxiliary_tx) { + if malformed { + break; + } + let Some(auxiliary) = auxiliary.as_ref() else { + continue; + }; + let Some(slot) = return_capability_sealed::Sealed::return_slot(&auxiliary.capability) + else { + return Err(not_moved!(CallError::Refused(Status::Revoked))); + }; + malformed = match auxiliary.transfer { + AuxiliaryTransfer::Derived => { + transfer.send(destination, slot, auxiliary.rights).is_err() + } + AuxiliaryTransfer::Exact => transfer.send_exact(destination, slot).is_err(), + }; + } + if malformed { + return Err(not_moved!(CallError::Refused(Status::Malformed))); + } + + let observation = ReplyObservation::begin(None).await; + if self.rings.req.push(request).is_err() { + return Err(not_moved!(CallError::RingFull)); + } + let committed = match self.transport.commit(&transfer).await { + Ok(committed) => committed, + Err(_) => { + drop(second); + drop(third); + drop(auxiliaries); + self.poison(); + return Err(ContextBundleCallError::AfterTransfer(CallError::Transport)); + } + }; + match classify_return_commit(committed, transfer.staged(), ReturnSource::Held) { + ReturnCommit::Moved => { + return_capability_sealed::Sealed::on_proven_moved(second); + return_capability_sealed::Sealed::on_proven_moved(third); + for auxiliary in auxiliaries.into_iter().flatten() { + return_capability_sealed::Sealed::on_proven_moved(auxiliary.capability); + } + self.state = ClientState::AwaitingReply { request }; + self.kick_published_request() + .await + .map_err(ContextBundleCallError::AfterTransfer)?; + self.collect_reply(observation) + .await + .map_err(ContextBundleCallError::AfterTransfer) + } + ReturnCommit::NotMoved(error) => { + // Custody stayed local, but the record is already visible + // and the ring has no rollback: fail closed, as the + // invocation bundle does. + self.poison(); + Err(not_moved!(error)) + } + ReturnCommit::AfterTransfer(error) => { + drop(second); + drop(third); + drop(auxiliaries); + self.poison(); + Err(ContextBundleCallError::AfterTransfer(error)) + } + } + } + /// Issue a protocol-declared implicit return request as one L4-style /// **send-and-wait** crossing. /// diff --git a/native/crates/rt/src/svc/mod.rs b/native/crates/rt/src/svc/mod.rs index 86aa8fb..db6c47e 100644 --- a/native/crates/rt/src/svc/mod.rs +++ b/native/crates/rt/src/svc/mod.rs @@ -48,7 +48,8 @@ mod recyclable; mod serve; pub use client::{ - AffineBundleCallError, CallError, CallReply, ClientEndpoints, GrantPin, InvocationAuxiliary, + AffineBundleCallError, CallError, CallReply, ClientEndpoints, ContextBundleCallError, GrantPin, + InvocationAuxiliary, InvocationAuxiliaryCapability, MAX_INVOCATION_AUXILIARIES, Observation, ReplyWaitStats, RetainedReturnCallError, ReturnCallError, ReturnCapability, ReturnDeriveError, ReturnDeriver, ReturnLeaseSource, diff --git a/native/crates/rt/src/svc/serve.rs b/native/crates/rt/src/svc/serve.rs index 1b79832..985f785 100644 --- a/native/crates/rt/src/svc/serve.rs +++ b/native/crates/rt/src/svc/serve.rs @@ -2358,6 +2358,39 @@ impl<'w, 't, T: Transport, S: SystemService> ServeHost<'w, 't, T, S> { // work: a pair a serve step just minted is attached before // the next wake, and a retired one stops now. self.settle_minted(); + self.deliver_owed_wakes().await; + } + } + + /// Send each peer the service owes a wake its one notice + /// ([`SystemService::take_owed_wakes`]), through the endpoint this host + /// owns. A service names a peer once per owed wake, so the batches end + /// within the peer table; the bound only keeps a service that kept + /// answering full batches from holding the loop. + async fn deliver_owed_wakes(&mut self) { + // Peers asked for per call: a stack array, so the ask allocates + // nothing; its size only trades calls for stack, and the founder + // roster's whole recyclable band fits one batch. + const BATCH: usize = 8; + for _ in 0..=self.pairs.len().div_ceil(BATCH) { + let mut owed = [None; BATCH]; + let count = self.svc.take_owed_wakes(&mut owed).min(BATCH); + for peer in owed[..count].iter().flatten() { + // A peer retired since its batch was published has no one + // left to wake; `notify_peer` refuses it by generation. + let _ = self + .notify_peer( + *peer, + Notice { + cookie: crate::event::COOKIE_EVENT, + payload: 0, + }, + ) + .await; + } + if count < BATCH { + break; + } } } @@ -2406,6 +2439,7 @@ impl<'w, 't, T: Transport, S: SystemService> ServeHost<'w, 't, T, S> { // The handshake drains even on an idle pass, so a host test can // attach a freshly minted pair without a request in flight. self.settle_minted(); + self.deliver_owed_wakes().await; served } } @@ -2820,6 +2854,116 @@ mod tests { ); } + /// A service that publishes into its peers' auxiliary planes and owes + /// some of them a wake (`SystemService::take_owed_wakes`). + struct Owing { + /// Peers still owed, or `None` for a service that answers full + /// batches forever. + owed: Option, + asks: u32, + } + + impl SystemService for Owing { + const NAME: &'static str = "owing-test"; + type Endowment = NoCaps; + + fn bind(_: NoCaps) -> Self { + Self { owed: Some(0), asks: 0 } + } + + fn serve(&mut self, _: u64, sv: &mut Serve<'_, Reply>) { + sv.reply(Reply::Granted); + } + + fn take_owed_wakes(&mut self, owed: &mut [Option]) -> usize { + self.asks += 1; + let count = match self.owed.as_mut() { + Some(left) => { + let count = (*left).min(owed.len()); + *left -= count; + count + } + None => owed.len(), + }; + for peer in &mut owed[..count] { + *peer = PeerId::from_arrival(0x102); + } + count + } + } + + impl SvcProtocol for Owing { + type Request = u64; + type Response = Reply; + type ReqWire = u64; + type RepWire = u64; + + const REPLY_GRANTS: usize = 0; + + fn encode_req(req: &u64) -> u64 { + *req + } + + fn decode_req(wire: &u64) -> Option { + Some(*wire) + } + + fn encode_rep(rep: &Reply, grants: u8) -> u64 { + DropAware::encode_rep(rep, grants) + } + + fn decode_rep(wire: &u64) -> Option<(Reply, u8)> { + DropAware::decode_rep(wire) + } + } + + /// The host asks for owed wakes after every pass — a served request and + /// an idle one alike — asks again while batches come back full, and stops + /// within the peer table even for a service that never stops owing. (A + /// notice to a peer that is not a live recyclable generation is refused + /// by generation, which is what a peer retired since its batch gets.) + #[test] + fn the_host_drains_owed_wakes_after_every_pass_and_bounds_the_batches() { + let window = { + let bytes = window_bytes::(REQ_CAP, REP_CAP); + let layout = std::alloc::Layout::from_size_align(bytes, 64).expect("window layout"); + // SAFETY: as `window()` above — one leaked aligned allocation. + unsafe { + let ptr = std::alloc::alloc_zeroed(layout); + assert!(!ptr.is_null()); + core::slice::from_raw_parts_mut(ptr, bytes) + } + }; + let (mut client, rings) = split_window::(window, REQ_CAP, REP_CAP).expect("rings"); + let endpoints = ServeEndpoints { + kick: SlotRef::from_wire(60), + grant_tx: [None; MAX_GRANTS], + scratch: [None; MAX_GRANTS], + return_post: None, + }; + let transport = OkTransport::new(); + let mut host = ServeHost::new(Owing::bind(NoCaps), &transport, rings, endpoints); + + client.req.push(1).expect("request fits"); + assert!(drive(host.serve_next())); + assert_eq!(host.service().asks, 1, "a pass that owes nothing asks once"); + + // Every peer the table holds owed at once: full batches, then the + // short ask that ends them. + let peers = host.pairs.len(); + host.service_mut().owed = Some(peers); + assert!(!drive(host.serve_next())); + assert_eq!(host.service().asks, 1 + peers as u32 / 8 + 1); + assert_eq!(host.service().owed, Some(0)); + + // A service that never stops owing is bounded by the peer table. + host.service_mut().owed = None; + host.service_mut().asks = 0; + assert!(!drive(host.serve_next())); + let bound = host.pairs.len().div_ceil(8) as u32 + 1; + assert_eq!(host.service().asks, bound); + } + /// Walking return posts are host-owned reply-and-wait state: the first /// stands before readiness, and every successor is acknowledged before /// `serve_next` returns with its reply visible. diff --git a/native/crates/rt/src/thread.rs b/native/crates/rt/src/thread.rs index bbbe78a..7064582 100644 --- a/native/crates/rt/src/thread.rs +++ b/native/crates/rt/src/thread.rs @@ -415,6 +415,22 @@ fn release_thread_index(n: u32) { } } +/// **How many thread bands are free now**: how many more threads this task +/// could found before its band plan is full. An upper bound — a founding also +/// needs its founder's arena names and, for a strip stack, room in the stack +/// window — and a snapshot, which a concurrent founding or join moves. +#[cfg(target_os = "none")] +pub fn free_bands() -> u32 { + let used = THREAD_SLOTS.load(Ordering::Acquire); + (!used & (((1u64 << MAX_THREADS) - 1) as u32)).count_ones() +} + +/// Off the metal no thread founds another: no band is free. +#[cfg(not(target_os = "none"))] +pub fn free_bands() -> u32 { + 0 +} + /// How a spawned thread enters Rust: the plain `fn()` every service /// worker uses, or an **spmd entry** carrying a region launch's context /// and the index it assigned (`res/SPMD.md` §2) — the CUDA @@ -480,18 +496,21 @@ pub struct Thread { /// The parent-arena slot holding the child's task handle: the join /// subject, and (revoked) the reap. handle: SlotRef, - /// The parent-held `Region`s the provisioning session produced — - /// stack, runtime block, boot staging page. Revoking them is what - /// actually unmaps the band; the parent holds them precisely so that - /// a dead thread cannot keep its own stack alive. - provisioned: [SlotRef; 3], + /// The parent-held `Region`s the provisioning session produced that + /// outlive the founding: the stack and the runtime block. Revoking them + /// is what actually unmaps the band; the parent holds them precisely so + /// that a dead thread cannot keep its own stack alive. The third, the + /// boot staging page, is the founding's delegation source, which its + /// commit consumes ([`HELD_REGIONS`]). + regions: [SlotRef; HELD_REGIONS], tls_region: Option, - /// Founding result slots the kernel already scrubbed (the staged - /// space, the delivered view, the boot delegation, one per - /// endowment): no capability is left in them, but the *userland* slot - /// reservation is, and a task that never returned it would run out of - /// arena long before it ran out of bands. - spent: [SlotRef; 3 + MAX_WORKER_CAPS], + /// Founding names whose return waits for [`join`]: always the staged + /// space, whose founder record the `Found` leaves standing until the reap + /// ([`STAGED_SPACE`]); and, for a gang founding that did not commit whole, + /// whose outcome only the join learns, what a whole commit returns at + /// once ([`release_committed`]: [`RELEASED_AT_COMMIT`] names and one per + /// endowment). + spent: [SlotRef; SPENT_CAPACITY], /// Live entries in [`spent`](Self::spent). spent_len: u8, /// The arena strip this stack was carved from — `Some` for SPMD @@ -629,12 +648,108 @@ pub(crate) fn heap_endowment() -> Option { Some(WorkerCap::heap(rt.bootstrap().handle(role::HEAP, 0)?)) } -/// Off metal no thread founds another; there is no heap to endow. +/// The rights of a wake send a thread may pass on: the notice itself +/// (`ACCESS`, [`verbs::NOTICE_RIGHTS`](crate::cap::verbs::NOTICE_RIGHTS)) and +/// `DERIVE`, so the thread can found children holding the same end. A send +/// into a wake channel conveys nothing but the right to kick it. +#[cfg_attr(not(target_os = "none"), allow(dead_code))] +pub(crate) const WAKE_SEND_RIGHTS: kore_abi::cap::Rights = + kore_abi::cap::Rights::ACCESS.union(kore_abi::cap::Rights::DERIVE); + +/// [`WorkerCap`] re-deriving this thread's own `DOORBELL@PEER_BASE` send for a +/// child, so the child's [`kick_parent`] reaches the same thread this one's +/// does: a [`DoorbellWorker`]'s wake end, or one inherited the same way. +/// +/// `None` when this thread holds no such send, or holds one it may not derive +/// from (a band peer's notice-only end, [`spmd`](crate::spmd)'s mesh): an +/// inheritance this thread cannot make is not offered, so it never fails the +/// spawn it would have rode on. The arena record decides, which is the +/// kernel's own word for the slot's kind and rights. +#[cfg(target_os = "none")] +pub(crate) fn wake_endowment() -> Option { + use kore_abi::cap::{CapRecord, ResourceKind}; + let rt = crate::entry::thread_context()?; + let send = rt + .bootstrap() + .handle(role::DOORBELL, crate::futex::PEER_BASE)?; + let record_va = crate::entry::geometry::slot_record_va(send)?; + // SAFETY: the slot alias is this thread's kernel-authored, read-only view + // of its own arena record for a slot its boot block names. + let record = unsafe { core::ptr::read_volatile(record_va as *const CapRecord) }; + if record.kind != ResourceKind::ChanSend as u16 || !record.rights.contains(WAKE_SEND_RIGHTS) { + return None; + } + Some(WorkerCap { + role: role::DOORBELL, + index: crate::futex::PEER_BASE, + subject: send, + kind: ResourceKind::ChanSend as u16 as u8, + rights: WAKE_SEND_RIGHTS, + }) +} + +impl WorkerCap { + /// **Channel space, endowed** — `role::CHANNEL_SPACE@0`: a revocable + /// child of the mint grant `space`, so the thread can mint the channels + /// that wake the threads *it* founds (a `DoorbellWorker` of its own, or + /// the [`thread_controller`](crate::thread_controller) a C program's + /// `pthread_create` starts on its main thread). The parent keeps its + /// grant; revoking it ends the child's. + pub fn channel_space(space: SlotRef) -> Self { + use kore_abi::cap::ResourceKind; + Self { + role: role::CHANNEL_SPACE, + index: 0, + subject: space, + kind: ResourceKind::ChannelSpace as u16 as u8, + rights: crate::cap::CHANNEL_SPACE_RIGHTS, + } + } +} + +/// [`WorkerCap::channel_space`] over this thread's own `role::CHANNEL_SPACE@0` +/// — `None` when it holds no mint grant. +#[cfg(target_os = "none")] +pub fn channel_space_endowment() -> Option { + let rt = crate::entry::thread_context()?; + Some(WorkerCap::channel_space( + rt.bootstrap().handle(role::CHANNEL_SPACE, 0)?, + )) +} + +/// Off metal no thread founds another. #[cfg(not(target_os = "none"))] -pub(crate) fn heap_endowment() -> Option { +pub fn channel_space_endowment() -> Option { None } +/// This thread's **own kick**: a send into its own doorbell it may derive +/// from, as a slot — `DOORBELL@`[`OWN_KICK`](crate::futex::OWN_KICK), or for +/// thread 0 the founder-carved self-kick at `DOORBELL@PEER_BASE`. +/// +/// `None` when it holds none, or one it may not derive from: the authority a +/// thread hands a peer so the peer can wake it must itself be derivable. The +/// arena record decides, as for [`wake_endowment`]. +#[cfg(target_os = "none")] +pub(crate) fn own_kick() -> Option { + use kore_abi::cap::{CapRecord, ResourceKind}; + let rt = crate::entry::thread_context()?; + let boot = rt.bootstrap(); + let send = boot + .handle(role::DOORBELL, crate::futex::OWN_KICK) + .or_else(|| { + (crate::entry::geometry::shard_index() == Some(0)) + .then(|| boot.handle(role::DOORBELL, crate::futex::PEER_BASE)) + .flatten() + })?; + let record_va = crate::entry::geometry::slot_record_va(send)?; + // SAFETY: the slot alias is this thread's kernel-authored, read-only view + // of its own arena record for a slot its boot block names. + let record = unsafe { core::ptr::read_volatile(record_va as *const CapRecord) }; + (record.kind == ResourceKind::ChanSend as u16 && record.rights.contains(WAKE_SEND_RIGHTS)) + .then_some(send) +} + /// Build exactly the capability namespace a spawned worker receives. /// /// This stays target-independent so host tests can pin a subtle shared-space @@ -694,7 +809,12 @@ fn push_worker_thread_blocks(gblock: &mut GeomBlock, local: Window) -> bool { /// view. [`spawn_with`] when the thread must hold capabilities. #[cfg(target_os = "none")] pub fn spawn(entry: fn()) -> Option { - spawn_entry(ThreadEntry::Plain(entry), &[]) + spawn_entry( + ThreadEntry::Plain(entry), + &[], + WorkerStack::Band, + WorkerArena::Task, + ) } // (The per-worker `spawn_worker` shim retired with G5: workers found as a @@ -711,7 +831,158 @@ pub fn spawn(entry: fn()) -> Option { /// into [`exit`]`(0)`. #[cfg(target_os = "none")] pub fn spawn_with(entry: fn(), endow: &[WorkerCap]) -> Option { - spawn_entry(ThreadEntry::Plain(entry), endow) + spawn_entry( + ThreadEntry::Plain(entry), + endow, + WorkerStack::Band, + WorkerArena::Task, + ) +} + +/// Where a spawned worker's stack lives. +/// +/// A plain thread's stack is its band slot by default: `STACK_BYTES` +/// (64 KiB) live under a guard page, tiled by [`MAX_THREADS`] — what a +/// service worker's bounded frames need. A synchronous engine whose +/// recursion depth follows its *input* — a C compiler's expression and block +/// nesting — needs more than a slot holds, so it asks for a **guarded +/// strip** of the task's stack window instead (`res/SPMD.md` §8's +/// kernel-thread class, the shape SPMD gang workers already take: usable +/// pages with one guard granule below, mapped by the founding and given +/// back by `join`). Overflow takes a fault either way, never a +/// neighbour's memory. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum WorkerStack { + /// The thread's band slot. + Band, + /// A guarded strip of at least this many bytes, rounded up to whole + /// pages. The strip comes from the task's one stack window (16 MiB of + /// reservation, shared with SPMD gangs); a request the window cannot + /// place — or a zero-byte one — refuses the spawn. + Bytes(usize), +} + +/// How many capability slots a spawned thread's arena holds: the one sizing +/// question a founding leaves to its founder (`FoundReq.arena_slots`, which +/// the kernel accepts in `1..=ARENA_SLOTS`). +/// +/// Every thread's arena holds its boot names, the runtime's serve scratch, +/// report and carve slots, and whatever it reserves while it runs. A thread +/// that **founds** holds more, for as long as each child lives: the child's +/// handle and the regions its stack, runtime block and boot page live in +/// (what [`join`] revokes), and, while a founding is in flight, that +/// founding's whole result plan. A thread that founds a gang, or a thread +/// controller that founds a process's threads, therefore needs an arena that +/// scales with its children, the case the kernel's founding granule is sized +/// for ("founders are the arena's real sizing case", `ARENA_SLOTS`). A thread +/// controller founded with a task's 64 slots held two live threads; with the +/// founding granule it holds over two dozen, within a couple of the band +/// plan's own limit ([`MAX_THREADS`]). +/// +/// Spawned arenas cannot grow: only thread 0 carries the extension window, so +/// the size is chosen once, here. +#[cfg(any(test, target_os = "none"))] +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum WorkerArena { + /// A thread that founds nothing: [`TASK_ARENA_SLOTS`]. + Task, + /// A thread that founds others: the founding granule, + /// [`ARENA_SLOT_CEILING`](crate::entry::ARENA_SLOT_CEILING). + Founder, +} + +/// A task's arena: one word of the runtime's slot ledger. +/// +/// It holds the widest endowment a worker can take, the serve scratch, the +/// report and carve slots behind it, and the short-lived reservations of a +/// thread that founds nothing (a channel mint, a join landing). The assert +/// below proves the first three fit; a widening of [`MAX_WORKER_CAPS`] or of +/// the serve scratch that broke that would fail the build here, not a +/// founding at run time. +#[cfg(any(test, target_os = "none"))] +pub(crate) const TASK_ARENA_SLOTS: u32 = u64::BITS; + +#[cfg(any(test, target_os = "none"))] +const _: () = { + // The highest boot name a worker can carry, then the runtime layout the + // entry derives after it: serve scratch, the report slot, the carve slot. + let top = child_slot::ENDOW_BASE as usize + MAX_WORKER_CAPS; + let carve = top + crate::entry::SERVE_SCRATCH_SLOTS + 2; + assert!(carve < TASK_ARENA_SLOTS as usize); + assert!(TASK_ARENA_SLOTS <= crate::entry::ARENA_SLOT_CEILING); + // Both sizes fit the per-thread slot alias band: the slot alias of a + // founder's arena is the band's whole width. + assert!(kore_abi::execution::alias_bytes(crate::entry::ARENA_SLOT_CEILING) <= SLOTS_BYTES); +}; + +#[cfg(any(test, target_os = "none"))] +impl WorkerArena { + /// The `FoundReq.arena_slots` this arena is founded with. + pub(crate) const fn slots(self) -> u32 { + match self { + Self::Task => TASK_ARENA_SLOTS, + Self::Founder => crate::entry::ARENA_SLOT_CEILING, + } + } +} + +/// The regions a founder provisions for each thread: the stack, the runtime +/// block and the boot staging page (a TLS block is one more, apart). +const PROVISIONED_REGIONS: usize = 3; + +/// The provisioned regions a founder holds until [`join`] revokes them: the +/// stack and the runtime block ([`Thread`]'s `regions`). The boot staging +/// page is the founding's delegation source: the commit moves its page into +/// the child and retires its founder record, so its name is free from then +/// on, a dead record any later landing or result replaces. +const HELD_REGIONS: usize = 2; + +/// The staged space a thread's `Found` consumes, whose founder name stays +/// spent until [`join`]'s reap. The commit moves the space into the child but +/// scrubs from the founder only what it delivers and delegates, so the +/// founder's record of the space stands — an `AddrSpace` with every right, +/// bound to nothing the founder holds — until the reap retires it. A name +/// returned before that is not vacant: the next post landing there is refused +/// as occupied, and the next result written there overwrites a live record. +const STAGED_SPACE: usize = 1; + +/// The founding results a committed `Found` scrubs from the founder, beside +/// one per endowment: the child's delivered view and the boot delegation's. +const SCRUBBED_RESULTS: usize = 2; + +/// The founding names a committed `Found` leaves without a live record, +/// beside one per endowment: the consumed staging page and the scrubbed +/// results ([`release_committed`]). +const RELEASED_AT_COMMIT: usize = PROVISIONED_REGIONS - HELD_REGIONS + SCRUBBED_RESULTS; + +/// [`Thread`]'s `spent`: the staged space, and what a gang founding that did +/// not commit whole could not return. +const SPENT_CAPACITY: usize = STAGED_SPACE + RELEASED_AT_COMMIT + MAX_WORKER_CAPS; + +/// Arena names one thread's founding holds in its founder while it is in +/// flight, beside one per endowment: its provisioned regions, the staged +/// space, the results the commit scrubs, and the handle. One more with a TLS +/// block. +#[cfg_attr(all(not(target_os = "none"), not(test)), allow(dead_code))] +pub(crate) const FOUNDING_SLOTS: usize = PROVISIONED_REGIONS + STAGED_SPACE + SCRUBBED_RESULTS + 1; + +/// Arena names a founded thread keeps in its founder until [`join`]: its +/// handle, its held regions and its staged space (one more with a TLS block). +/// The rest of the founding's plan returns at commit ([`release_committed`]). +#[cfg_attr(all(not(target_os = "none"), not(test)), allow(dead_code))] +pub(crate) const RETAINED_SLOTS: usize = 1 + HELD_REGIONS + STAGED_SPACE; + +// Every founding name is kept until the reap or returned at commit, never +// both and never neither: the launch budgets ([`crate::spmd`]) count the two +// halves apart. +const _: () = assert!(FOUNDING_SLOTS == RETAINED_SLOTS + RELEASED_AT_COMMIT); + +/// The arena names a gang session founding `workers` threads holds at its +/// peak, just before its founding commits: every worker's claims and every +/// endowment result. +#[cfg_attr(all(not(target_os = "none"), not(test)), allow(dead_code))] +pub(crate) const fn gang_founding_slots(workers: usize, tls: bool, endowments: usize) -> usize { + workers * (FOUNDING_SLOTS + tls as usize) + endowments } /// A worker that can **park and be kicked**, and kick back: two channels @@ -719,7 +990,9 @@ pub fn spawn_with(entry: fn(), endow: &[WorkerCap]) -> Option { /// doorbell (its recv; the parent keeps the send that kicks it) and a /// **wake** channel the other way (the worker holds the send at /// `DOORBELL@PEER_BASE`; the parent parks on the recv) — plus a heap -/// position over the task's supply so it may allocate. +/// position over the task's supply so it may allocate, and the worker's +/// own kick, a derivable send into its doorbell it can hand a thread it +/// founds. /// /// The wake channel exists because a launched task holds no self-kick: /// the founder carves `DOORBELL@2` for service rows that declare workers, @@ -795,6 +1068,19 @@ impl DoorbellWorker { } } +/// The calling thread's C `errno` word — what a C runtime's +/// `__errno_location` answers. +/// +/// Per thread, as ISO C requires (C11 7.5p2): the word lives in this +/// thread's runtime block rather than an ELF TLS block, so an image that +/// carries C code needs no TLS segment for it. Zero when the thread starts. +/// The pointer is valid for the calling thread's life, and only that thread +/// may read or write through it. +#[cfg(target_os = "none")] +pub fn c_errno() -> *mut i32 { + crate::entry::geometry::c_errno() +} + /// Worker side of [`DoorbellWorker`]: post one notice into the parent's /// wake channel (the send founded at `DOORBELL@PEER_BASE`). `false` when /// this thread holds no such send or the post was refused — the latter @@ -815,11 +1101,31 @@ pub fn kick_parent() -> bool { } /// Spawn `entry` on a worker founded holding its own doorbell, a send -/// into thread 0's doorbell, a heap position, and `extra`. `None` when -/// the task holds no `CHANNEL_SPACE` (its profile lacks `channels`), the -/// mint or a derive refuses, or [`spawn_with`] does. +/// into thread 0's doorbell, its own kick ([`own_kick`]), a heap position, +/// and `extra`. `None` when the task holds no `CHANNEL_SPACE` (its profile +/// lacks `channels`), the mint or a derive refuses, or [`spawn_with`] does. +/// +/// The worker takes a founder's arena, the founding granule of capability +/// slots rather than a task's 64: a doorbell worker is where a host runs a +/// program's own code (libkore's C `main`, a +/// language engine), and that code founds threads of its own — its first +/// `pthread_create` starts the [thread controller](crate::thread_controller) +/// and an SPMD dispatch founds a gang, each holding the founder's arena for +/// as long as its children live. #[cfg(target_os = "none")] pub fn spawn_with_doorbell(entry: fn(), extra: &[WorkerCap]) -> Option { + spawn_with_doorbell_on(entry, extra, WorkerStack::Band) +} + +/// [`spawn_with_doorbell`] on the stack `stack` names — a guarded strip +/// for an engine whose depth its input decides ([`WorkerStack`]). `None` +/// also when the stack window cannot place the strip. +#[cfg(target_os = "none")] +pub fn spawn_with_doorbell_on( + entry: fn(), + extra: &[WorkerCap], + stack: WorkerStack, +) -> Option { use kore_abi::cap::{ResourceKind, Rights}; use crate::cap::verbs; @@ -854,7 +1160,8 @@ pub fn spawn_with_doorbell(entry: fn(), extra: &[WorkerCap]) -> Option caps.len() { + const OWN: usize = 4; + if OWN + extra.len() > caps.len() { let _ = pair.send.try_retire(); let _ = pair.root.try_retire(); let _ = wake.send.try_retire(); @@ -869,16 +1176,34 @@ pub fn spawn_with_doorbell(entry: fn(), extra: &[WorkerCap]) -> Option Option Option { - spawn_entry(ThreadEntry::Argument(entry, argument), endow) + spawn_entry( + ThreadEntry::Argument(entry, argument), + endow, + WorkerStack::Band, + WorkerArena::Task, + ) +} + +/// [`spawn_with_arg`] founding a thread that founds others, with a founder's +/// arena ([`WorkerArena::Founder`]): the thread controller, and the threads +/// it founds, any of which may found a gang or ask for more threads. +#[cfg(target_os = "none")] +pub(crate) fn spawn_founder_with_arg( + entry: fn(usize), + argument: usize, + endow: &[WorkerCap], +) -> Option { + spawn_entry( + ThreadEntry::Argument(entry, argument), + endow, + WorkerStack::Band, + WorkerArena::Founder, + ) } #[cfg(not(target_os = "none"))] pub fn spawn_with_arg(_entry: fn(usize), _argument: usize, _endow: &[WorkerCap]) -> Option { None } +/// Return to the slot ledger the founding names a committed `Found` left +/// without a live record. +/// +/// A `Found` that commits delivers the child's view and every endowment out +/// of the founder (the delivery map scrubs each one's founder slot), clears +/// the founder's delegated `Region` results, the boot page's among them, and +/// retires the tag of the staging page its delegation consumed. No live +/// record is left under these names, and none can have been copied: the +/// founder was inside its own trap from apply to scrub. They are free the +/// moment the commit reports `Ok`, and only that proves it: a refused or +/// unreadable commit keeps its names (the founding's leak-on-refusal rule), +/// as a gang founding that did not commit whole keeps them in its tickets for +/// `join`. The staged space is not among them: the commit consumes it without +/// scrubbing its founder record ([`STAGED_SPACE`]). +#[cfg(target_os = "none")] +fn release_committed(slots: impl IntoIterator) { + for slot in slots { + crate::entry::geometry::release_cap_slot(slot); + } +} + /// The one founding every public spawn shape resolves to — the entry /// shape is the only thing the shapes disagree on, and it rides the /// start record untouched by everything below. #[cfg(target_os = "none")] -fn spawn_entry(entry: ThreadEntry, endow: &[WorkerCap]) -> Option { +fn spawn_entry( + entry: ThreadEntry, + endow: &[WorkerCap], + stack: WorkerStack, + arena: WorkerArena, +) -> Option { if endow.len() > MAX_WORKER_CAPS { return None; } @@ -927,7 +1300,7 @@ fn spawn_entry(entry: ThreadEntry, endow: &[WorkerCap]) -> Option { // ordinary slot exhaustion before spending a band or provisioning memory, // and leave a report slot available to poll/join existing children. // Session A owns three regions (+ TLS); B owns four results + endowments. - let slots_needed = 3 + usize::from(template.is_some()) + 4 + endow.len() + 1; + let slots_needed = FOUNDING_SLOTS + usize::from(template.is_some()) + endow.len() + 1; if crate::entry::geometry::available_cap_slots() < slots_needed { return None; } @@ -952,17 +1325,22 @@ fn spawn_entry(entry: ThreadEntry, endow: &[WorkerCap]) -> Option { w.map = p.remaining; } - // ── The stack: an arena strip for SPMD workers (G4b), the band - // slot for plain threads. Carved here and — exactly like the + // ── The stack: an arena strip for SPMD workers (G4b) and for a + // plain thread that asked for one ([`WorkerStack::Bytes`]), the + // band slot otherwise. Carved here and — exactly like the // band index above — left spent if a later founding step // refuses: the teardown that gives both back is join's, never // spawn's, and the leak-on-refusal is the safe direction for // the same reason the band's is. ── - let strip = match &entry { - ThreadEntry::Spmd(..) => { + let strip = match (&entry, stack) { + (ThreadEntry::Spmd(..), _) => { Some(crate::spmd::STACKS.carve_guarded(crate::spmd::STACK_GRANULES)?) } - ThreadEntry::Plain(_) | ThreadEntry::Argument(..) => None, + (ThreadEntry::Plain(_) | ThreadEntry::Argument(..), WorkerStack::Band) => None, + (ThreadEntry::Plain(_) | ThreadEntry::Argument(..), WorkerStack::Bytes(bytes)) => { + let granules = u32::try_from(bytes.div_ceil(crate::stack_arena::GRANULE)).ok()?; + Some(crate::spmd::STACKS.carve_guarded(granules)?) + } }; let (stack_base, stack_len) = match &strip { Some(s) => (s.base(), s.bytes() as u64), @@ -1034,7 +1412,7 @@ fn spawn_entry(entry: ThreadEntry, endow: &[WorkerCap]) -> Option { // ── Stamp the boot page (at its staging VA — the delegation moves // it to `boot_va`, which is what the child receives as `arg`). ── - let arena_slots: u32 = 64; + let arena_slots = arena.slots(); if let (Some(t), Some(p)) = (template, tls) { unsafe { t.initialize(p.mapping.base()) }; } @@ -1098,22 +1476,25 @@ fn spawn_entry(entry: ThreadEntry, endow: &[WorkerCap]) -> Option { if committed.status() != kore_abi::session::Status::Ok { return None; } - // The join ticket: what the thread cost, in the order `join` must - // give it back. - let mut spent = [space_res; 3 + MAX_WORKER_CAPS]; - spent[1] = view_res; - spent[2] = boot_res; - for (i, e) in staged[..endow.len()].iter().enumerate() { - spent[3 + i] = e.result; - } + // The founding's plan is spent, and the names it left without a live + // record come back now: the commit moved the view and every endowment + // into the child, scrubbing their founder slots, cleared the boot + // delegation's, and consumed the staging page (`release_committed`). The + // staged space's record stands until the reap ([`STAGED_SPACE`]), so its + // name rides the join ticket beside what `join` revokes. + release_committed( + [stage_res, view_res, boot_res] + .into_iter() + .chain(staged[..endow.len()].iter().map(|e| e.result)), + ); Some(Thread { owner, index, handle: handle_res, - provisioned: [stack_res, tblock_res, stage_res], + regions: [stack_res, tblock_res], tls_region, - spent, - spent_len: (3 + endow.len()) as u8, + spent: [space_res; SPENT_CAPACITY], + spent_len: STAGED_SPACE as u8, strip, }) } @@ -1317,6 +1698,19 @@ pub(crate) fn spawn_gang_workers( if star.is_some() && n > crate::star::MESH_CEILING { return false; } + // Refuse a session this arena cannot finish before claiming anything: + // past here a refusal leaks what it claimed. A planned endowment is the + // star's mesh closed with the heap; an explicit one is as given. + let endowments = if endows.is_empty() { + n * (star.map_or(0, |plan| plan.mesh_caps(n)) + usize::from(heap.is_some())) + } else { + endows.iter().map(|caps| caps.len()).sum() + }; + if crate::entry::geometry::available_cap_slots() + < gang_founding_slots(n, template.is_some(), endowments) + { + return false; + } let Some(self_space) = crate::entry::geometry::self_space_slot() else { return false; }; @@ -1339,7 +1733,8 @@ pub(crate) fn spawn_gang_workers( tls_region: Option, } let mut claims: [Option; MAX_SESSION_FOUNDS] = [const { None }; MAX_SESSION_FOUNDS]; - let arena_slots: u32 = 64; + // A gang worker runs a region's body and founds nothing. + let arena_slots = WorkerArena::Task.slots(); let transport = TrapTransport::new(); // ── Session A′: provision every worker's three pages, one commit. ── let mut a = SessionBuilder::new(); @@ -1609,23 +2004,35 @@ pub(crate) fn spawn_gang_workers( let founded_all = committed.status() == kore_abi::session::Status::Ok; // Mint every ticket now that the founding committed: whichever // workers exist are joinable, and a vacant handle's join refuses in - // the caller's abort arm rather than orphaning anyone. + // the caller's abort arm rather than orphaning anyone. A whole commit + // returns what it left of each worker's founding without a live record + // now (`release_committed`); a partial one leaves those names to the + // tickets, for the join that learns which workers exist. Either way the + // staged space's name waits for the reap ([`STAGED_SPACE`]). for (k, claim) in claims.iter_mut().take(n).enumerate() { let Some(c) = claim.take() else { return false }; - let mut spent = [c.space_res; 3 + MAX_WORKER_CAPS]; - spent[1] = c.view_res; - spent[2] = c.boot_res; - for (i, e) in endow_store[k][..endow_lens[k]].iter().enumerate() { - spent[3 + i] = e.result; - } + let endowed = &endow_store[k][..endow_lens[k]]; + let committed = [c.stage_res, c.view_res, c.boot_res] + .into_iter() + .chain(endowed.iter().map(|e| e.result)); + let mut spent = [c.space_res; SPENT_CAPACITY]; + let spent_len = if founded_all { + release_committed(committed); + STAGED_SPACE + } else { + for (name, slot) in spent[STAGED_SPACE..].iter_mut().zip(committed) { + *name = slot; + } + STAGED_SPACE + RELEASED_AT_COMMIT + endowed.len() + }; out[k] = Some(Thread { owner, index: c.index, handle: c.handle_res, - provisioned: [c.stack_res, c.tblock_res, c.stage_res], + regions: [c.stack_res, c.tblock_res], tls_region: c.tls_region, spent, - spent_len: (3 + endow_lens[k]) as u8, + spent_len: spent_len as u8, strip: Some(c.strip), }); } @@ -1827,6 +2234,13 @@ pub fn exit(code: ExitCode) -> ! { let Some(rt) = crate::entry::thread_context() else { crate::entry::refuse(); }; + // A thread whose end is awaited announces it first ([`on_exit`]): every + // way a thread ends that runs code — its entry returning, a panic, an + // explicit exit — comes through here. Taken, so a hook that itself ends + // the thread runs once. + if let Some((call, argument)) = crate::entry::geometry::take_exit_hook() { + call(argument); + } let Some(slot) = rt.bootstrap().handle(role::SELF, 0) else { crate::entry::refuse(); }; @@ -1852,6 +2266,22 @@ pub fn exit(_code: ExitCode) -> ! { crate::entry::refuse() } +/// Have this thread's [`exit`] run `call(argument)` first, on this thread, +/// before the exit commits — how the thread that reaps it learns it is +/// ending without polling for it (the +/// [`thread_controller`](crate::thread_controller)'s children). The last hook +/// set stands. `call` runs after the thread's own code, so it must not block, +/// and must not end the thread itself. +/// +/// The kernel's exit commits after the hook returns, so a reaper woken by +/// the hook can still find the thread live for a moment; it parks on the +/// thread's end for that moment (`futex::wait_or_end`), which no store could +/// announce. +#[cfg(target_os = "none")] +pub(crate) fn on_exit(call: fn(usize), argument: usize) { + crate::entry::geometry::set_exit_hook(call, argument); +} + /// One caller-owned report landing reserved for a complete worker-join fold. /// /// The slot is affine for the same reason a [`Thread`] is: once worker @@ -1908,8 +2338,9 @@ impl Drop for JoinScratch { /// 2. **the parent-held regions** — stack, runtime block, boot staging. /// These are the parent's precisely so a dead thread cannot keep its /// own stack alive; -/// 3. **the arena slots** — the founding results the kernel scrubbed, plus -/// the handle; +/// 3. **the arena slots** — the held regions, the staged space, whose +/// founder record the reap has just retired, whatever a partial gang +/// commit left in the ticket, and the handle; /// 4. **the band index — last.** Every window in that band is now /// unmapped, so the next [`spawn`] can be handed it. Released any /// earlier and a new thread would map its stack over a corpse's: the @@ -1978,21 +2409,21 @@ pub(crate) fn join_reserved_at( }); } // The stack and runtime-block regions are parent-held and die here. - // The STAGE region is deliberately NOT presented: the founding's + // The STAGE region is deliberately not among them: the founding's // delegate consumed it (the boot page re-rooted under the child, its // parent record retired with the move), so revoking it refuses the // WHOLE session as `Revoked` — which un-revokes the handle too and // silently leaves `torn_down` false. That exact shape leaked every // joined thread's strip and band index since #407 (bounded, and - // invisible until the G7 retirement witness demanded a real reap); - // the slot itself is still released below, corpse and all. - for slot in [ - Some(thread.provisioned[0]), - Some(thread.provisioned[1]), - thread.tls_region, - ] - .into_iter() - .flatten() + // invisible until the G7 retirement witness demanded a real reap). + // Its name came back when the founding committed, or rides `spent` + // from a gang founding that did not commit whole. + for slot in thread + .regions + .into_iter() + .map(Some) + .chain([thread.tls_region]) + .flatten() { if b.revoke(slot).is_err() { return Err(ThreadJoinError { @@ -2019,7 +2450,7 @@ pub(crate) fn join_reserved_at( if let Some(slot) = thread.tls_region { crate::entry::geometry::release_cap_slot(slot); } - for slot in thread.provisioned { + for slot in thread.regions { crate::entry::geometry::release_cap_slot(slot); } for slot in &thread.spent[..thread.spent_len as usize] { @@ -2187,6 +2618,19 @@ fn join_park( } } +/// [`Thread::poll`] through a landing reserved for it: a repeat observer — +/// the [`thread_controller`](crate::thread_controller), between a child's +/// announced end and its reap — probes without spending an arena slot each +/// time, and so without the one refusal that is about the observer rather +/// than the thread. +#[cfg(target_os = "none")] +pub(crate) fn poll_reserved( + thread: &Thread, + scratch: &JoinScratch, +) -> Result { + poll_task_handle_in(&TrapTransport::new(), thread.handle, scratch.report) +} + /// Issue one unflagged join probe. `WouldBlock` is the one positive live-task /// result; unlike the blocking path, an OCC conflict or unrelated park status /// is surfaced so a nonblocking caller controls its own retry policy. diff --git a/native/crates/rt/src/thread_controller.rs b/native/crates/rt/src/thread_controller.rs index 65c6289..764447a 100644 --- a/native/crates/rt/src/thread_controller.rs +++ b/native/crates/rt/src/thread_controller.rs @@ -4,13 +4,56 @@ //! records. Only the controller touches native tickets. A detached record is //! reusable only after native exit and successful reclamation. Storage is //! supplied by the process and must outlive every client and child. +//! +//! # Every wait is a futex park +//! +//! Nothing here polls. The controller thread waits on its hub word +//! ([`futex::Hub`]), and everything that needs it — a create request, a +//! child's end, a wake relayed through it — stores, then wakes that word, the +//! order [`futex`] requires. A client waits on its record's event word, which +//! the controller bumps and wakes after every transition a client can wait +//! for. A child announces its end through the hook [`thread::exit`] runs, on +//! every way a thread ends that runs code; the kernel's exit commits a moment +//! later, an edge no store can announce, so the controller reaps through a +//! park whose join arm watches that child beside its word +//! ([`futex::wait_or_end`]). +//! +//! # The controller is its threads' hub +//! +//! The star's rule (`res/SPMD.md` §3) for threads that come and go. The +//! starter mints the controller's doorbell, keeps a send into it (an adopted +//! peer), and founds it holding that doorbell, its own kick, the starter's +//! kick, the host's wake, the heap and a channel-space grant. For each child +//! the controller mints a doorbell, keeps the send (an adopted peer, withdrawn +//! before the child's band can be reused), and founds the child holding it, a +//! send into the controller, the starter's kick, the host's wake and the heap. +//! Children and the starter relay through the controller's hub, so a wake +//! between two threads neither can kick directly still arrives: the controller +//! sweeps ([`futex::sweep`]). +//! +//! Both the controller and every child take a founder's arena (the +//! founding granule of capability slots): the controller holds each live child's handle, +//! its stack and runtime-block regions, its staged space and the root and send +//! of its doorbell until the reap, a whole process's worth of them, and a +//! child may found an SPMD gang of its own. +//! +//! # Degrade, never wedge +//! +//! A starter with no channel space founds a controller without a doorbell: +//! neither it nor its children can park, and each of their waits takes the +//! deadline park every thread can ([`pace::cede`]). A client the controller +//! cannot kick waits in bounded slices ([`UNREACHED_SLICE`]). A thread that +//! cannot wake a parking controller is refused a create, which would +//! otherwise wait on a request nothing announced. +//! +//! [`thread::exit`]: crate::thread::exit +//! [`pace::cede`]: crate::pace::cede use core::cell::UnsafeCell; -use core::sync::atomic::{AtomicU64, AtomicUsize, Ordering}; +use core::sync::atomic::{AtomicU32, AtomicU64, AtomicUsize, Ordering}; +use core::time::Duration; -// Each owner derives the process heap into its children using its own local -// slot. No creator's slot number is ever forwarded across the controller seam. -use super::thread::heap_endowment; +use super::futex::{self, Hub, WaitOutcome}; const FREE: u64 = 0; const BUILD: u64 = 1; @@ -26,6 +69,37 @@ const RETIRED: u64 = 15; const MASK: u64 = 15; const COUNT: usize = super::thread::MAX_THREADS as usize - 2; +/// Lifecycle of the controller thread itself, in [`Controller::started`]: +/// not started, being founded, or started — parking on its doorbell, or +/// without one, taking the deadline park. +const IDLE: u32 = 0; +const STARTING: u32 = 1; +const PARKING: u32 = 2; +#[cfg_attr( + not(target_os = "none"), + expect(dead_code, reason = "only a metal founding starts") +)] +const POLLING: u32 = 3; + +/// How long one park lasts for a client the controller cannot kick — a +/// thread it holds no send into, such as an initial thread launched without +/// a self-kick. Nothing can end such a park early, so its slice is the +/// latency it pays, and this is the cadence those waits paced themselves +/// at before the controller parked (`pace::CEDE_NS`): a client the wake +/// fabric cannot reach is no worse off than it was. Shorter would wake an +/// unreachable joiner more often for nothing; longer would delay it. +const UNREACHED_SLICE: Duration = Duration::from_nanos(crate::pace::CEDE_NS); + +/// How long the controller rides its doorbell after a reap the kernel did +/// not complete — a refused lifecycle observation or reclamation, which no +/// later event is bound to retry. The joiner's own pacing for a join that +/// cannot establish its park (`thread::join_park`, two milliseconds). +#[cfg_attr( + not(target_os = "none"), + expect(dead_code, reason = "reaping is metal-only") +)] +const REAP_RETRY: Duration = Duration::from_millis(2); + /// An identifier within one controller, never a capability-slot number. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub struct Id { @@ -49,14 +123,30 @@ pub enum Error { struct Start { call: fn(usize) -> usize, argument: usize, + /// Read by the controller thread, after the reap. + #[cfg_attr(not(target_os = "none"), allow(dead_code))] retire: Option, + /// The controller this record belongs to, for the child's hooks. + home: usize, } fn empty(_: usize) -> usize { 0 } +/// One managed thread's shared record, on its own interference span: its +/// creator, its joiner, the child and the controller each write it, and a +/// waiting client's monitored spin sleeps on its event word's line, which a +/// neighbour's traffic would end for nothing. The span is the one the +/// barrier isolates against (`barrier::Line`). +#[repr(align(128))] struct Record { state: AtomicU64, + /// Bumped by the controller after each transition a client can wait + /// for; clients park on it. An event count: the state is the truth. + event: AtomicU32, + /// Set by the child's exit hook: it has stopped running its own code + /// and is committing its end. + ending: AtomicU32, start: UnsafeCell, value: AtomicUsize, identity: AtomicUsize, @@ -69,10 +159,13 @@ impl Record { const fn new() -> Self { Self { state: AtomicU64::new(FREE), + event: AtomicU32::new(0), + ending: AtomicU32::new(0), start: UnsafeCell::new(Start { call: empty, argument: 0, retire: None, + home: 0, }), value: AtomicUsize::new(0), identity: AtomicUsize::new(0), @@ -85,13 +178,34 @@ impl Record { Ordering::Release, ); } + /// Publish a transition a client may be parked on: the state was + /// stored; now the event moves, then its waiters are woken. + #[cfg_attr(not(target_os = "none"), allow(dead_code))] + fn announce(&self) { + self.event.fetch_add(1, Ordering::SeqCst); + futex::wake_all(&self.event); + } } /// Named process-owned storage. Start once during application startup and /// share its static reference with workers. This consumes one native thread /// for the process lifetime. No native ticket escapes its owner. pub struct Controller { - started: AtomicUsize, + /// [`IDLE`], [`STARTING`], [`PARKING`] or [`POLLING`]; clients park on + /// it while the controller is being founded. + started: AtomicU32, + /// What the controller thread waits on, and what its threads relay + /// through. + hub: Hub, + /// The controller thread's band, plus one (`0` until it is founded). + band: AtomicU32, + /// The band of the thread that started the controller, plus one. + #[cfg_attr(not(target_os = "none"), allow(dead_code))] + starter: AtomicU32, + /// The bands whose doorbells the controller can kick, a bit each: the + /// clients its wakes reach. A child's bit is set once it is founded and + /// cleared before its band can be reused. + reach: AtomicU32, records: [Record; COUNT], } impl Default for Controller { @@ -99,35 +213,50 @@ impl Default for Controller { Self::new() } } +// The reach bitmap has one bit per band. +const _: () = assert!(super::thread::MAX_THREADS <= u32::BITS); + impl Controller { pub const fn new() -> Self { Self { - started: AtomicUsize::new(0), + started: AtomicU32::new(IDLE), + hub: Hub::new(), + band: AtomicU32::new(0), + starter: AtomicU32::new(0), + reach: AtomicU32::new(0), records: [const { Record::new() }; COUNT], } } /// Start the owner. Concurrent/repeated calls do not create another owner. + /// + /// The calling thread becomes the controller's starter: it mints the + /// controller's wake channel from its channel-space grant and holds that + /// channel's root, as a `DoorbellWorker`'s parent holds its worker's, so + /// start from a thread that outlives the controller's use — the initial + /// thread, or a program's main. pub fn start(&'static self) -> Result<(), Error> { self.started - .compare_exchange(0, 1, Ordering::AcqRel, Ordering::Acquire) + .compare_exchange(IDLE, STARTING, Ordering::AcqRel, Ordering::Acquire) .map_err(|_| Error::Busy)?; - let heap = heap_endowment(); - match super::thread::spawn_with_arg( - controller_entry, - self as *const Self as usize, - heap.as_slice(), - ) { - Some(ticket) => { - // This owner deliberately lives until process teardown. Managed - // children never use native detach: their tickets are retained. - ticket.detach(); - self.started.store(2, Ordering::Release); - Ok(()) - } - None => { - self.started.store(0, Ordering::Release); - Err(Error::Unavailable) + let mode = self.found(); + self.started.store(mode.unwrap_or(IDLE), Ordering::Release); + futex::wake_all(&self.started); + mode.map(|_| ()).ok_or(Error::Unavailable) + } + + /// How the started controller waits, waiting out its founding; `None` + /// when it is not started. + fn mode(&self) -> Option { + loop { + match self.started.load(Ordering::Acquire) { + IDLE => return None, + STARTING => { + // The starter wakes every thread it can when it is done; + // a thread it cannot reach rides its slice. + park(&self.started, STARTING, Some(UNREACHED_SLICE)); + } + mode => return Some(mode), } } } @@ -143,13 +272,19 @@ impl Controller { /// runs once on the controller after native reclamation, including when the /// callback exits without returning. On error custody remains with caller. /// The cleanup must not make blocking calls back into this controller. + /// + /// Call it from a thread that can wake the controller — its starter, or a + /// thread it created. A parking controller sleeps until something wakes + /// it, so any other thread is refused [`Error::Unavailable`] rather than + /// left waiting on a request nothing announced. pub fn create_with_cleanup( &'static self, call: fn(usize) -> usize, argument: usize, retire: Option, ) -> Result { - if self.started.load(Ordering::Acquire) != 2 { + let mode = self.mode().ok_or(Error::Unavailable)?; + if mode == PARKING && !self.wakes_controller() { return Err(Error::Unavailable); } for (slot, record) in self.records.iter().enumerate() { @@ -170,31 +305,54 @@ impl Controller { call, argument, retire, + home: self as *const Self as usize, }; } record.identity.store(0, Ordering::Relaxed); record.value.store(0, Ordering::Relaxed); record.returned.store(0, Ordering::Relaxed); + record.ending.store(0, Ordering::Relaxed); record.state.store(old | REQUEST, Ordering::Release); - loop { - let state = record.state.load(Ordering::Acquire); - if state == old | FAILED { - record.release(old); - return Err(Error::Unavailable); - } - if state != old | REQUEST { - return Ok(Id { - controller: self as *const Self as usize, - slot, - generation: old, - }); - } - pause(); + self.hub.signal(); + let state = self.settle(record, |state| state != old | REQUEST); + if state == old | FAILED { + record.release(old); + return Err(Error::Unavailable); } + return Ok(Id { + controller: self as *const Self as usize, + slot, + generation: old, + }); } Err(Error::Full) } + /// Whether this thread holds a send into the controller's doorbell. + fn wakes_controller(&self) -> bool { + self.band + .load(Ordering::Acquire) + .checked_sub(1) + .is_some_and(futex::reaches) + } + + /// Park until `record`'s state satisfies `done`, and return that state. + fn settle(&self, record: &Record, done: impl Fn(u64) -> bool) -> u64 { + loop { + // The event count before the state: a transition published after + // this load moves the count, which the park re-reads once it is + // registered (`futex`'s order), so none is missed. + let seen = record.event.load(Ordering::SeqCst); + let state = record.state.load(Ordering::Acquire); + if done(state) { + return state; + } + let reached = futex::current_thread() + .is_some_and(|band| self.reach.load(Ordering::SeqCst) & (1 << band) != 0); + park(&record.event, seen, (!reached).then_some(UNREACHED_SLICE)); + } + } + fn record(&self, id: Id) -> Result<&Record, Error> { if id.controller != self as *const Self as usize { return Err(Error::Invalid); @@ -263,9 +421,7 @@ impl Controller { continue; } if next == JOIN { - while record.state.load(Ordering::Acquire) != id.generation | JOIN_READY { - pause(); - } + self.settle(record, |state| state == id.generation | JOIN_READY); } // Acquire completion before reading the value it publishes. The owner // is a different thread from the callback on weakly ordered ISAs. @@ -314,96 +470,511 @@ impl Controller { } } +/// One wait on `word`, still holding `expected`: the futex park, or — on a +/// thread that cannot park at all — the deadline park every thread can take, +/// which a spin hint stands in for only off metal. +fn park(word: &AtomicU32, expected: u32, slice: Option) { + if futex::wait(word, expected, slice) == WaitOutcome::Unsupported { + pause(); + } +} + fn pause() { if !crate::pace::cede() { core::hint::spin_loop(); } } +/// The managed thread's entry: announce its end however it comes, relay +/// through the controller, run the callback. fn child_entry(argument: usize) { // SAFETY: controller storage is static; record cannot be reused until join. let record = unsafe { &*(argument as *const Record) }; #[cfg(target_os = "none")] - record.identity.store( - crate::entry::geometry::thread_owner_identity().unwrap_or(0), - Ordering::Release, - ); + { + crate::thread::on_exit(child_ending, argument); + record.identity.store( + crate::entry::geometry::thread_owner_identity().unwrap_or(0), + Ordering::Release, + ); + } // Acquire the request publication (or the owner's subsequent state) // before reading callback storage. It remains immutable through native join. let _ = record.state.load(Ordering::Acquire); let start = unsafe { *record.start.get() }; + // SAFETY: `home` names the static controller whose record this is. + let _ = futex::adopt_hub(&unsafe { &*(start.home as *const Controller) }.hub); record .value .store((start.call)(start.argument), Ordering::Release); record.returned.store(1, Ordering::Release); } -fn controller_entry(argument: usize) { - // SAFETY: start requires static storage. - let controller = unsafe { &*(argument as *const Controller) }; - let heap = heap_endowment(); - let mut tickets: [Option; COUNT] = [const { None }; COUNT]; - loop { - for (record, ticket) in controller.records.iter().zip(tickets.iter_mut()) { - let state = record.state.load(Ordering::Acquire); - let generation = state & !MASK; - if state & MASK == REQUEST { - *ticket = super::thread::spawn_with_arg( - child_entry, - record as *const Record as usize, - heap.as_slice(), - ); - record.state.store( - generation | if ticket.is_some() { RUNNING } else { FAILED }, - Ordering::Release, - ); +/// The child's exit hook: say the thread is ending, then wake the +/// controller to reap it. +#[cfg(target_os = "none")] +fn child_ending(argument: usize) { + // SAFETY: as in `child_entry`; the record outlives its child. + let record = unsafe { &*(argument as *const Record) }; + record.ending.store(1, Ordering::SeqCst); + // SAFETY: `start` is immutable until the controller has reaped this + // thread, which it cannot do before this hook returns. + let start = unsafe { *record.start.get() }; + let controller = unsafe { &*(start.home as *const Controller) }; + controller.hub.signal(); +} + +#[cfg(target_os = "none")] +mod owner { + //! The controller thread and its founding. + + use core::sync::atomic::Ordering; + + use kore_abi::cap::{ResourceKind, Rights}; + use kore_abi::session::SlotRef; + use kore_abi::task::role; + + use super::{ + COMPLETE, COUNT, Controller, DETACHED, JOIN, JOIN_READY, MASK, PARKING, POLLING, + REAP_RETRY, RUNNING, Record, child_entry, + }; + use crate::cap::{ + CHANNEL_ENDPOINT_RIGHTS, ChanSendToken, ChannelRoot, ChannelSpaceToken, verbs, + }; + use crate::futex::{self, PEER_BASE, WaitOutcome}; + use crate::thread::{self, JoinScratch, Thread, WAKE_SEND_RIGHTS, WorkerCap}; + + /// The most capabilities either founding here endows: the controller's + /// doorbell, its own kick, the heap, the host's wake, a channel-space + /// grant and the starter's kick. + const MAX_ENDOW: usize = 6; + + /// A founding's endowment, filled in order. + struct Endowment { + caps: [WorkerCap; MAX_ENDOW], + len: usize, + } + + impl Endowment { + fn new() -> Self { + let vacant = WorkerCap { + role: 0, + index: 0, + subject: SlotRef::from_wire(0), + kind: 0, + rights: Rights::empty(), + }; + Self { + caps: [vacant; MAX_ENDOW], + len: 0, } - let Some(child) = ticket.as_ref() else { - continue; + } + + /// Append `cap`, if any; `true` when it is now in the endowment. + fn push(&mut self, cap: Option) -> bool { + let (Some(cap), Some(at)) = (cap, self.caps.get_mut(self.len)) else { + return false; }; - if !matches!( - child.poll(), - Ok(crate::cap::TaskPollOutcome::Exited(_) | crate::cap::TaskPollOutcome::Reaped) - ) { - continue; + *at = cap; + self.len += 1; + true + } + + fn as_slice(&self) -> &[WorkerCap] { + &self.caps[..self.len] + } + } + + /// A send at `DOORBELL@index`, derived from `subject` with `rights`. + fn send(index: u16, subject: SlotRef, rights: Rights) -> WorkerCap { + WorkerCap { + role: role::DOORBELL, + index, + subject, + kind: ResourceKind::ChanSend as u16 as u8, + rights, + } + } + + /// A thread's own doorbell (`DOORBELL@0`), derived from its channel's root. + fn doorbell(root: &ChannelRoot) -> WorkerCap { + WorkerCap { + role: role::DOORBELL, + index: 0, + subject: root.slot(), + kind: ResourceKind::ChanRecv as u16 as u8, + rights: CHANNEL_ENDPOINT_RIGHTS, + } + } + + /// One thread's wake channel: its root and the send its founder keeps. + /// The founder's own recv is surplus — the thread's recv is a fresh + /// derive of the root at its founding (the star's rule). + struct Wires { + root: ChannelRoot, + kick: ChanSendToken, + } + + impl Wires { + fn mint(space: &ChannelSpaceToken) -> Option { + let pair = verbs::mint_channel(space).ok()?; + let _ = pair.recv.try_retire(); + Some(Self { + root: pair.root, + kick: pair.send, + }) + } + + fn retire(self) { + let _ = self.kick.try_retire(); + let _ = self.root.try_retire(); + } + } + + impl Controller { + /// Found the controller thread from the calling thread: the mode it + /// will wait in, or `None` when it could not be founded. + pub(super) fn found(&'static self) -> Option { + let starter = futex::current_thread()? as u32; + self.starter.store(starter + 1, Ordering::Release); + let rt = crate::entry::thread_context()?; + let space = rt + .bootstrap() + .handle(role::CHANNEL_SPACE, 0) + .and_then(verbs::claim_channel_space); + let wires = space.as_ref().and_then(Wires::mint); + let own_kick = thread::own_kick(); + + let mut endowment = Endowment::new(); + if let Some(wires) = &wires { + endowment.push(Some(doorbell(&wires.root))); + // The controller derives every child's send into itself from + // this one, so it keeps DERIVE. + endowment.push(Some(send( + futex::OWN_KICK, + wires.root.slot(), + WAKE_SEND_RIGHTS, + ))); + } + endowment.push(thread::heap_endowment()); + // Peer 0. For a starter that is thread 0 it is that thread's own + // kick, and the starter is reached through it. + let peer_zero = endowment.push(thread::wake_endowment()); + endowment.push(thread::channel_space_endowment()); + // Any other starter is reached at its own band's peer index. + let reached = if starter == 0 { + own_kick.is_some() && peer_zero + } else { + endowment.push( + own_kick.map(|kick| send(PEER_BASE + starter as u16, kick, WAKE_SEND_RIGHTS)), + ) + }; + let Some(ticket) = thread::spawn_founder_with_arg( + super::controller_entry, + self as *const Self as usize, + endowment.as_slice(), + ) else { + if let Some(wires) = wires { + wires.retire(); + } + return None; + }; + let band = ticket.band_index(); + self.band.store(band + 1, Ordering::Release); + if reached { + self.reach.fetch_or(1 << starter, Ordering::SeqCst); + } + // This owner deliberately lives until process teardown. Managed + // children never use native detach: their tickets are retained. + ticket.detach(); + // The claimed grant is a witness, not a reservation: dropping it + // retires nothing (the star's rule, `star::Wires`). + drop(space); + let Some(wires) = wires else { + return Some(POLLING); + }; + // Wake the controller directly, and relay through it. + if !futex::adopt_peer(band, wires.kick.slot()) { + wires.retire(); + return Some(POLLING); + } + let _ = futex::adopt_hub(&self.hub); + // The channel lives as long as the controller does: its root and + // this thread's send stay in this thread's arena for good. + core::mem::forget(wires); + Some(PARKING) + } + } + + /// A managed child the controller founded and will reap. + struct Child { + thread: Thread, + band: u32, + /// Its doorbell's root and the controller's send into it; `None` + /// when the controller has no channel space to mint them. + wires: Option, + } + + /// The controller thread's own state, on its stack for the process life. + pub(super) struct Owner { + controller: &'static Controller, + band: u32, + /// The grant children's doorbells are minted from. + space: Option, + /// This thread's own kick: every child's send into it derives from it. + kick: Option, + /// The starter's band and the send into it, when the starter is not + /// thread 0 (which children reach as peer 0). + starter: Option<(u32, SlotRef)>, + /// One report landing, reused serially by every probe, join arm and + /// reap: the controller never spends an arena slot to observe. + scratch: Option, + children: [Option; COUNT], + /// The hub's relay count at the last sweep. + swept: u32, + } + + /// What one reap attempt came to. + enum Reap { + /// Nothing left to reap. + Done, + /// The child announced its end and has not committed it. + Pending, + /// The kernel refused an observation or the reclamation. + Failed, + } + + impl Owner { + pub(super) fn new(controller: &'static Controller) -> Self { + let band = futex::current_thread().map_or(u32::MAX, |band| band as u32); + let rt = crate::entry::thread_context(); + let boot = rt.as_ref().map(|rt| rt.bootstrap()); + let space = boot + .as_ref() + .and_then(|boot| boot.handle(role::CHANNEL_SPACE, 0)) + .and_then(verbs::claim_channel_space); + // Derivable, or no child is handed a send into this thread. + let kick = thread::own_kick(); + let starter = controller + .starter + .load(Ordering::Acquire) + .checked_sub(1) + .filter(|&starter| starter != 0) + .and_then(|starter| { + let send = boot + .as_ref()? + .handle(role::DOORBELL, PEER_BASE + starter as u16)?; + Some((starter, send)) + }); + Self { + controller, + band, + space, + kick, + starter, + scratch: JoinScratch::reserve(), + children: [const { None }; COUNT], + swept: controller.hub.relays(), } - match super::thread::join(ticket.take().unwrap()) { - Err(error) => *ticket = Some(error.into_thread()), - Ok(_) => { - // Child can no longer access its context; cleanup also runs - // for abrupt native exit and detached completion. - let start = unsafe { *record.start.get() }; - if let Some(retire) = start.retire { - retire(start.argument); + } + + /// Serve the records for the process's life. + pub(super) fn serve(mut self) -> ! { + let controller = self.controller; + loop { + let seen = controller.hub.word().load(Ordering::SeqCst); + let owed = controller.hub.relays(); + if owed != self.swept { + self.swept = owed; + futex::sweep(); + } + let mut watch = None; + let mut failed = false; + for (slot, record) in controller.records.iter().enumerate() { + let state = record.state.load(Ordering::Acquire); + if state & MASK == super::REQUEST { + self.found_child(slot, record, state & !MASK); } - loop { - let old = record.state.load(Ordering::Acquire); - let next = match old & MASK { - RUNNING => COMPLETE, - JOIN => JOIN_READY, - DETACHED => { - record.release(generation); - break; + if record.ending.load(Ordering::Acquire) != 0 { + match self.reap(slot, record) { + Reap::Done => {} + Reap::Pending => { + watch.get_or_insert(slot); } - _ => unreachable!(), - }; - if record - .state - .compare_exchange( - old, - generation | next, - Ordering::Release, - Ordering::Acquire, - ) - .is_ok() - { - break; + Reap::Failed => failed = true, } } } + let word = controller.hub.word(); + let outcome = match (watch, &self.scratch) { + (Some(slot), Some(scratch)) if !failed => match &self.children[slot] { + Some(child) => futex::wait_or_end( + word, + seen, + None, + child.thread.handle(), + scratch.report_slot(), + ), + None => futex::wait_quiet(word, seen, None), + }, + _ => futex::wait_quiet(word, seen, failed.then_some(REAP_RETRY)), + }; + if outcome == WaitOutcome::Unsupported { + super::pause(); + } } } - pause(); + + /// Found the child `record` asks for, then publish the outcome. + fn found_child(&mut self, slot: usize, record: &'static Record, generation: u64) { + let child = self.spawn(record); + let next = if child.is_some() { + RUNNING + } else { + super::FAILED + }; + self.children[slot] = child; + record.state.store(generation | next, Ordering::Release); + record.announce(); + } + + fn spawn(&mut self, record: &'static Record) -> Option { + let wires = self.space.as_ref().and_then(Wires::mint); + let mut endowment = Endowment::new(); + if let Some(wires) = &wires { + endowment.push(Some(doorbell(&wires.root))); + } + // A child hands these on to no one (the threads it asks for are + // founded here, not by it): notice rights only. + endowment.push( + self.kick + .map(|kick| send(PEER_BASE + self.band as u16, kick, verbs::NOTICE_RIGHTS)), + ); + endowment.push(self.starter.map(|(starter, kick)| { + send(PEER_BASE + starter as u16, kick, verbs::NOTICE_RIGHTS) + })); + endowment.push(thread::heap_endowment()); + endowment.push(thread::wake_endowment()); + let Some(thread) = thread::spawn_founder_with_arg( + child_entry, + record as *const Record as usize, + endowment.as_slice(), + ) else { + if let Some(wires) = wires { + wires.retire(); + } + return None; + }; + let band = thread.band_index(); + if let Some(wires) = &wires + && futex::adopt_peer(band, wires.kick.slot()) + { + self.controller.reach.fetch_or(1 << band, Ordering::SeqCst); + } + Some(Child { + thread, + band, + wires, + }) + } + + /// Reap `slot`'s child if it has ended, then publish its end. + fn reap(&mut self, slot: usize, record: &Record) -> Reap { + if self.scratch.is_none() { + // The landing was not available when this thread started. + self.scratch = JoinScratch::reserve(); + } + let (Some(child), Some(scratch)) = + (self.children[slot].as_ref(), self.scratch.as_mut()) + else { + return if self.children[slot].is_none() { + Reap::Done + } else { + Reap::Failed + }; + }; + match thread::poll_reserved(&child.thread, scratch) { + Ok(crate::cap::TaskPollOutcome::Running) => return Reap::Pending, + Ok(_) => {} + Err(_) => return Reap::Failed, + } + // Withdraw the child's kick before its band can be handed to + // another thread: no wake may present this send, and no client + // at that band may count on this controller reaching it. + futex::forget_peer(child.band); + self.controller + .reach + .fetch_and(!(1 << child.band), Ordering::SeqCst); + let Some(Child { + thread, + band, + wires, + }) = self.children[slot].take() + else { + return Reap::Done; + }; + if let Err(error) = thread::join_reserved(thread, scratch) { + self.children[slot] = Some(Child { + thread: error.into_thread(), + band, + wires, + }); + return Reap::Failed; + } + if let Some(wires) = wires { + wires.retire(); + } + // Child can no longer access its context; cleanup also runs + // for abrupt native exit and detached completion. + let start = unsafe { *record.start.get() }; + if let Some(retire) = start.retire { + retire(start.argument); + } + record.ending.store(0, Ordering::Relaxed); + loop { + let old = record.state.load(Ordering::Acquire); + let generation = old & !MASK; + let next = match old & MASK { + RUNNING => COMPLETE, + JOIN => JOIN_READY, + DETACHED => { + record.release(generation); + break; + } + // Only this thread moves a founded child's record out of + // these three. + _ => break, + }; + if record + .state + .compare_exchange(old, generation | next, Ordering::Release, Ordering::Acquire) + .is_ok() + { + break; + } + } + record.announce(); + Reap::Done + } + } +} + +fn controller_entry(argument: usize) { + // SAFETY: start requires static storage. + let controller = unsafe { &*(argument as *const Controller) }; + #[cfg(target_os = "none")] + owner::Owner::new(controller).serve(); + #[cfg(not(target_os = "none"))] + let _ = controller; +} + +#[cfg(not(target_os = "none"))] +impl Controller { + /// Off metal no thread founds another: the controller never starts. + fn found(&'static self) -> Option { + let _ = child_entry; + let _ = controller_entry; + None } } @@ -456,4 +1027,48 @@ mod tests { ); assert_eq!(controller.join(final_id), Err(Error::Invalid)); } + #[test] + fn a_parked_joiner_is_released_by_the_announced_transition() { + // The joiner's half of the protocol, with a host thread standing in + // for the controller: it moves the record JOIN → JOIN_READY and + // announces it; the joiner returns the value only after that. + let controller = Controller::new(); + let record = &controller.records[0]; + record.state.store(RUNNING, Ordering::Release); + let id = Id { + controller: &controller as *const Controller as usize, + slot: 0, + generation: 0, + }; + std::thread::scope(|scope| { + let joiner = scope.spawn(|| controller.join(id)); + while record.state.load(Ordering::Acquire) != JOIN { + std::thread::yield_now(); + } + record.value.store(5, Ordering::Relaxed); + record.returned.store(1, Ordering::Release); + let before = record.event.load(Ordering::SeqCst); + record.state.store(JOIN_READY, Ordering::Release); + record.announce(); + assert_eq!(record.event.load(Ordering::SeqCst), before.wrapping_add(1)); + assert_eq!(joiner.join().unwrap(), Ok(5)); + }); + assert_eq!(record.state.load(Ordering::Acquire), 16); + } + #[test] + fn an_unstarted_controller_refuses_and_never_waits() { + let controller: &'static Controller = + std::boxed::Box::leak(std::boxed::Box::new(Controller::new())); + assert_eq!(controller.create(|_| 0, 0), Err(Error::Unavailable)); + // Off metal the founding cannot happen; the refusal leaves it idle + // and restartable rather than stuck in its founding state. + assert_eq!(controller.start(), Err(Error::Unavailable)); + assert_eq!(controller.started.load(Ordering::Acquire), IDLE); + assert_eq!(controller.create(|_| 0, 0), Err(Error::Unavailable)); + } + #[test] + fn records_own_their_interference_span() { + assert_eq!(core::mem::align_of::(), 128); + assert_eq!(core::mem::size_of::(), 128); + } } diff --git a/native/crates/window/src/client.rs b/native/crates/window/src/client.rs index bd6892b..2fc2bd0 100644 --- a/native/crates/window/src/client.rs +++ b/native/crates/window/src/client.rs @@ -53,6 +53,9 @@ pub enum ClientError { EventMalformed, /// A menu the encoder refused before anything reached the compositor. Menu(crate::menu::MenuError), + /// A clipboard text past [`crate::clipboard::CLIPBOARD_BYTES`], or one + /// longer than its reader can hold: refused whole, never cut short. + TooLong, } impl ClientError { @@ -946,6 +949,101 @@ impl<'w, 't, T: Transport> WindowClient<'w, 't, T> { } } + /// Put `text` on this window's seat's clipboard (`res/CLIPBOARD.md`), a + /// request plane at a time. Only the seat's focused window may, on a key + /// press, button press or menu choice it was just delivered and has not + /// spent ([`SurfaceRefusal::Denied`] otherwise). An empty text is no + /// copy; one past the clipboard's bound is refused before anything is + /// sent, and one that is not UTF-8 the compositor refuses whole. + /// `OutOfMemory` leaves the clipboard and the gesture as they were, so a + /// retry may succeed. + pub async fn copy(&mut self, text: &[u8]) -> Result<(), ClientError> { + let total = u32::try_from(text.len()) + .ok() + .filter(|&total| total as usize <= crate::clipboard::CLIPBOARD_BYTES) + .ok_or(ClientError::TooLong)?; + if total == 0 { + return Err(ClientError::Protocol); + } + let mut offset = 0_u32; + for piece in text.chunks(crate::clipboard::CHUNK_BYTES.min(self.menu.len())) { + self.menu[..piece.len()].copy_from_slice(piece); + let len = piece.len() as u16; + match self + .surface_request(SurfaceRequest::CopyChunk { offset, len, total }) + .await? + { + SurfaceResponse::Copied { through } if through == offset + u32::from(len) => { + offset = through; + } + _ => return Err(ClientError::Protocol), + } + } + Ok(()) + } + + /// Read this window's seat's clipboard (`res/CLIPBOARD.md`): `begin` + /// learns the text's length before any of it, and may refuse it + /// (`false`: longer than the caller can hold, [`ClientError::TooLong`]); + /// `piece` then takes the text in order, a request plane at a time, and + /// answers whether it wants more (`false` ends the read there: a reader + /// that needs only the text's start reads no more of it). Answers the + /// length, zero when the clipboard holds no text. Gated as [`Self::copy`] + /// is; a copy landing while the read is under way refuses it + /// ([`SurfaceRefusal::Unknown`]) rather than mixing two texts. + pub async fn paste( + &mut self, + begin: impl FnOnce(u32) -> bool, + mut piece: impl FnMut(&[u8]) -> bool, + ) -> Result { + let mut begin = Some(begin); + let mut offset = 0_u32; + let mut buffer = [0_u8; PASTE_STAGE_BYTES]; + loop { + let (at, len, total) = match self.surface_request(SurfaceRequest::Paste { offset }).await? { + SurfaceResponse::Clipboard { offset: at, len, total } => (at, len, total), + SurfaceResponse::ClipboardEmpty if offset == 0 => return Ok(0), + _ => return Err(ClientError::Protocol), + }; + if at != offset || usize::from(len) > self.menu.len() { + return Err(ClientError::Protocol); + } + if let Some(begin) = begin.take() + && !begin(total) + { + return Err(ClientError::TooLong); + } + // The compositor wrote the plane before it answered; read it as + // the shared memory it is, after the reply's acquire. + let plane = shared_plane(self.menu); + for chunk in plane[..usize::from(len)].chunks(buffer.len()) { + for (byte, shared) in buffer.iter_mut().zip(chunk) { + *byte = shared.load(core::sync::atomic::Ordering::Relaxed); + } + if !piece(&buffer[..chunk.len()]) { + return Ok(total); + } + } + offset += u32::from(len); + if offset == total { + return Ok(total); + } + } + } + + /// Ask for the clipboard to be typed into this terminal window's shell + /// line (`res/CLIPBOARD.md` §Shell): queued for its controller, which + /// takes what the line can hold. Gated as [`Self::copy`] is, and only the + /// seat's terminal window may ask. Answers the text's length, zero when + /// the clipboard holds no text. + pub async fn paste_to_shell(&mut self) -> Result { + match self.surface_request(SurfaceRequest::PasteToShell).await? { + SurfaceResponse::PasteQueued { total } => Ok(total), + SurfaceResponse::ClipboardEmpty => Ok(0), + _ => Err(ClientError::Protocol), + } + } + async fn read_configuration(&mut self) -> Result { let reply = self .control @@ -1392,6 +1490,22 @@ impl<'w, 't, T: Transport> WindowClient<'w, 't, T> { } } +/// A paste's piece reaches its reader through this much stack at a time: +/// the plane is read atomically, a byte at a time, and handed on as a slice. +/// Small enough for any task's stack, large enough that a 4 KiB piece is +/// sixteen calls. +const PASTE_STAGE_BYTES: usize = 256; + +/// The request plane as the memory it is: shared with the compositor, which +/// writes it while it answers a paste. +fn shared_plane(plane: &[u8]) -> &[core::sync::atomic::AtomicU8] { + // SAFETY: `AtomicU8` has `u8`'s size and alignment, and every access the + // returned view makes is atomic, so the compositor's writes race with none + // of this process's plain accesses: the plane's plain writers run only + // between requests, never while this view is live. + unsafe { &*(core::ptr::from_ref(plane) as *const [core::sync::atomic::AtomicU8]) } +} + fn valid_configuration(previous: Option, next: Configure) -> bool { next.window != 0 && next.serial != 0 diff --git a/native/crates/window/src/clipboard.rs b/native/crates/window/src/clipboard.rs new file mode 100644 index 0000000..eca5220 --- /dev/null +++ b/native/crates/window/src/clipboard.rs @@ -0,0 +1,107 @@ +//! The seat's clipboard (`res/CLIPBOARD.md`): one text the compositor holds +//! for each seat, which every window of that seat can put text on and read +//! text from, and which no window can touch without the user asking. +//! +//! The text crosses a window's own pair lease, never a lent view: a piece at +//! a time through the request plane the window already maps +//! ([`crate::menu`]'s page), copied out by the compositor before it answers a +//! copy and copied in before it answers a paste. A lent view per copy or +//! paste would spend the window's map cursor, which no unmap gives back, and +//! carry a capability and a revocation for a few hundred bytes of text. +//! +//! **Gesture-gated.** Only the seat's focused window may begin a copy or a +//! paste, and only on a key press, button press or menu choice the +//! compositor delivered to it moments before — each such gesture begins one +//! copy or one paste. A window in the background can neither overwrite the +//! clipboard nor read it. + +/// The most the clipboard holds, in bytes: a copy past it is refused, never +/// cut short. Equal to `kore_edit::CLIPBOARD_BYTES`, the bound every text +/// field already copies within, so any selection a field can copy fits. +pub const CLIPBOARD_BYTES: usize = 512 << 10; + +/// The most one piece of a copy or a paste carries: the request plane whole. +pub const CHUNK_BYTES: usize = crate::menu::MENU_PLANE_BYTES; + +const _: () = assert!(CLIPBOARD_BYTES <= u32::MAX as usize && CHUNK_BYTES <= u16::MAX as usize); + +/// What a paste into a shell's line takes from the text (`res/CLIPBOARD.md` +/// §Shell): the printable ASCII before the first line break, and never a line +/// break, so a paste can never run a command the user has not read. Why the +/// rest was held back is told to the user, never silently dropped. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum ShellStop { + /// The whole text was taken. + Whole, + /// The text goes on past a line break. + LineBreak, + /// A character the line cannot hold: a control byte, or a character + /// outside ASCII. + Unholdable, + /// The line is full. + Full, +} + +impl ShellStop { + /// What the user is told about the text held back: nothing when the + /// whole text was typed. + pub const fn describe(self) -> Option<&'static str> { + match self { + Self::Whole => None, + Self::LineBreak => Some("typed the first line; the other lines were held back"), + Self::Unholdable => { + Some("stopped at a tab, control or non-ASCII character the line cannot hold") + } + Self::Full => Some("the line is full; the rest was held back"), + } + } +} + +/// Scan `bytes`, the text from where the last scan stopped, for what a shell +/// paste may take: the length of the run it takes, and why it stops there. +/// `last` says `bytes` reaches the text's end; `room` bounds the run. `None` +/// means `bytes` ran out before the text did: scan the next piece with the +/// room that is left, even none, to learn what follows. +pub fn shell_run(bytes: &[u8], last: bool, room: usize) -> (usize, Option) { + let limit = bytes.len().min(room); + let taken = bytes[..limit] + .iter() + .position(|&byte| !(b' '..=b'~').contains(&byte)) + .unwrap_or(limit); + let stop = match bytes.get(taken) { + Some(b'\r' | b'\n') => Some(ShellStop::LineBreak), + Some(_) if taken < room => Some(ShellStop::Unholdable), + Some(_) => Some(ShellStop::Full), + None if last => Some(ShellStop::Whole), + None => None, + }; + (taken, stop) +} + +#[cfg(test)] +mod tests { + use super::*; + + /// A shell paste takes printable ASCII up to the first line break, the + /// first character the line cannot hold, or the line's room. + #[test] + fn a_shell_run_stops_before_what_the_line_cannot_take() { + assert_eq!(shell_run(b"ls -la", true, 255), (6, Some(ShellStop::Whole))); + assert_eq!(shell_run(b"ls -la", false, 255), (6, None)); + assert_eq!(shell_run(b"echo hi\nrm -rf /", true, 255), (7, Some(ShellStop::LineBreak))); + assert_eq!(shell_run(b"a\r\nb", true, 255), (1, Some(ShellStop::LineBreak))); + assert_eq!(shell_run(b"tab\there", true, 255), (3, Some(ShellStop::Unholdable))); + assert_eq!(shell_run("caf\u{e9}".as_bytes(), true, 255), (3, Some(ShellStop::Unholdable))); + assert_eq!(shell_run(b"abcdef", true, 4), (4, Some(ShellStop::Full))); + // A run that fills the room exactly at the text's end took it whole; + // short of the end, what follows decides. + assert_eq!(shell_run(b"abcd", true, 4), (4, Some(ShellStop::Whole))); + assert_eq!(shell_run(b"abcd", false, 4), (4, None)); + assert_eq!(shell_run(b"\nmore", true, 0), (0, Some(ShellStop::LineBreak))); + assert_eq!(shell_run(b"more", true, 0), (0, Some(ShellStop::Full))); + assert_eq!(shell_run(b"", true, 4), (0, Some(ShellStop::Whole))); + // A line break at the room's edge is still a line break: the line + // is not full, the text has another line. + assert_eq!(shell_run(b"abcd\n", true, 4), (4, Some(ShellStop::LineBreak))); + } +} diff --git a/native/crates/window/src/control.rs b/native/crates/window/src/control.rs index 8aec8e2..21ccca6 100644 --- a/native/crates/window/src/control.rs +++ b/native/crates/window/src/control.rs @@ -40,10 +40,17 @@ const SET_LAYOUT_HINTS: u8 = 0x4e; const PREPARE_CAPTURE: u8 = 0x4f; const REQUEST_CAPTURE: u8 = 0x50; const RELEASE_CAPTURE: u8 = 0x51; +const READ_CLIPBOARD: u8 = 0x54; const ACCEPTED: u8 = 0x40; const ADMITTED: u8 = 0x41; const REFUSED: u8 = 0x42; const SHELL_ACTION: u8 = 0x4d; +const CLIPBOARD_TEXT: u8 = 0x54; + +/// Bytes of clipboard text one [`DisplayRequest::ReadClipboard`] answer +/// carries inline: the reply record's four data words. A shell paste takes at +/// most one line of the shell's line editor, a few of these. +pub const CLIPBOARD_TEXT_BYTES: usize = 32; /// Bound on one window's published tab strip. pub const MAX_TABS: usize = 8; @@ -310,6 +317,12 @@ pub enum ShellAction { /// with `token` (never zero), by which it claims the pick. Data only; /// the compositor brokers the answer. Pick { token: u64, types: u64 }, + /// The user pasted into this controller's terminal window + /// (`res/CLIPBOARD.md`): read the clipboard with + /// [`DisplayRequest::ReadClipboard`] under `ticket` (never zero) and type + /// what the line can hold. The ticket reads the text the user pasted and + /// no other: a copy made since refuses it. + Paste { ticket: u64 }, } impl ShellAction { @@ -361,6 +374,7 @@ impl ShellAction { refusal: CaptureRefusal::from_code(b as u8)?, })), 6 if a != 0 && b != 0 => Some(Some(Self::Pick { token: a, types: b })), + 7 if a != 0 && b == 0 => Some(Some(Self::Paste { ticket: a })), _ => None, } } @@ -397,6 +411,7 @@ impl ShellAction { (5, mode.code() as u64, refusal.code() as u64) } Some(Self::Pick { token, types }) => (6, token, types), + Some(Self::Paste { ticket }) => (7, ticket, 0), } } } @@ -668,8 +683,65 @@ pub enum DisplayRequest { ReleaseCapture { ticket: u32, }, + /// Controller verb: read the clipboard text the user pasted into this + /// controller's terminal, from `offset`, under the ticket its + /// [`ShellAction::Paste`] carried. + ReadClipboard { ticket: u64, offset: u32 }, +} + +/// Clipboard text read for a controller: bytes `offset..offset + len` of a +/// `total`-byte text, the first `len` of `bytes` (the rest zero). +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct ClipboardText { + pub offset: u32, + pub len: u8, + pub total: u32, + pub bytes: [u8; CLIPBOARD_TEXT_BYTES], +} + +impl ClipboardText { + /// The text this answer carries. + pub fn text(&self) -> &[u8] { + &self.bytes[..usize::from(self.len)] + } + + /// The answer's word: `offset | total << 24 | len << 48`. + const fn word(&self) -> u64 { + self.offset as u64 | (self.total as u64) << 24 | (self.len as u64) << 48 + } + + fn from_words(word: u64, data: [u64; 4]) -> Option { + let (offset, total, len) = ((word & 0xff_ffff) as u32, (word >> 24 & 0xff_ffff) as u32, (word >> 48) as u16); + let mut bytes = [0; CLIPBOARD_TEXT_BYTES]; + for (word, chunk) in data.into_iter().zip(bytes.chunks_exact_mut(8)) { + chunk.copy_from_slice(&word.to_le_bytes()); + } + let len = u8::try_from(len).ok()?; + (len != 0 + && usize::from(len) <= CLIPBOARD_TEXT_BYTES + && total != 0 + && total as usize <= crate::clipboard::CLIPBOARD_BYTES + && u64::from(offset) + u64::from(len) <= u64::from(total) + && bytes[usize::from(len)..].iter().all(|&byte| byte == 0)) + .then_some(Self { + offset, + len, + total, + bytes, + }) + } + + fn data(&self) -> [u64; 4] { + core::array::from_fn(|word| { + u64::from_le_bytes(self.bytes[word * 8..word * 8 + 8].try_into().unwrap()) + }) + } } +const _: () = assert!( + crate::clipboard::CLIPBOARD_BYTES < 1 << 24 && CLIPBOARD_TEXT_BYTES == 4 * 8 +); + #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum DisplayResponse { Desktop(DesktopState), @@ -689,6 +761,7 @@ pub enum DisplayResponse { Configured(Configure), ControlRefused(ControlRefusal), ShellAction(Option), + ClipboardText(ClipboardText), } #[derive(Clone, Copy, Debug, PartialEq, Eq)] @@ -848,6 +921,11 @@ pub fn encode_req(request: &DisplayRequest) -> SurReqWire { w.op = RELEASE_CAPTURE; w.subject = u64::from(ticket).into(); } + DisplayRequest::ReadClipboard { ticket, offset } => { + w.op = READ_CLIPBOARD; + w.subject = ticket.into(); + w.a = u64::from(offset).into(); + } } w } @@ -981,6 +1059,16 @@ pub fn decode_req(w: &SurReqWire) -> Option { .ok() .filter(|ticket| *ticket != 0)?, }, + READ_CLIPBOARD => { + let offset = u32::try_from(w.a.get()).ok()?; + if w.subject.get() == 0 || offset as usize >= crate::clipboard::CLIPBOARD_BYTES { + return None; + } + DisplayRequest::ReadClipboard { + ticket: w.subject.get(), + offset, + } + } op if op & OUTPUT != 0 => { DisplayRequest::Output(kore_svc_output::wire::decode_req(&OutReqWire { op: op & !OUTPUT, @@ -1053,6 +1141,15 @@ pub fn encode_rep(response: &DisplayResponse, grants: u8) -> SurRepWire { w.a = a.into(); w.b = b.into(); } + DisplayResponse::ClipboardText(text) => { + w.tag = CLIPBOARD_TEXT; + w.id = text.word().into(); + let [a, b, c, d] = text.data(); + w.a = a.into(); + w.b = b.into(); + w.c = c.into(); + w.d = d.into(); + } } w } @@ -1119,6 +1216,9 @@ pub fn decode_rep(w: &SurRepWire) -> Option<(DisplayResponse, u8)> { SHELL_ACTION => { DisplayResponse::ShellAction(ShellAction::from_wire(w.id.get(), w.a.get(), w.b.get())?) } + CLIPBOARD_TEXT if w.id.get() >> 56 == 0 => DisplayResponse::ClipboardText( + ClipboardText::from_words(w.id.get(), [w.a.get(), w.b.get(), w.c.get(), w.d.get()])?, + ), tag if tag & OUTPUT != 0 => { DisplayResponse::Output(kore_svc_output::wire::decode_rep(&OutRepWire { tag: tag & !OUTPUT, @@ -1193,6 +1293,10 @@ pub fn reply_matches(w: &SurReqWire, response: &DisplayResponse, grants: u8) -> DisplayResponse::Accepted, ) => true, (DisplayRequest::TakeShellAction, DisplayResponse::ShellAction(_)) => true, + ( + DisplayRequest::ReadClipboard { offset, .. }, + DisplayResponse::ClipboardText(text), + ) => text.offset == offset, (DisplayRequest::TakeAdmission, DisplayResponse::Admitted { .. }) => true, ( DisplayRequest::CancelAdmission, @@ -1220,7 +1324,8 @@ pub fn reply_matches(w: &SurReqWire, response: &DisplayResponse, grants: u8) -> | DisplayRequest::TakeShellAction | DisplayRequest::PrepareCapture { .. } | DisplayRequest::RequestCapture { .. } - | DisplayRequest::ReleaseCapture { .. }, + | DisplayRequest::ReleaseCapture { .. } + | DisplayRequest::ReadClipboard { .. }, DisplayResponse::ControlRefused(_), ) => true, (DisplayRequest::CancelAdmission, DisplayResponse::ControlRefused(_)) => true, @@ -1819,6 +1924,67 @@ mod tests { assert_eq!(decode_rep(&wire), None); } + /// A shell paste names its ticket; a clipboard read answers its own + /// offset with a piece inside its text, zero past its length, and + /// nothing else; and an unticketed or out-of-bound read refuses. + #[test] + fn a_shell_paste_reads_its_ticketed_text_and_fails_closed() { + let paste = DisplayResponse::ShellAction(Some(ShellAction::Paste { ticket: 0xfeed })); + assert_eq!(decode_rep(&encode_rep(&paste, 0)), Some((paste, 0))); + let mut bad = encode_rep(&paste, 0); + bad.a = 0.into(); + assert_eq!(decode_rep(&bad), None, "no ticket"); + bad.a = 1.into(); + bad.b = 1.into(); + assert_eq!(decode_rep(&bad), None, "a stray word"); + + let read = DisplayRequest::ReadClipboard { + ticket: 0xfeed, + offset: 32, + }; + let wire = encode_req(&read); + assert_eq!(decode_req(&wire), Some(read)); + for (ticket, offset) in [(0, 0), (1, crate::clipboard::CLIPBOARD_BYTES as u32)] { + assert_eq!( + decode_req(&encode_req(&DisplayRequest::ReadClipboard { ticket, offset })), + None, + "ticket {ticket} offset {offset}" + ); + } + let mut bytes = [0; CLIPBOARD_TEXT_BYTES]; + bytes[..5].copy_from_slice(b"ls -l"); + let text = ClipboardText { + offset: 32, + len: 5, + total: 37, + bytes, + }; + let answer = DisplayResponse::ClipboardText(text); + let reply = encode_rep(&answer, 0); + assert_eq!(decode_rep(&reply), Some((answer, 0))); + assert_eq!(text.text(), b"ls -l"); + assert!(reply_matches(&wire, &answer, 0)); + assert!(!reply_matches(&wire, &answer, 1)); + assert!(reply_matches(&wire, &DisplayResponse::ControlRefused(ControlRefusal::Stale), 0)); + let elsewhere = encode_req(&DisplayRequest::ReadClipboard { + ticket: 0xfeed, + offset: 0, + }); + assert!(!reply_matches(&elsewhere, &answer, 0)); + let mut tail = reply; + tail.d = (tail.d.get() | 1 << 56).into(); + assert_eq!(decode_rep(&tail), None, "a live byte past the piece"); + for (offset, len, total) in [(0u32, 0u16, 5u32), (0, 33, 64), (33, 5, 37), (0, 1, 0)] { + let mut bad = reply; + bad.id = (u64::from(offset) | u64::from(total) << 24 | u64::from(len) << 48).into(); + bad.a = 0.into(); + bad.b = 0.into(); + bad.c = 0.into(); + bad.d = 0.into(); + assert_eq!(decode_rep(&bad), None, "{offset} {len} {total}"); + } + } + #[test] fn the_ninth_dock_row_round_trips_publication_and_launch() { let request = DisplayRequest::PublishDockItem { diff --git a/native/crates/window/src/keysplit.rs b/native/crates/window/src/keysplit.rs new file mode 100644 index 0000000..2b3fc08 --- /dev/null +++ b/native/crates/window/src/keysplit.rs @@ -0,0 +1,315 @@ +//! The keys a terminal window keeps from its shell (`res/KEYBOARD.md` §8). +//! +//! A terminal's keys are its shell's line, except the chords that act on the +//! window itself: its [`TerminalKeys`] — by default Command+C to copy the +//! selection, Command+V to paste and Command+A to select all, as on a Mac, so +//! Control stays the shell's (Control+C interrupts), and Shift with Page Up, +//! Page Down, Home or End to scroll the window's history. With Command held +//! every key is the window's, bound or not: Command never types. +//! +//! A key belongs to whoever it was pressed for, until it is released: Command +//! let go while C is still held does not hand the shell a fresh `c`, and a +//! letter held down before Command was pressed stays the shell's. So the +//! split is a state machine over complete snapshots, one per source, run by +//! whatever stands between the keyboard and the line: input-stream for the +//! founded terminal, the terminal app for a program it hosts. + +use kore_abi::keyboard::{modifier, usage}; + +use crate::shortcut::{Chord, Keymap, Mods}; + +/// Keys one snapshot lists. +pub const SNAPSHOT_KEYS: usize = 6; + +/// What a terminal window does with a key it keeps. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum TerminalAction { + /// Put the selection on the clipboard. + Copy, + /// Type the clipboard into the line. + Paste, + /// Select every row held. + SelectAll, + /// Scroll history a page back, a page forward, to its oldest row, or + /// back to the live grid. + PageUp, + PageDown, + Top, + Bottom, +} + +/// A terminal window's keys: the shortcuts it acts on, and whether Command +/// keeps every key from the line. Both sides of the split read one table — +/// input-stream, which takes the keys out of the founded shell's line, and +/// the window, which acts on them — so a keymap the user edits changes both +/// at once. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct TerminalKeys { + pub keymap: Keymap, + /// With Command held, every key is the window's, bound or not. + pub command_reserved: bool, +} + +impl TerminalKeys { + pub const DEFAULT: Self = Self { + keymap: Keymap::new() + .with(Chord::new(usage::C, Mods::COMMAND), TerminalAction::Copy) + .with(Chord::new(usage::V, Mods::COMMAND), TerminalAction::Paste) + .with(Chord::new(usage::A, Mods::COMMAND), TerminalAction::SelectAll) + .with(Chord::new(usage::PAGE_UP, Mods::SHIFT), TerminalAction::PageUp) + .with(Chord::new(usage::PAGE_DOWN, Mods::SHIFT), TerminalAction::PageDown) + .with(Chord::new(usage::HOME, Mods::SHIFT), TerminalAction::Top) + .with(Chord::new(usage::END, Mods::SHIFT), TerminalAction::Bottom), + command_reserved: true, + }; + + /// Whether `key`, pressed with the report bitmap `modifiers` held, is the + /// window's rather than the line's. + pub fn owns(&self, key: u8, modifiers: u8) -> bool { + if key == 0 || modifier::bit(key).is_some() { + return false; + } + self.keymap.action(key, modifiers).is_some() + || (self.command_reserved && Mods::of(modifiers).contains(Mods::COMMAND)) + } +} + +impl Default for TerminalKeys { + fn default() -> Self { + Self::DEFAULT + } +} + +/// Sources a split tells apart at once: input-stream's own bound on the +/// keyboards one terminal reads (its `PrevTable`). A further source shares +/// the last place, which forgets its holder's keys as it changes hands — +/// degraded the way every per-source table there is, never dropped. +const SOURCES: usize = 4; + +/// What one source holds, by whom it was pressed for. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +struct Held { + source: u8, + window: [u8; SNAPSHOT_KEYS], + shell: [u8; SNAPSHOT_KEYS], + /// The window's view as last handed out: the modifiers and its keys. + shown: (u8, [u8; SNAPSHOT_KEYS]), +} + +impl Held { + /// A source as the window first knows it: nothing held, no modifiers — + /// the state a fresh window's keyboard starts in, so plain typing never + /// reaches the window at all. + const FRESH: Self = Self { + source: 0, + window: [0; SNAPSHOT_KEYS], + shell: [0; SNAPSHOT_KEYS], + shown: (0, [0; SNAPSHOT_KEYS]), + }; +} + +/// One snapshot, split. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Split { + /// The keys the shell's line reads: the snapshot without the window's. + pub shell: [u8; SNAPSHOT_KEYS], + /// The keys the window reads, with the snapshot's modifiers, when what + /// the window sees changed: a key of its own pressed or released, or a + /// modifier, which it needs for a Shift-click. + pub window: Option<[u8; SNAPSHOT_KEYS]>, +} + +/// The keys of `keys` that `side` holds, in their order, packed to the front +/// as a keyboard reports them. +fn pack(keys: [u8; SNAPSHOT_KEYS], side: &[u8; SNAPSHOT_KEYS]) -> [u8; SNAPSHOT_KEYS] { + let mut packed = [0; SNAPSHOT_KEYS]; + for (slot, key) in packed + .iter_mut() + .zip(keys.into_iter().filter(|key| *key != 0 && side.contains(key))) + { + *slot = key; + } + packed +} + +/// The split's state, per source, under one window's keys. +pub struct KeySplit { + keys: TerminalKeys, + held: [Held; SOURCES], + used: usize, +} + +impl KeySplit { + /// A split under `keys`: [`TerminalKeys::DEFAULT`] until the user's + /// keymap names others. + pub const fn new(keys: TerminalKeys) -> Self { + Self { + keys, + held: [Held::FRESH; SOURCES], + used: 0, + } + } + + /// `source`'s place in `held`, of which `used` are taken. + fn place<'h>(held: &'h mut [Held; SOURCES], used: &mut usize, source: u8) -> &'h mut Held { + let index = match held[..*used].iter().position(|held| held.source == source) { + Some(index) => index, + None => { + let index = (*used).min(SOURCES - 1); + *used = (*used + 1).min(SOURCES); + held[index] = Held { source, ..Held::FRESH }; + index + } + }; + &mut held[index] + } + + /// Split `source`'s complete snapshot: `modifiers` held, `keys` down. + pub fn split(&mut self, source: u8, modifiers: u8, keys: [u8; SNAPSHOT_KEYS]) -> Split { + let keys_owned = &self.keys; + let held = Self::place(&mut self.held, &mut self.used, source); + // A key no longer down is released, whoever held it. + for owned in held.window.iter_mut().chain(held.shell.iter_mut()) { + if *owned != 0 && !keys.contains(owned) { + *owned = 0; + } + } + // A key newly down is the owner's its press made it. + for &key in keys.iter().filter(|&&key| key != 0) { + if held.window.contains(&key) || held.shell.contains(&key) { + continue; + } + let side = if keys_owned.owns(key, modifiers) { + &mut held.window + } else { + &mut held.shell + }; + // The two sides partition one snapshot's keys, so a free place is + // always there. + if let Some(free) = side.iter_mut().find(|slot| **slot == 0) { + *free = key; + } + } + let shell = pack(keys, &held.shell); + let window = pack(keys, &held.window); + let view = (modifiers, window); + let changed = held.shown != view; + held.shown = view; + Split { + shell, + window: changed.then_some(window), + } + } + + /// A route boundary: whatever was held is forgotten, as the window + /// forgets it on the focus change that made the boundary. + pub fn reset(&mut self) { + *self = Self::new(self.keys); + } +} + +#[cfg(test)] +mod tests { + use super::*; + use kore_abi::keyboard::modifier::{GUI, LEFT_GUI, LEFT_SHIFT}; + + const C: u8 = 0x06; + const V: u8 = 0x19; + const A: u8 = 0x04; + + fn keys_of(down: &[u8]) -> [u8; SNAPSHOT_KEYS] { + keys(down) + } + + fn keys(down: &[u8]) -> [u8; SNAPSHOT_KEYS] { + let mut keys = [0; SNAPSHOT_KEYS]; + keys[..down.len()].copy_from_slice(down); + keys + } + + /// Command chords and Shift with the scrolling keys are the window's; + /// everything else, Control chords included, is the shell's. + #[test] + fn the_window_keeps_command_chords_and_shifted_scrolling() { + let keys = TerminalKeys::DEFAULT; + assert!(keys.owns(C, LEFT_GUI)); + assert!(keys.owns(V, GUI)); + assert!(keys.owns(usage::PAGE_UP, LEFT_SHIFT)); + assert!(keys.owns(usage::END, LEFT_SHIFT | LEFT_GUI), "Command keeps every key"); + assert!(!keys.owns(usage::PAGE_UP, 0)); + assert!(!keys.owns(C, modifier::LEFT_CTRL)); + assert!(!keys.owns(A, LEFT_SHIFT)); + assert!(!keys.owns(usage::LEFT_GUI, LEFT_GUI), "a modifier's own usage is state"); + assert!(!keys.owns(0, LEFT_GUI)); + assert_eq!(keys.keymap.action(V, modifier::RIGHT_GUI), Some(TerminalAction::Paste)); + assert_eq!(keys.keymap.action(usage::HOME, LEFT_SHIFT), Some(TerminalAction::Top)); + } + + /// The split follows the keymap it was given: a user's binding takes its + /// key from the line, and without Command reserved, unbound Command keys + /// stay the line's. + #[test] + fn the_split_reads_the_keymap_it_was_given() { + let mut keys = TerminalKeys::DEFAULT; + keys.command_reserved = false; + keys.keymap + .bind(Chord::new(usage::INSERT, Mods::SHIFT).unwrap(), TerminalAction::Paste) + .unwrap(); + assert!(keys.owns(usage::INSERT, LEFT_SHIFT)); + assert!(!keys.owns(usage::Z, LEFT_GUI)); + let mut split = KeySplit::new(keys); + let paste = split.split(0, LEFT_SHIFT, keys_of(&[usage::INSERT])); + assert_eq!((paste.shell, paste.window), (keys_of(&[]), Some(keys_of(&[usage::INSERT])))); + } + + /// A key stays with whoever it was pressed for until it is released, and + /// the window hears its own keys and every modifier change, nothing else. + #[test] + fn a_key_belongs_to_whoever_it_was_pressed_for() { + let mut split = KeySplit::new(TerminalKeys::DEFAULT); + // `a` held, then Command: the `a` stays the shell's, and no Command+A. + let first = split.split(1, 0, keys(&[A])); + assert_eq!((first.shell, first.window), (keys(&[A]), None), "plain typing is the line's alone"); + let chord = split.split(1, LEFT_GUI, keys(&[A])); + assert_eq!((chord.shell, chord.window), (keys(&[A]), Some(keys(&[])))); + // Command+C, with `a` still down. + let copy = split.split(1, LEFT_GUI, keys(&[A, C])); + assert_eq!((copy.shell, copy.window), (keys(&[A]), Some(keys(&[C])))); + // Command let go while C is held: still the window's, no `c` typed. + let late = split.split(1, 0, keys(&[A, C])); + assert_eq!((late.shell, late.window), (keys(&[A]), Some(keys(&[C])))); + // Released: C pressed again now is the shell's. + let up = split.split(1, 0, keys(&[A])); + assert_eq!((up.shell, up.window), (keys(&[A]), Some(keys(&[])))); + let again = split.split(1, 0, keys(&[A, C])); + assert_eq!((again.shell, again.window), (keys(&[A, C]), None), "nothing changed for the window"); + // Another source is its own. + let other = split.split(2, LEFT_SHIFT, keys(&[usage::PAGE_UP])); + assert_eq!((other.shell, other.window), (keys(&[]), Some(keys(&[usage::PAGE_UP])))); + assert_eq!(split.split(1, 0, keys(&[A, C])).window, None); + } + + /// A source past the table's bound takes the last place, forgetting the + /// keys its holder had there rather than reading them as its own. + #[test] + fn a_source_past_the_bound_shares_the_last_place() { + let mut split = KeySplit::new(TerminalKeys::DEFAULT); + for source in 0..SOURCES as u8 { + let _ = split.split(source, LEFT_GUI, keys(&[C])); + } + let stranger = split.split(9, 0, keys(&[C])); + assert_eq!((stranger.shell, stranger.window), (keys(&[C]), None)); + let first = split.split(0, 0, keys(&[C])); + assert_eq!((first.shell, first.window), (keys(&[]), Some(keys(&[C]))), "the first sources keep theirs"); + } + + /// A boundary forgets every key, and the window hears the next state. + #[test] + fn a_reset_forgets_the_holders() { + let mut split = KeySplit::new(TerminalKeys::DEFAULT); + let _ = split.split(0, LEFT_GUI, keys(&[V])); + split.reset(); + let after = split.split(0, 0, keys(&[V])); + assert_eq!((after.shell, after.window), (keys(&[V]), None)); + } +} diff --git a/native/crates/window/src/lib.rs b/native/crates/window/src/lib.rs index b0e622d..98bc056 100644 --- a/native/crates/window/src/lib.rs +++ b/native/crates/window/src/lib.rs @@ -6,6 +6,7 @@ pub mod appearance; pub mod capture; #[cfg(feature = "runtime")] pub mod channel; +pub mod clipboard; #[cfg(feature = "runtime")] pub mod client; pub mod command; @@ -21,6 +22,8 @@ pub mod configure; pub mod control; #[cfg(feature = "runtime")] pub mod event; +pub mod keysplit; +pub mod shortcut; mod face; #[cfg(feature = "runtime")] pub mod relay; diff --git a/native/crates/window/src/relay.rs b/native/crates/window/src/relay.rs index 6ecf091..e6fe9a8 100644 --- a/native/crates/window/src/relay.rs +++ b/native/crates/window/src/relay.rs @@ -148,7 +148,25 @@ impl RelayProducer<'_> { /// Loss is explicit, bounded and nonblocking. While a Reset is pending, /// no snapshot is published, even if the consumer has freed ring slots. pub fn push(&mut self, focus: Focus, report: InputReport) -> bool { - if focus.target != FocusTarget::Window || focus.encode().is_none() || !keyboard(report) { + if focus.target != FocusTarget::Window { + return false; + } + self.publish(focus, report) + } + + /// On the terminal route, the keys its window keeps from the shell + /// ([`crate::keysplit`]): the same snapshot ring and loss rule, tagged + /// with the terminal route's epoch, so the compositor hands them to the + /// terminal window and to nothing a later route focuses. + pub fn push_window_keys(&mut self, focus: Focus, report: InputReport) -> bool { + if focus.target != FocusTarget::Terminal { + return false; + } + self.publish(focus, report) + } + + fn publish(&mut self, focus: Focus, report: InputReport) -> bool { + if focus.encode().is_none() || !keyboard(report) { return false; } if self.cell.reset_epoch.load(Ordering::Acquire) == 0 @@ -546,6 +564,7 @@ mod tests { malformed.data[0] = 1; assert!(!input.reports.push(focus, malformed)); assert!(!input.reports.push(Focus { epoch: 0, ..focus }, report)); + assert!(!input.reports.push_window_keys(focus, report), "a window route's keys are pushed whole"); assert_eq!(input.reports.dropped(), 0); assert!(!input.reports.needs_wake()); assert!(!compositor.reports.pending()); @@ -553,6 +572,25 @@ mod tests { assert_eq!(compositor.echo.pop(), None); } + /// The terminal route's window keys ride the ring under its own epoch. + #[test] + fn a_terminal_routes_window_keys_ride_under_its_epoch() { + let mut page = Page([0; 4096]); + let (mut input, mut compositor) = split(&mut page.0).unwrap(); + let chord = InputReport::keyboard(1, kore_abi::keyboard::modifier::LEFT_GUI, [6, 0, 0, 0, 0, 0]); + assert!(input.reports.push_window_keys(Focus::INITIAL, chord)); + assert_eq!( + compositor.reports.poll(), + Some(RelayEvent::Snapshot { + epoch: Focus::INITIAL.epoch, + report: chord + }) + ); + let mut malformed = chord; + malformed.data[0] = 1; + assert!(!input.reports.push_window_keys(Focus::INITIAL, malformed)); + } + /// The seat's lock state rides the snapshot to the window unchanged — a /// client resolves Caps Lock exactly as the terminal discipline does. #[test] diff --git a/native/crates/window/src/shortcut.rs b/native/crates/window/src/shortcut.rs new file mode 100644 index 0000000..a7535cc --- /dev/null +++ b/native/crates/window/src/shortcut.rs @@ -0,0 +1,283 @@ +//! Shortcuts as data (`res/KEYBOARD.md` §8): a [`Chord`] — one key and the +//! modifiers held with it — bound to an action in a [`Keymap`]. +//! +//! A window's shortcuts are a table it reads, never a match over usages: the +//! same table answers what a key does, what its menu says the shortcut is, +//! and — for a terminal — which keys leave the shell's line at all +//! ([`crate::keysplit`]). A keymap the user edits (Settings ▸ Keyboard & +//! Pointer) replaces a table; nothing that reads one changes. +//! +//! Modifiers are side-agnostic: either Shift satisfies a chord's Shift. A +//! chord matches its modifiers exactly, so Command+Shift+C is not Command+C. +//! Locks never take part: Caps Lock changes what a letter types, not which +//! shortcut its key is. + +use core::fmt; + +use kore_abi::keyboard::{modifier, usage}; + +/// The modifiers a chord needs, left and right alike. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash)] +pub struct Mods(u8); + +impl Mods { + pub const NONE: Self = Self(0); + pub const CTRL: Self = Self(1); + pub const SHIFT: Self = Self(1 << 1); + pub const ALT: Self = Self(1 << 2); + /// The GUI key: Command on a Mac keyboard, the Super (Windows) key on a + /// PC's. + pub const COMMAND: Self = Self(1 << 3); + + /// The modifiers a report's bitmap holds. + pub const fn of(modifiers: u8) -> Self { + const fn held(modifiers: u8, mask: u8, bit: Mods) -> u8 { + if modifiers & mask != 0 { bit.0 } else { 0 } + } + Self( + held(modifiers, modifier::CTRL, Self::CTRL) + | held(modifiers, modifier::SHIFT, Self::SHIFT) + | held(modifiers, modifier::ALT, Self::ALT) + | held(modifiers, modifier::GUI, Self::COMMAND), + ) + } + + pub const fn with(self, other: Self) -> Self { + Self(self.0 | other.0) + } + + pub const fn contains(self, other: Self) -> bool { + self.0 & other.0 == other.0 + } +} + +/// One key and the modifiers held with it. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub struct Chord { + key: u8, + mods: Mods, +} + +impl Chord { + /// `key` (a Keyboard/Keypad-page usage) with `mods` held. `None` for no + /// key or a modifier's own usage: a modifier is held with a chord, never + /// its key. + pub const fn new(key: u8, mods: Mods) -> Option { + if key == 0 || modifier::bit(key).is_some() { + return None; + } + Some(Self { key, mods }) + } + + pub const fn key(self) -> u8 { + self.key + } + + pub const fn mods(self) -> Mods { + self.mods + } + + /// Whether `key`, pressed with the report bitmap `modifiers` held, is + /// this chord. + pub const fn matches(self, key: u8, modifiers: u8) -> bool { + self.key == key && Mods::of(modifiers).0 == self.mods.0 + } + + /// The chord as a menu or the settings page spells it: `Cmd+C`, + /// `Shift+Page Up`. + pub fn write_label(self, out: &mut impl fmt::Write) -> fmt::Result { + for (bit, name) in [ + (Mods::CTRL, "Ctrl"), + (Mods::ALT, "Alt"), + (Mods::SHIFT, "Shift"), + (Mods::COMMAND, "Cmd"), + ] { + if self.mods.contains(bit) { + write!(out, "{name}+")?; + } + } + match self.key { + usage::A..=usage::Z => out.write_char(char::from(b'A' + (self.key - usage::A))), + usage::DIGIT_1..=usage::DIGIT_9 => out.write_char(char::from(b'1' + (self.key - usage::DIGIT_1))), + usage::DIGIT_0 => out.write_char('0'), + usage::F1..=usage::F12 => write!(out, "F{}", self.key - usage::F1 + 1), + key => out.write_str(match key { + usage::ENTER => "Enter", + usage::ESCAPE => "Esc", + usage::BACKSPACE => "Backspace", + usage::TAB => "Tab", + usage::SPACE => "Space", + usage::INSERT => "Insert", + usage::HOME => "Home", + usage::PAGE_UP => "Page Up", + usage::DELETE => "Delete", + usage::END => "End", + usage::PAGE_DOWN => "Page Down", + usage::RIGHT => "Right", + usage::LEFT => "Left", + usage::DOWN => "Down", + usage::UP => "Up", + _ => "Key", + }), + } + } +} + +/// The most bindings one keymap holds: a window's shortcuts, with room for a +/// user's own. +pub const MAX_BINDINGS: usize = 16; + +/// A keymap is full. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct KeymapFull; + +/// Chords bound to a window's actions: one action per chord, any number of +/// chords per action. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Keymap { + bindings: [Option<(Chord, A)>; MAX_BINDINGS], +} + +impl Default for Keymap { + fn default() -> Self { + Self::new() + } +} + +impl Keymap { + pub const fn new() -> Self { + Self { + bindings: [None; MAX_BINDINGS], + } + } + + /// This keymap with `chord` bound to `action`, for a table built as a + /// constant; a chord already bound or no room left is a build error. + pub const fn with(mut self, chord: Option, action: A) -> Self { + let Some(chord) = chord else { + panic!("a binding names a key"); + }; + let mut free = None; + let mut index = 0; + while index < MAX_BINDINGS { + match self.bindings[index] { + Some((bound, _)) if bound.key == chord.key && bound.mods.0 == chord.mods.0 => { + panic!("a chord is bound once"); + } + None if free.is_none() => free = Some(index), + _ => {} + } + index += 1; + } + let Some(free) = free else { + panic!("the keymap is full"); + }; + self.bindings[free] = Some((chord, action)); + self + } +} + +impl Keymap { + /// Bind `chord` to `action`, replacing what it was bound to. + pub fn bind(&mut self, chord: Chord, action: A) -> Result<(), KeymapFull> { + if let Some(bound) = self.bindings.iter_mut().flatten().find(|(bound, _)| *bound == chord) { + bound.1 = action; + return Ok(()); + } + let free = self.bindings.iter_mut().find(|slot| slot.is_none()).ok_or(KeymapFull)?; + *free = Some((chord, action)); + Ok(()) + } + + /// Leave `chord` unbound. + pub fn unbind(&mut self, chord: Chord) { + for slot in &mut self.bindings { + if slot.is_some_and(|(bound, _)| bound == chord) { + *slot = None; + } + } + } + + /// What `key` pressed with the report bitmap `modifiers` held does. + pub fn action(&self, key: u8, modifiers: u8) -> Option { + self.bindings + .iter() + .flatten() + .find(|(chord, _)| chord.matches(key, modifiers)) + .map(|&(_, action)| action) + } + + /// The chord a menu shows for `action`: the first bound to it. + pub fn chord(&self, action: A) -> Option { + self.bindings + .iter() + .flatten() + .find(|(_, bound)| *bound == action) + .map(|&(chord, _)| chord) + } +} + +#[cfg(test)] +mod tests { + extern crate std; + use super::*; + use std::string::String; + + #[derive(Clone, Copy, Debug, PartialEq, Eq)] + enum Act { + Copy, + Paste, + } + + const MAP: Keymap = Keymap::new() + .with(Chord::new(usage::C, Mods::COMMAND), Act::Copy) + .with(Chord::new(usage::V, Mods::COMMAND), Act::Paste) + .with(Chord::new(usage::INSERT, Mods::SHIFT), Act::Paste); + + /// A chord matches its modifiers exactly, either side's, and no lock. + #[test] + fn a_chord_matches_its_modifiers_exactly_either_side() { + assert_eq!(MAP.action(usage::C, modifier::LEFT_GUI), Some(Act::Copy)); + assert_eq!(MAP.action(usage::C, modifier::RIGHT_GUI), Some(Act::Copy)); + assert_eq!(MAP.action(usage::C, modifier::LEFT_GUI | modifier::LEFT_SHIFT), None); + assert_eq!(MAP.action(usage::C, modifier::LEFT_CTRL), None); + assert_eq!(MAP.action(usage::INSERT, modifier::RIGHT_SHIFT), Some(Act::Paste)); + assert_eq!(Chord::new(usage::LEFT_SHIFT, Mods::NONE), None, "a modifier is never a chord's key"); + assert_eq!(Chord::new(0, Mods::COMMAND), None); + } + + /// Bindings change as data: rebinding replaces, unbinding removes, and a + /// menu reads the chord an action is bound to. + #[test] + fn a_keymap_is_rebound_as_data() { + let mut map = MAP; + assert_eq!(map.chord(Act::Paste), Chord::new(usage::V, Mods::COMMAND)); + let ctrl_c = Chord::new(usage::C, Mods::CTRL.with(Mods::SHIFT)).unwrap(); + map.bind(ctrl_c, Act::Copy).unwrap(); + assert_eq!(map.action(usage::C, modifier::LEFT_CTRL | modifier::LEFT_SHIFT), Some(Act::Copy)); + map.bind(ctrl_c, Act::Paste).unwrap(); + assert_eq!(map.action(usage::C, modifier::LEFT_CTRL | modifier::LEFT_SHIFT), Some(Act::Paste)); + map.unbind(Chord::new(usage::V, Mods::COMMAND).unwrap()); + assert_eq!(map.action(usage::V, modifier::LEFT_GUI), None); + assert_eq!(map.chord(Act::Paste), Chord::new(usage::INSERT, Mods::SHIFT)); + let mut full = Keymap::::new(); + for key in 0..MAX_BINDINGS as u8 { + full.bind(Chord::new(usage::A + key, Mods::COMMAND).unwrap(), Act::Copy).unwrap(); + } + assert_eq!(full.bind(Chord::new(usage::F1, Mods::NONE).unwrap(), Act::Copy), Err(KeymapFull)); + } + + /// A chord spells itself as menus show it. + #[test] + fn a_chord_labels_itself() { + let label = |chord: Option| { + let mut text = String::new(); + chord.unwrap().write_label(&mut text).unwrap(); + text + }; + assert_eq!(label(Chord::new(usage::C, Mods::COMMAND)), "Cmd+C"); + assert_eq!(label(Chord::new(usage::PAGE_UP, Mods::SHIFT)), "Shift+Page Up"); + assert_eq!(label(Chord::new(usage::DIGIT_0, Mods::CTRL.with(Mods::ALT))), "Ctrl+Alt+0"); + assert_eq!(label(Chord::new(usage::F1 + 11, Mods::NONE)), "F12"); + } +} diff --git a/native/crates/window/src/wire.rs b/native/crates/window/src/wire.rs index 3581b10..0432959 100644 --- a/native/crates/window/src/wire.rs +++ b/native/crates/window/src/wire.rs @@ -16,6 +16,7 @@ use kore_abi::session::DISPLAY_FORMAT_XRGB8888; use kore_abi::svc::{CommitWire, MAX_COMMIT_RECTS, SurRectWire, SurRepWire, SurReqWire}; use crate::capture::{CaptureFrame, CaptureMode}; +use crate::clipboard::{CHUNK_BYTES, CLIPBOARD_BYTES}; use crate::cursor::Cursor; use crate::document::DocumentKind; use crate::menu::MENU_PLANE_BYTES; @@ -210,6 +211,21 @@ pub enum SurfaceRequest { /// stable identity, or zero if the user cancelled; its name is the /// first `len` bytes of this window's menu plane (none when cancelled). Picked { token: u64, object: u64, len: u8 }, + /// Put text on this window's seat's clipboard ([`crate::clipboard`]), a + /// piece at a time: bytes `offset..offset + len` of a `total`-byte text + /// are at the head of this window's request plane. Offset zero begins the + /// copy (focused window, unspent gesture); the piece that reaches `total` + /// ends it, and the text then replaces the clipboard's. + CopyChunk { offset: u32, len: u16, total: u32 }, + /// Read the seat's clipboard from `offset` into this window's request + /// plane. Offset zero begins the read (focused window, unspent gesture); + /// later offsets continue it until it reaches the end. + Paste { offset: u32 }, + /// The seat's terminal window asks for the clipboard to be typed into its + /// shell's line: queued as an inert + /// [`ShellAction::Paste`](crate::control::ShellAction::Paste) for its + /// controller (focused window, unspent gesture). + PasteToShell, } /// The surface-class reply vocabulary. @@ -263,6 +279,17 @@ pub enum SurfaceResponse { /// The answer is delivered and the pick has ended; the plane may be /// reused. The token, echoed. PickAnswered { token: u64 }, + /// The copy has taken the text through `through`; when that is the whole + /// text, the text is the seat's clipboard. + Copied { through: u32 }, + /// Bytes `offset..offset + len` of the clipboard's `total`-byte text are + /// at the head of the request plane. + Clipboard { offset: u32, len: u16, total: u32 }, + /// The seat's clipboard holds no text. + ClipboardEmpty, + /// The shell paste is queued for this window's controller; the text is + /// `total` bytes. + PasteQueued { total: u32 }, /// The refusal, echoing the subject the verb named (zero if none). Refused { refusal: SurfaceRefusal, @@ -288,6 +315,9 @@ pub enum SurfaceRefusal { Unknown, /// Authority delivery or revocation has not settled; retry is safe. Busy, + /// The user did not ask: a clipboard verb from a window that is not its + /// seat's focused window, or with no unspent gesture behind it. + Denied, } impl SurfaceRefusal { @@ -299,6 +329,7 @@ impl SurfaceRefusal { Self::OutOfMemory => SurRepWire::REFUSE_OUT_OF_MEMORY, Self::Unknown => SurRepWire::REFUSE_UNKNOWN, Self::Busy => SurRepWire::REFUSE_BUSY, + Self::Denied => SurRepWire::REFUSE_DENIED, } } @@ -310,6 +341,7 @@ impl SurfaceRefusal { SurRepWire::REFUSE_OUT_OF_MEMORY => Self::OutOfMemory, SurRepWire::REFUSE_UNKNOWN => Self::Unknown, SurRepWire::REFUSE_BUSY => Self::Busy, + SurRepWire::REFUSE_DENIED => Self::Denied, _ => return None, }) } @@ -457,10 +489,49 @@ pub fn encode_req(req: &SurfaceRequest) -> SurReqWire { w.b = U64::new(object); w.c = U64::new(u64::from(len)); } + SurfaceRequest::CopyChunk { offset, len, total } => { + w.op = SurReqWire::OP_COPY_CHUNK; + w.a = U64::new(piece_word(offset, len)); + w.b = U64::new(u64::from(total)); + } + SurfaceRequest::Paste { offset } => { + w.op = SurReqWire::OP_PASTE; + w.a = U64::new(u64::from(offset)); + } + SurfaceRequest::PasteToShell => { + w.op = SurReqWire::OP_PASTE_TO_SHELL; + } } w } +/// The piece word a copy and a paste share: `offset | len << 32`. +const fn piece_word(offset: u32, len: u16) -> u64 { + offset as u64 | (len as u64) << 32 +} + +/// A clipboard text's length: 1 to [`CLIPBOARD_BYTES`]. +fn text_length(word: u64) -> Option { + u32::try_from(word) + .ok() + .filter(|&len| len != 0 && len as usize <= CLIPBOARD_BYTES) +} + +/// A piece of a clipboard text, lifted: `offset..offset + len` of `total` +/// bytes, `len` 1 to [`CHUNK_BYTES`] and `total` 1 to [`CLIPBOARD_BYTES`]. +fn piece(word: u64, total: u64) -> Option<(u32, u16, u32)> { + let (offset, len) = (word as u32, (word >> 32) as u16); + let total = u32::try_from(total).ok()?; + let fits = u64::from(offset) + u64::from(len) <= u64::from(total); + (word >> 48 == 0 + && len != 0 + && usize::from(len) <= CHUNK_BYTES + && total != 0 + && total as usize <= CLIPBOARD_BYTES + && fits) + .then_some((offset, len, total)) +} + /// Structural lift; stray bits are the re-encode comparison's to catch. fn raw_decode_req(w: &SurReqWire) -> Option { Some(match w.op { @@ -532,6 +603,18 @@ fn raw_decode_req(w: &SurReqWire) -> Option { } SurfaceRequest::Picked { token, object, len } } + SurReqWire::OP_COPY_CHUNK => { + let (offset, len, total) = piece(w.a.get(), w.b.get())?; + SurfaceRequest::CopyChunk { offset, len, total } + } + SurReqWire::OP_PASTE => { + let offset = u32::try_from(w.a.get()).ok()?; + if offset as usize >= CLIPBOARD_BYTES { + return None; + } + SurfaceRequest::Paste { offset } + } + SurReqWire::OP_PASTE_TO_SHELL => SurfaceRequest::PasteToShell, _ => return None, }) } @@ -674,6 +757,22 @@ pub fn encode_rep(rep: &SurfaceResponse, grants: u8) -> SurRepWire { w.tag = SurRepWire::TAG_PICK_ANSWERED; w.a = U64::new(token); } + SurfaceResponse::Copied { through } => { + w.tag = SurRepWire::TAG_COPIED; + w.a = U64::new(u64::from(through)); + } + SurfaceResponse::Clipboard { offset, len, total } => { + w.tag = SurRepWire::TAG_CLIPBOARD; + w.a = U64::new(piece_word(offset, len)); + w.b = U64::new(u64::from(total)); + } + SurfaceResponse::ClipboardEmpty => { + w.tag = SurRepWire::TAG_CLIPBOARD_EMPTY; + } + SurfaceResponse::PasteQueued { total } => { + w.tag = SurRepWire::TAG_PASTE_QUEUED; + w.a = U64::new(u64::from(total)); + } SurfaceResponse::Refused { refusal, subject } => { w.tag = SurRepWire::TAG_REFUSED; w.refusal = refusal.code(); @@ -751,6 +850,17 @@ fn raw_decode_rep(w: &SurRepWire) -> Option { }, SurRepWire::TAG_PICK_CLAIMED => SurfaceResponse::PickClaimed { token: w.a.get() }, SurRepWire::TAG_PICK_ANSWERED => SurfaceResponse::PickAnswered { token: w.a.get() }, + SurRepWire::TAG_COPIED => SurfaceResponse::Copied { + through: text_length(w.a.get())?, + }, + SurRepWire::TAG_CLIPBOARD => { + let (offset, len, total) = piece(w.a.get(), w.b.get())?; + SurfaceResponse::Clipboard { offset, len, total } + } + SurRepWire::TAG_CLIPBOARD_EMPTY => SurfaceResponse::ClipboardEmpty, + SurRepWire::TAG_PASTE_QUEUED => SurfaceResponse::PasteQueued { + total: text_length(w.a.get())?, + }, SurRepWire::TAG_REFUSED => SurfaceResponse::Refused { refusal: SurfaceRefusal::from_code(w.refusal)?, subject: w.id.get(), @@ -867,6 +977,20 @@ pub fn reply_matches(request: &SurReqWire, response: &SurfaceResponse, grants: u | (SurfaceRequest::Picked { token, .. }, SurfaceResponse::PickAnswered { token: got }) => { token == got } + // The copy echoes how far it has taken the text: exactly to the end + // of the piece just sent. + (SurfaceRequest::CopyChunk { offset, len, .. }, SurfaceResponse::Copied { through }) => { + u64::from(through) == u64::from(offset) + u64::from(len) + } + // A read answers from the offset asked, and an empty clipboard + // answers only the read's first piece: a read under way that finds + // the clipboard changed is refused instead. + (SurfaceRequest::Paste { offset }, SurfaceResponse::Clipboard { offset: got, .. }) => { + offset == got + } + (SurfaceRequest::Paste { offset }, SurfaceResponse::ClipboardEmpty) => offset == 0, + (SurfaceRequest::PasteToShell, SurfaceResponse::PasteQueued { .. }) => true, + (SurfaceRequest::PasteToShell, SurfaceResponse::ClipboardEmpty) => true, _ => false, } } @@ -1167,7 +1291,7 @@ mod tests { } } - fn every_request() -> [SurfaceRequest; 14] { + fn every_request() -> [SurfaceRequest; 19] { [ SurfaceRequest::Query, SurfaceRequest::CreateSurface { @@ -1217,10 +1341,25 @@ mod tests { object: u64::MAX, len: PICK_NAME_BYTES as u8, }, + SurfaceRequest::CopyChunk { + offset: 0, + len: 1, + total: 1, + }, + SurfaceRequest::CopyChunk { + offset: (CLIPBOARD_BYTES - CHUNK_BYTES) as u32, + len: CHUNK_BYTES as u16, + total: CLIPBOARD_BYTES as u32, + }, + SurfaceRequest::Paste { offset: 0 }, + SurfaceRequest::Paste { + offset: CLIPBOARD_BYTES as u32 - 1, + }, + SurfaceRequest::PasteToShell, ] } - fn every_response() -> [SurfaceResponse; 13] { + fn every_response() -> [SurfaceResponse; 19] { [ SurfaceResponse::Info { width: 1920, @@ -1260,9 +1399,104 @@ mod tests { SurfaceResponse::PickQueued { serial: 3 }, SurfaceResponse::PickClaimed { token: 0x7071_ce4e }, SurfaceResponse::PickAnswered { token: 0x7071_ce4e }, + SurfaceResponse::Copied { through: 1 }, + SurfaceResponse::Copied { + through: CLIPBOARD_BYTES as u32, + }, + SurfaceResponse::Clipboard { + offset: 4096, + len: 17, + total: 4113, + }, + SurfaceResponse::ClipboardEmpty, + SurfaceResponse::PasteQueued { total: 9 }, + SurfaceResponse::Refused { + refusal: SurfaceRefusal::Denied, + subject: 0, + }, ] } + /// A clipboard piece names a span inside its text, the text inside the + /// clipboard's bound and the piece inside the plane; nothing crosses + /// with a grant; and each reply answers only the verb and offset asked. + #[test] + fn clipboard_pieces_stay_inside_their_text_and_answer_their_own_offset() { + let copy = SurfaceRequest::CopyChunk { + offset: 4096, + len: 100, + total: 4196, + }; + let wire = encode_req(©); + assert_eq!(SurfaceWire::request_grants(&wire), 0); + assert_eq!(decode_req(&wire), Some(copy)); + assert!(reply_matches(&wire, &SurfaceResponse::Copied { through: 4196 }, 0)); + assert!(!reply_matches(&wire, &SurfaceResponse::Copied { through: 4096 }, 0)); + assert!(!reply_matches(&wire, &SurfaceResponse::Copied { through: 4196 }, 1)); + for (offset, len, total) in [ + (0, 0, 1), + (0, 1, 0), + (1, 1, 1), + (0, CHUNK_BYTES as u16 + 1, CLIPBOARD_BYTES as u32), + (0, 1, CLIPBOARD_BYTES as u32 + 1), + (u32::MAX, 1, CLIPBOARD_BYTES as u32), + ] { + let wire = encode_req(&SurfaceRequest::CopyChunk { offset, len, total }); + assert_eq!(decode_req(&wire), None, "{offset} {len} {total}"); + } + let mut stray = encode_req(©); + stray.a = (stray.a.get() | 1 << 50).into(); + assert_eq!(decode_req(&stray), None); + + let read = encode_req(&SurfaceRequest::Paste { offset: 0 }); + let piece = SurfaceResponse::Clipboard { + offset: 0, + len: 12, + total: 12, + }; + assert!(reply_matches(&read, &piece, 0)); + assert!(reply_matches(&read, &SurfaceResponse::ClipboardEmpty, 0)); + let later = encode_req(&SurfaceRequest::Paste { offset: 12 }); + assert!(!reply_matches(&later, &piece, 0)); + assert!( + !reply_matches(&later, &SurfaceResponse::ClipboardEmpty, 0), + "a read under way never learns the clipboard emptied: it is refused" + ); + let past = encode_req(&SurfaceRequest::Paste { + offset: CLIPBOARD_BYTES as u32, + }); + assert_eq!(decode_req(&past), None); + for piece in [ + (0, 13, 12), + (0, 0, 12), + (0, CHUNK_BYTES as u16 + 1, CLIPBOARD_BYTES as u32), + ] { + let mut w = encode_rep(&SurfaceResponse::ClipboardEmpty, 0); + w.tag = SurRepWire::TAG_CLIPBOARD; + w.a = piece_word(piece.0, piece.1).into(); + w.b = u64::from(piece.2).into(); + assert_eq!(decode_rep(&w), None, "{piece:?}"); + } + + let shell = encode_req(&SurfaceRequest::PasteToShell); + assert!(reply_matches(&shell, &SurfaceResponse::PasteQueued { total: 5 }, 0)); + assert!(reply_matches(&shell, &SurfaceResponse::ClipboardEmpty, 0)); + assert!(!reply_matches(&shell, &SurfaceResponse::Copied { through: 5 }, 0)); + let denied = SurfaceResponse::Refused { + refusal: SurfaceRefusal::Denied, + subject: 0, + }; + for request in [copy, SurfaceRequest::Paste { offset: 0 }, SurfaceRequest::PasteToShell] { + assert!(reply_matches(&encode_req(&request), &denied, 0)); + } + for zero in [ + SurfaceResponse::Copied { through: 0 }, + SurfaceResponse::PasteQueued { total: 0 }, + ] { + assert_eq!(decode_rep(&encode_rep(&zero, 0)), None); + } + } + /// A pick is answered only by its own serial's echo, a claim and an /// answer only by their token's, none carries a grant, and a pick of no /// types, a zero serial or token, or an answer whose name contradicts diff --git a/native/provenance.json b/native/provenance.json index 6f475c1..6832942 100644 --- a/native/provenance.json +++ b/native/provenance.json @@ -1,9 +1,10 @@ { "repository": "https://github.com/MicroPerceptron/kore", - "revision": "7074bda28c416ffe34562857ee43cbc883482782", + "revision": "dfece6447cd331375a28b2bb290691cb8d5ab060", "packages": { "crates/abi": "kore-abi", "crates/app": "kore-app", + "crates/c-shim": "kore-c-shim", "crates/collections": "kore-collections", "crates/dagfs": "kore-dagfs", "crates/driver": "kore-driver", @@ -29,17 +30,18 @@ "services/swap": "kore-svc-swap", "tools/kore-vol": "kore-vol" }, - "workspace_sha256": "17edafdee9c32e592cda34f7db82a7075ab803793d04119a25b42b11a89784e3", + "workspace_sha256": "da01593a48d1a10e9995fa7af159b5d8675f3bba89cabb1104ac370934643fde", "files": { "LICENSE": "c9b2b12d3bad9080090798a442b4b2a77c48312ae75089f041281b7c8ad21738", "crates/abi/Cargo.toml": "f9a4811b4b6d06947a47a763354cad3bc739a5ea387194a4dcf22409351f3c7b", "crates/abi/src/cap.rs": "41ba3318d372ae447a47f1ef05932204057de17c7df3dd6b9d9a3dff4a935b3e", "crates/abi/src/clock.rs": "cce1c2839cba2377d3cc8696d60812f9195f071f422bd97bb4530ea0368972fc", - "crates/abi/src/command.rs": "206b8b150688cff68345c9237b78d1170e92f01c8323951e2a3e54b5fb808b97", + "crates/abi/src/command.rs": "e994b1527c989cc106281aa127df9a54dc03e1c85f3b15c49ba70ad5e5d65721", "crates/abi/src/crash.rs": "f3c9e5a954e06743f6993b0a5eeb6937d2199c50af5fe95b6be446393ccf999b", - "crates/abi/src/device.rs": "ad4578975e8056f375b0338124f497c9b4b5caa6272ea8a9930e2877d44efac6", + "crates/abi/src/device.rs": "06e8891917210870b64fa2ec3d2018067233ad333cf9d1bd13b3ce848fca97ed", "crates/abi/src/device/census.rs": "6fb54ecaade94b116f41e045b098dc5926e8e1c75eeea33870dd4be01bd1ce23", "crates/abi/src/device/i2c_admission.rs": "1ea2fc04cdb0b15186d2f1d80bde53187860e2197263e225f330965d96e40876", + "crates/abi/src/device/state.rs": "8caf4b751b4cae54f3d78626440bd847a39f1138a4dbc6055d6e8a1a1576ac32", "crates/abi/src/execution.rs": "106d679a3f993c3f4e7ad4eb625d40f4399f5b9b50947cb0336ba44a1d6b51af", "crates/abi/src/founding.rs": "801b8e9771763ff9223c3537aabb5653713d1883d1df62ff673f43982aa3e802", "crates/abi/src/guest/aarch64/mod.rs": "1fd24be3fa6cbc344e9873a0e3e75fabd1b919326e477ec4ea4cf148c01b66f8", @@ -57,17 +59,36 @@ "crates/abi/src/machine.rs": "7427c50ec38f71dbc83818a5075d03096cd34f5509e22646b04839eeb93cc783", "crates/abi/src/mem.rs": "ae502311778be8ef73f7b9edb1c47f157564f96a58db9add4ad74a313eb3a147", "crates/abi/src/monitor.rs": "8f01b9165d493f78127c3e2d37da401e3c969777b69535612ab2bcbd27eb7871", - "crates/abi/src/pkg.rs": "2893ad39761611ddee84d314bebbed45a9c9823ff145c6a8f28d3844cb40155d", + "crates/abi/src/pkg.rs": "da294e9e37fef2decda03dc5601a2a1131e499378bb14853845c2783b8e1a85a", "crates/abi/src/pointer.rs": "9971a292c1d00b275d8030e4f11fdd71206eaf12671cda8775d220b106ebcbf8", - "crates/abi/src/process.rs": "1dc29781e8d5f500e1e44e8ac36ca466aacf7f14a74b0f730af84336a4319cce", + "crates/abi/src/process.rs": "a5deca9b1d379949b98809a3fcb2bb596e2c06ab18cd27b826151374e6d8a634", "crates/abi/src/processor_status.rs": "c963e48f9b07046ab6c82d4cc2735ee2cc6c94561509a4e6e15927c837c0b0c6", - "crates/abi/src/session.rs": "030b2bee9b46426656db7ebde001738a2cfbdbe775a3be1ae892a14330f2d66f", + "crates/abi/src/session.rs": "ba8e90dc3d1167ac81b52c6b4d6369a7a112b9417342f6697793a7c1673e96b3", "crates/abi/src/stor.rs": "eabad4e5588fd343472660e15cccf32cfcc8d8f4c25eb18dffbd1598b6007db2", - "crates/abi/src/svc.rs": "d857cc812bdbc228c5f852a02ed02a349951cdc268291e51abe4628b894610da", - "crates/abi/src/task.rs": "57cb01a29bd2dbf870a1c49e35424f091fad597396ed5d080fc4f4cdd79c5f2a", + "crates/abi/src/svc.rs": "ea59db02200cdaa8f6b8f242b43ed9f4d5da4e95c0154749812ca4d22c3fbbd0", + "crates/abi/src/task.rs": "a65caa221d4cd7bcf67755fa1bab4391a1cf8cac80985cc750616ed86e50e276", "crates/abi/src/uring.rs": "e98d9db77fbe00d5547b582700a1104d2c5b2546bbe57742dfff8362c32dcc2e", "crates/app/Cargo.toml": "fb793b25eec583da7b5e404b19bd656a6b8bcee2a86a20b0da399be2210a9398", "crates/app/src/lib.rs": "5e7e8d791236f278ee85a02a3c37f41412d879cc41d67f6f6d3d701811973d7d", + "crates/c-shim/Cargo.toml": "85a4ad7f806ec588631dda39ef312513282217f3b0ddcdeeb3b5cbf6c1223685", + "crates/c-shim/src/alloc_symbols.rs": "e42b67a45271e7af676425cc915655e95153ee1f198864f5f527541e9c7092d8", + "crates/c-shim/src/errno.rs": "3fe0a17c84d6678f193e63e2db3fad89fef843cdd3bec348e064412b1d730352", + "crates/c-shim/src/format.rs": "d49d93e313f1eb0997deb40cd68942e8987152292c42feaf37546a883d21f4a9", + "crates/c-shim/src/format/float.rs": "eca54620399362cb35d09c842b7f3e40bef6af497c1a5b808df7d9e55c91b413", + "crates/c-shim/src/format/float/tests.rs": "8d579334d89afedcc9945c231a713e36b0baf5e99c5eb8a72d134100068dce9c", + "crates/c-shim/src/format_symbols.rs": "638807fa2f1e33095bfd295707d7ad7b63147d9cdd3cb66a132b49b167682a2f", + "crates/c-shim/src/lib.rs": "d52774104d1c0a3761f2e2825d74154b4e26050ee34bd0dcc93e43226ae41ee1", + "crates/c-shim/src/long_double.rs": "6d4434d610e973ff777f72b728a96537539c98bf076033c906d56e606796ab7b", + "crates/c-shim/src/number.rs": "e1fd500132cafdc70289f713f4ec04ae4e1bc9bc44884ebabbc8591fd6bc96ac", + "crates/c-shim/src/sort.rs": "4afb36cbb5c334d05712780bdee4d11ec30ab6be899f86e9828728f415d4f9db", + "crates/c-shim/src/stdio.rs": "d737a7b844aba0dbb852b6388ccc67c2342205992aaf609a9ac5353b97eea7d2", + "crates/c-shim/src/stdlib_symbols.rs": "5bd8645c999dcbe6e3722971d7ff57c654b74257aae48d9e76aba152f6906b72", + "crates/c-shim/src/string_symbols.rs": "cdcb976d8706218f0478de58993744e213d58a5012c0f25e999d96597c8928c0", + "crates/c-shim/src/va_long_double/aarch64.rs": "b954e8316eae7a30cce9823dcebecc7e8a258eb76a0b82bdc82cb03bd7b90c45", + "crates/c-shim/src/va_long_double/mod.rs": "96bb94ca1c36e0d1933e5abc5740c01a4b208b1d69b3102b0562109898ae35d0", + "crates/c-shim/src/va_long_double/riscv64.rs": "f170be5efa22626436e3b37749d828c20f90b75515a236f7bcf61c725b58673b", + "crates/c-shim/src/va_long_double/x86_64.rs": "49719570f536ac6aa665d7c2e6a5dc2d84ba4601f278f95bb9253fdfb0563f02", + "crates/c-shim/src/wide_symbols.rs": "494d37f1325b5a563c75f51c9ced46f6b338a4eb648a630bcdcd6d09a5d1d4a7", "crates/collections/Cargo.toml": "0b570a215ecd3d18b86b0f1b554e95de6636520c513eefd6f02d16cc2f37ebd6", "crates/collections/src/bitmat.rs": "35c1b7e572de0fa4e3dfc8c1c2cb8b7c5a031a812fb9ed7d4c86466b2a8399ec", "crates/collections/src/bitset.rs": "941e147ef204a6f3100c7d06652c6b9c3efb796ca7bdb738660bf6975a7ed815", @@ -88,23 +109,24 @@ "crates/dagfs/src/entry.rs": "ae077f13df5f0949fbc7c867b5f026aa494843ae605dde4fda845c2a531c33a3", "crates/dagfs/src/error.rs": "f8b93e779e3cc7966db4eaee3098fac4d3615d89441a6fc7ed34061eb6cbaf88", "crates/dagfs/src/face.rs": "b22d6f071397395a8ffda7487f5b876a03fc39590e4a1adefea622d5ae737bc9", - "crates/dagfs/src/fs.rs": "0e0f98c347c6e71363e7f022445e1dc3ccfe6921d2e9ad94f092fcc0296ce1b3", + "crates/dagfs/src/fs.rs": "de64b3e50009141d1be13e6cab3f510ab05cd4ea3d3814642ba2a3b39d388549", "crates/dagfs/src/graph.rs": "833a50a1e6a722ac0d5cefb34b101b9553a9ddd59a5c3d7af0fb723141a063c3", "crates/dagfs/src/ids.rs": "a6cd3127adbbfb3e02c7467f5736e2cabe6b53b9047d98f1cee2ba5de2c78227", "crates/dagfs/src/index.rs": "cd20892b18859bcdf19f63152f994003556bc048b0b0de3816bce250e99d9b81", "crates/dagfs/src/io.rs": "a91034153e6f82d32c9e5b5c8ce21f8b53f35c96adaeb2d507b56b07c75ea577", - "crates/dagfs/src/lib.rs": "aee6c0dd3bec2d44667691a110623a4526a50ff1af3133869b2c7b110e578132", + "crates/dagfs/src/lib.rs": "9d01b5ebc80ba67c7698670e3e9da5c2386d7ba2b4cbe16e3ed2ad4538470bd9", "crates/dagfs/src/name.rs": "a59b8464b96e3a4bb65bd1056e3db5ea80bc3a99da528624ee77757cff4efaf9", "crates/dagfs/src/object.rs": "7c6821771008a5f83b45d8fa53fd9aa00bba405f7a26abec2274962adc1cc00c", "crates/dagfs/src/path.rs": "db0bef7330be961a483e7582df14647d043f2a8fe722eb6320ece488d2fc243d", - "crates/dagfs/src/protocol.rs": "e9a35a3a2d28b1b09e44dc45200323fa258c3ef240d5494afd0b07ea19e17ece", + "crates/dagfs/src/protocol.rs": "de2613cacd587fe64bf613ff5d6de53ed8ccb403d5278bccadc7666e484512f4", "crates/dagfs/src/refusal.rs": "71d612406e9a86509056e6a755e3b3530396232007790588868e8756a5797fe8", "crates/dagfs/src/search.rs": "cfc79463a6367d2fc62a9598ca3fe20d6771c3d9aad07709192da93338a9e4ab", "crates/dagfs/src/stor.rs": "06a270d01e2f3a2dcdb41f803926d863d5fca6229aa14e0cc84337c651acdddb", - "crates/dagfs/src/text.rs": "878ee0abc41bfd30ece0f0a7c6be5935487fb945248773d3eebda8066f8f6032", + "crates/dagfs/src/text.rs": "56517a357cc933ff0f485a8a250d2177baa567d95521b107cc38b558101e7119", "crates/dagfs/src/view.rs": "24d661589ce1bb05e79112cda5ce1027166c7794adc3bcbcfdb3d63e663dfd69", "crates/dagfs/src/walk.rs": "3903476532c8f0933a0db3bf7cae9075c6db8dc06953b479709497329ed7e6a4", - "crates/dagfs/src/wire.rs": "3b5da5293e34e8a6c8b6133228e32b5477c6fef8550c58755ec174b7734feaee", + "crates/dagfs/src/watch.rs": "5070799a20aa65f1c074ec6f605973841547bc378731ea667ee48aac28ab995f", + "crates/dagfs/src/wire.rs": "1777e39074f4da19def29359ce4fe7d079377e4b6911489e37ab540959a2b802", "crates/driver/Cargo.toml": "15154ecb6517360f34580dba34799de5a38d546d8be8a64a626562c59e25e158", "crates/driver/src/accel.rs": "c2355916f1f6647d4bcf8414806c3bb9da45799e458133da70e31190316febfb", "crates/driver/src/block/image.rs": "5bb433b99cafc1a51812469ef8ea637dcc84ad2d2b8a79a3385fb8972884ffb8", @@ -168,7 +190,7 @@ "crates/ld/src/store.rs": "ba5a432a2a3e34e64cdd8f02dccc2c69bab5719b6a4223393c72beca0c7a2b18", "crates/ld/src/stub.rs": "b670715026a889f5baaf233a35d3d24c1891223a0e57665782ee326d90e49f5e", "crates/ld/tests/apply_engine.rs": "e01f5f805f34d614717ceb297275d65936c1b48989df90079fcd2f8ec83b6b1c", - "crates/ld/tests/built_images.rs": "7e6716758f34fab028ac67ce2f016b7e8d7ea93966dd2d300ed4ca254872c055", + "crates/ld/tests/built_images.rs": "400d8f9f4c3a5e62efdbbb790a9eeb3575a6ee3ecfb1aa1ba36c0f8e1fd441fc", "crates/ld/tests/check_gate.rs": "0ead133777f18f929d5e9b0353a289a5b4fa9e2af929155f525524096ee5578c", "crates/ld/tests/common/mod.rs": "1e4bdfa37511848d9e9458d9a550a1ff2410154411c6d026e9106035044ea438", "crates/ld/tests/late_layout.rs": "9bfc460d8036d69c1acdc52c430ee475e25062519cdc7dd30f235b882414a339", @@ -190,24 +212,24 @@ "crates/rt/src/barrier.rs": "c5bc3bef52f2bcc24edb1016fde955bd01e73074d4ce357d79cae9d3b63e6f3b", "crates/rt/src/block.rs": "70c2aef8bf149da2f32c7c995cd0f4c63cb492647896b107fdd5dbecd3d3042a", "crates/rt/src/cap/mapped.rs": "b29ce431638a29b01edfd72500348f5ae201afe97ec4684ea4449d01e75ec1e7", - "crates/rt/src/cap/mod.rs": "959a357dedc329cb84563f2a8677b462e89da379373022d644ca08ccdb157bc5", + "crates/rt/src/cap/mod.rs": "2e7bd2965128172e4f0f700dd8b64b161236ba4091df13e87d753bb71a71bf33", "crates/rt/src/cap/record.rs": "640b0ff60052d4775c259489af5fbff5cd03164370aceb4775186be956817bcd", - "crates/rt/src/cap/verbs.rs": "0de798ceebe67399a5994980cce2a5a49b8152639af2e3758cd84673701d7cee", + "crates/rt/src/cap/verbs.rs": "7d445d43fd71971a313d98b42e39bf4cb21ed528a1f2a15cbcd49ece02ee9007", "crates/rt/src/chan.rs": "fd50f962b23c2bfa15e72a1800790475d5cf6fcc55503ed39269852f025397e5", "crates/rt/src/clock/aarch64.rs": "47828a573f5fc30a988f012a648a8cb55596b45331cdcdfe1b390cd1d8543339", "crates/rt/src/clock/mod.rs": "d402d793448359338a361563a0fea91ea43d0264a5eb38c75b9abbb87df89af9", "crates/rt/src/clock/riscv64.rs": "17d4ecdb2b4f8152c0cf51f71ce28fe331d929a62405810a1b7eb332bb275c3d", "crates/rt/src/clock/x86_64.rs": "ca499d4dc51b7ed0eb196b831295b5fdecb09a1bd2df628a82e749e27011994a", "crates/rt/src/core_id.rs": "cadde39801a1d0ed05d70d0634a9e52d58043878737d790c4130183751b622cf", - "crates/rt/src/crash.rs": "c750a44492cd981bef6a55c6e0891ef1039e1663f3959e2a5f5289fb354ffe86", - "crates/rt/src/entry.rs": "c08657dd5fa6d88774173730db2501c73a4186aff2689ff866125c2e2189260f", + "crates/rt/src/crash.rs": "b6463a490f80124e22fdb5e13652bbaa94461eb1c4aefcffeca2e7dc7e977611", + "crates/rt/src/entry.rs": "499fd1ac8c1161ceb1241f093f167d7fcb8a5ef0405e5b43dc8e7293942058f3", "crates/rt/src/event.rs": "709a5500231735ad5bdb9ab9ab2fcee7b8f72f93e1b4741afba72fdc8b7c54f3", "crates/rt/src/executor/mod.rs": "071e3dc66bca448e6dabf96976702600bc77fdc98933d58847662fb2de073cb8", "crates/rt/src/executor/task.rs": "b3017cb97f287b8796c61d245c49e5c1b0ebfabe8c535cc2b3842cdbc854dc29", "crates/rt/src/executor/time.rs": "62f9ca9e5e17b395ec2b19a52fc6ed1bb53fd4acc3554cfca81eaf20e48fc272", "crates/rt/src/exit.rs": "3bf7e8ce6dac63b4ce365bdf2e743a9fd171851f1c5e40f8afbb4548877cef1b", "crates/rt/src/fiber_stack.rs": "48bf4fedba07ea2826615368e22882301ae4e0bbce0d893d0d7754d86bd02ce6", - "crates/rt/src/futex.rs": "9ec11227ff7260d141589fa63fdbb238eed3f52e9731ed4d9c7a14c9523a6709", + "crates/rt/src/futex.rs": "df5622a1bd811d7d97ec20dc44a568dba3a2c995254fc30405a60d41371e7390", "crates/rt/src/heap/mod.rs": "a9c39d3de56187a14d4633a229cbbb6e7ddb2c438d6f9f154b3b2551031b9f81", "crates/rt/src/heap/pool.rs": "67d42c6ffc7c7e176d21e97dd3179a3e4ff331f46dd457ed4d29914b1a404de8", "crates/rt/src/heap/shard.rs": "d97f91b3ecc71e3ec56fde0017f9d98cb99c5fbeeffe6ff79ddd153f5c6690fa", @@ -221,31 +243,31 @@ "crates/rt/src/mem_source.rs": "41cbfb3200c222c2d2d0094b19445a9cf7a016e80f20241f8f5cf22872c5ce56", "crates/rt/src/pace.rs": "81217909cee01fa148df9e953e6281d7abf328eed7c6257aac8fbcebc824710c", "crates/rt/src/pinned.rs": "9f86a46572178965c3a111363eb20431fc19916eb9e06c2da50f8f918f3eccf8", - "crates/rt/src/process.rs": "b9286ee4cda4417019e607f258aced1e71fa85d677690099b9602267d27f86a6", + "crates/rt/src/process.rs": "a1de9c1d05a18817adc252fb0c15c82077fbf08659c249670a301978ca039974", "crates/rt/src/process/interrupt.rs": "553bb4aa5678b4efa8d7537c96df0369d8fb810d0b3abb398b71dc7edb3db174", "crates/rt/src/process/words.rs": "cccc85ee65665f1a313c79be8aced6e408899a73986e0d8341ae3734ca79ce31", "crates/rt/src/random.rs": "b6d2738f97a9717d3d9481c49003e6c00df545faf52564189ad1cec508d16af6", "crates/rt/src/reserve.rs": "b2a9cb91873cc404555818d66740e54d4517d7676ad1c1efb620cfd3e43fcdc0", "crates/rt/src/room/mod.rs": "c6f56dda5064ebe75ee85ef122d82c81a507f3fe498b6f9a8463a977a0566c04", "crates/rt/src/room/x86_64.rs": "122e283074fa754a49c73beb733a92660a5d8c53701b5223297ded797c61fc70", - "crates/rt/src/service/mod.rs": "b59017ca341af1f2267dfc1b06d1f11a6e505255b8c327bccee1dfe5d04356fe", - "crates/rt/src/session/builder.rs": "3c08d7a5d4c9c085f11163b4fbf715f31fd1895e0ba67869d06ce73a65401255", + "crates/rt/src/service/mod.rs": "2f31433c345c77bb773036ec1ac259d0a81e74df3b8b24fba0769cc31d50cf8b", + "crates/rt/src/session/builder.rs": "4bea448f727d93bca9f956d7d4b5dda6982390ea616d06fb42b0804281e803b1", "crates/rt/src/session/flows.rs": "7205afbbe1d859868b313a24dccff8902e6ba3c1c8c45031e9531f6286259cae", "crates/rt/src/session/mod.rs": "466c2fbd8af228b3277075ad90a6d81031c1cb0a4c877f75669fff80e15338ea", "crates/rt/src/session/slots.rs": "82594125aa491429ed8c10b1316c19a82a162dc99f6dc97cdaa76398b92558e9", - "crates/rt/src/session/tests.rs": "f0f0ccce35ec87e133e3dcd2d29c7f32790e4be094d31aa113efe4010caf03f2", - "crates/rt/src/spmd.rs": "ebaaa608c8177ed80091215b2b00240ae0d689e620f5fd418c860e78034b7e4b", + "crates/rt/src/session/tests.rs": "80aa5e5d3b0ace53cbbb7a8e9f5bd8760d20f93e3655021172319e88097961ae", + "crates/rt/src/spmd.rs": "8c2264263b737af8538b10c278055bb1843e9c01ead56553022aa754e222c69e", "crates/rt/src/stack_arena.rs": "b252d45bfe845038ca9c8dc32941c634e448a2150dd49cf130e537048602f77b", - "crates/rt/src/star.rs": "a6cf5ff1cbf01f18bb3605e3597da6921ca611da087a3a30ecd61cd026ce6d30", - "crates/rt/src/startup.rs": "b1106a10e82659b8526ba7c94b7e40849edfdcbc30b077a77242712da8ebbd93", - "crates/rt/src/svc/client.rs": "72fb7d66de1981ae253512a1f2026922ce618c9c34eb46ccb47bde465a7fecea", - "crates/rt/src/svc/mod.rs": "a0f3497792d67233da0c96a31a33fe9d291f3ae870fc5570a3940e559795406d", + "crates/rt/src/star.rs": "91de20ffe5734e7a3c553408b6866087af179b7deddb2d576f05be0ca709ab66", + "crates/rt/src/startup.rs": "a005158737d337ea8c270a2860d09762048347e65d2b937734f8b59e3890d324", + "crates/rt/src/svc/client.rs": "062b52a0544eb11b9d4f8a938ebe75299bfd6c2ff7f42a1d8d21e8cb487c16d7", + "crates/rt/src/svc/mod.rs": "3c62c81946943268c24aee3a0a8e3778442a5475dc7bf3bdb42780aeaaa958e9", "crates/rt/src/svc/recyclable.rs": "9e5af8ad316bdd89e001fef7fbb6914b5a936dcdfbab2122740e03c1cf0ee7ec", - "crates/rt/src/svc/serve.rs": "a22172638a94e3ab52b88095dff9d78c88a1d693c81c6242af4932881d268706", + "crates/rt/src/svc/serve.rs": "01ead15cbff6004fe7d13990c373cb6d50c659526942d1b1cf883f05ec72c8d6", "crates/rt/src/task_watch.rs": "96b646f8785e427640519191cbf1f294c6908e5062a8082db271364048aa5b9a", "crates/rt/src/termination.rs": "920685ebb2071db58ae667e57bd02a6127166e8224ed13ec81216465ca315a55", - "crates/rt/src/thread.rs": "33c50deece11724565c0d5ca0f8843d41156e6b03123f1f565585c1be1ddc8b7", - "crates/rt/src/thread_controller.rs": "d64362e7e42f310b5fb7ad61c166938b05ee9edf2cedf7742ca6eba59a222137", + "crates/rt/src/thread.rs": "06d0eefa92c46fe31392db8b10e891e008e95c3ee4162f9f4034f7dec7d9b35c", + "crates/rt/src/thread_controller.rs": "6ce535ded212d30b4838d15108cf9e9625b4f9e9fd83e0fecde51be833b1326d", "crates/rt/src/tls.rs": "85bd95bafb3de73d249d7a2eb8779b5afc929077e27b4baa5f7c7428063063ab", "crates/rt/src/transport/mod.rs": "d5af890c800dd53abdbea8329383068cf43f0bf294ccb6b46dd7d8592ae5f321", "crates/rt/src/upcall.rs": "c0cbd2a59479dfba006850bb1bc5dd4f20f59317d178af776ba678f0e6404f02", @@ -268,21 +290,24 @@ "crates/window/src/appearance.rs": "b22e65286b94215bc4265a09ef280a5118f368ac45120311b870a56f7cc40a81", "crates/window/src/capture.rs": "d793f70fff35dfb9b13547f016b5abf8d851a23e53cfe28c1c24ad9949e6ef44", "crates/window/src/channel.rs": "6c96578badb21ddf6f3eb587e65b78af275bd8bcc0e03fd359c88eba6d91138f", - "crates/window/src/client.rs": "2e82a529684f4413cbb1d1475f5d6f108a8eb64f961aa0caec441045c4862d35", + "crates/window/src/client.rs": "485d4bc67d4178c38d1ea53ea7b1c979ea0cde6662df888163f6b2b61a0d82c7", + "crates/window/src/clipboard.rs": "c10c8d7d198b3562900e789077d16761db9cdb69e6425c02bb4a97991a44e55e", "crates/window/src/command.rs": "3ad4867ce62f3a430a68e4d105991229938e131e8cfde8afbbb0f522f99355d3", "crates/window/src/configure.rs": "5731662e919827aa6c9b189dce1d4ae06ebc7f7940c57fafd8662fc33bc46154", - "crates/window/src/control.rs": "0ccad3310bc1abc5efa44ff26d929ee7f7e54c6b83d1a3467c8dca5b3b3f8ca3", + "crates/window/src/control.rs": "3ab2400d32503f75b3419310049252167035179e76f8c838a89094d17946c9a5", "crates/window/src/cursor.rs": "2c320f4f19af4a5f9cd0d4059d1452661ac8cea23f0f6a44e0833cccd4d5f4bd", "crates/window/src/desktop.rs": "2aca94383249fac9f770c4edae35f634ad89388c463b17e4484125190757532b", "crates/window/src/document.rs": "1c4fb5a4bd3ce170a8a79ea97e7efd9aa013b1d2678b5ca1dde06ea1508ddaca", "crates/window/src/event.rs": "cf6565ea882e6c77672f31f0b354b03c57f25591a558893bb9466070b3f503a4", "crates/window/src/face.rs": "1078d8664e6da529c9d4fd43a7e3df77f287d0061f418ed21859d289e86765a0", - "crates/window/src/lib.rs": "310eeb5e3fdafe4e881008d311d0de4346bc1f7a78d417cba2c1e59e354670d6", + "crates/window/src/keysplit.rs": "044360a955b04a5229cba55956e9307a04d7d78facf2dfcf9903a43e3012e3fb", + "crates/window/src/lib.rs": "8b61e7f5e21e981069fc7409d2a7b5fb7add9c3a05aa5db29aa5685cbd5dc479", "crates/window/src/menu.rs": "8baf547a2dd58f83eef635e16e8658a011c917b60125b5d041c885b9eb69db1e", "crates/window/src/notify.rs": "8215605e26509fd615a983eab746ddcec18a156543349f52ddff6f301bdfe569", "crates/window/src/pick.rs": "d861daa5137f7513bc142737da8bd104f84c423e9c26af4f48ef19a3afe1b5aa", - "crates/window/src/relay.rs": "8f56568bcf643825906c74180ee13a14acb8299f6fb08f968919370e82669348", - "crates/window/src/wire.rs": "572daa34bc6233540779c542cd47533ec9c294123562de88ea6a5c2fb54572bf", + "crates/window/src/relay.rs": "bd96d827e9074dfdaddc4bb5a0ea5f924b4c63e5fd2bbffba033ab12f5c6b10b", + "crates/window/src/shortcut.rs": "ce72430d30a58833ddb4bc5c95ff12802b09719bd99383296a52a4d55af70e97", + "crates/window/src/wire.rs": "43789b5b72b003ae74b05b79ae4d1c25a3d015a3798dcddd31c664fe6cbb421f", "drivers/nvme/Cargo.toml": "21acf81448d0cc403eb10ada1c407baf7ff8cea4778ccdeb6c4f0d3508a66fbd", "drivers/nvme/src/block.rs": "4df5122b9c8f28ae878bffb6eeacfdba94569f73182c147a555f345a4d877723", "drivers/nvme/src/commands.rs": "a4cdbfb2b7822143beb7f351f4bba7b4643da1bffa80ffe443f065e47545c097", @@ -295,13 +320,13 @@ "drivers/nvme/src/regs.rs": "d9410b5c64bb70f99e042f7209119c0f0fdb57f859c6a1c07eba90199f1e98c1", "drivers/nvme/src/swap.rs": "f5e1099b83b2cdeb9d288f4d337f1a234d9dfd16c31e9d9303c10f6f14972141", "services/block/Cargo.toml": "2729b36369486a6f5c1c29200eaa4603b25c26352f06f52301b70d9d57260e43", - "services/block/boot.toml": "a6d8defc624013f6edd2f489cda144375b9e5d94bbaf25f11efd24ecb0ebc68f", + "services/block/boot.toml": "99615c152634c9eb62532eb87c917aa3da99fc26d31a1409c4d269043ceace00", "services/block/src/client.rs": "6aed79a404620f89ffa13078f615e846fdf6f3db5629c5cee5fbccbeaa7636cf", "services/block/src/lib.rs": "cf45960460cf6c9dba72f7a6727b51497275c686166a90be531d675097c275f1", "services/block/src/service.rs": "191e9d887d3f3267d6e2aa96e81996f01321bb2febb85e68581159f6e098b4b5", "services/block/src/wire.rs": "a9d64aef9ec3d58d37ca55b841933fc51b76ab94e18b245278b4519e96c65939", "services/dagfs/Cargo.toml": "d229a126616b1d7638f0f5a6edb4ac37d9d116f32afaa1897ffe484fbb286e5e", - "services/dagfs/boot.toml": "60d7176256fbdf96b7d8f6025c8346db9f847f806842bc58b5cfb2432a3051aa", + "services/dagfs/boot.toml": "58eda46ef6e6f45bba43df5b72bd7b1c5f45fb039957f9ddedb5d333f10b6b4d", "services/dagfs/examples/fsdemo.rs": "1666a14ef62fd4fc46f7f9efa2eaa5021ed79823f273a77f53758307f884e7d0", "services/dagfs/examples/fswire.rs": "994b97d699796b317741e2c87cbe812cc1776ddcd01173323e72a5f74c8ac267", "services/dagfs/examples/model_volume.rs": "0e3623e962da02b2126062a19d198b9e0a8f6e91896fd9a6f7d13f6f30ddc5f1", @@ -310,12 +335,12 @@ "services/dagfs/src/bounds.rs": "c825fcab0f30777e772c21fcf68cd092b10cf57a741eac22747de87ec3742e87", "services/dagfs/src/client.rs": "3532931af9484f548f7ff60bfd58f56fe392d8a770779f766f4e0edb0580d7d4", "services/dagfs/src/digests.rs": "88a256596ab314714ef523a2bea44dd8d5265436b284534d25d4538535454f3a", - "services/dagfs/src/durable.rs": "93df70108d12fd28134dbaf2434a8e067df2a415bbcd0cdb969a84b86403495d", - "services/dagfs/src/engine.rs": "3c52e78d1acf1bdd89d4bc9a77d95e97c139cebdb2009c2cd25921caa634365e", + "services/dagfs/src/durable.rs": "3caf0dc7bd6c7c13cde2c1a83f1db6d70169efb92f448b54ace80f07b6057456", + "services/dagfs/src/engine.rs": "e7cfda6e4f05a532cfc5bf15854b80254f64f40d25a6e7910bee38bfc0ba52c8", "services/dagfs/src/error.rs": "0a0cac522b38baa34221adda4ccca010fd8119c5aaf1f7b6015ab7a5f602ac33", "services/dagfs/src/extraction.rs": "502c7ba6cc8a37016a6c4a43f46664c3f27fe4c01d2cac21f8eb2cc2591a531d", - "services/dagfs/src/lib.rs": "9e4c0e957c81a9504e4217e0c16a83a0dad48fe35ead5a5785e7d139105d6cc8", - "services/dagfs/src/namespace.rs": "638a66619320f580de82b082f5520f35d47d8ab9b39b21ce413365dbf89dacaa", + "services/dagfs/src/lib.rs": "34140702b546448b1b3e326b0112f1be23e1beeb13af836b7f5377b529bb4e76", + "services/dagfs/src/namespace.rs": "72714f185853fbc2e0b67ab59bca00bf4665896adfc120e42fe5d5e3cdb40173", "services/dagfs/src/object.rs": "ce3468a2a536069eddd3598e8593c7eddc8e5107bd96f6e939af0133eb02fb91", "services/dagfs/src/parent_index.rs": "a46346a378ff2e9dd2ffe2d8a9514d8c777a7d0170ff3b90a9345def3b452a02", "services/dagfs/src/path.rs": "98ddd0026e43f6a4957b896bfa0c33ec217b016229662ebc0402283d1e8b0518", @@ -325,11 +350,12 @@ "services/dagfs/src/provider/mod.rs": "1e40047a77fc1f5140626bc25c9d57e681a38302d8b7a30c9e1c01c140cb3db8", "services/dagfs/src/provider/ram.rs": "fa3175f2d98854f2dc99f41cec1eb47ae7e4a560f9307de5f2c3778f95b7f7f1", "services/dagfs/src/reach.rs": "0b72f192ef37320d0a9a6883b7bdba2cd1a291f76dc343f24c52059d588e94ac", - "services/dagfs/src/router.rs": "07e9ceb72b8af7ab3ec944212df2009fe89d276b9179013d32baaa1e70f7f8d4", + "services/dagfs/src/router.rs": "a598a55029a034a95c14bcf242c1bcb86f3d80759dd797122d4f026af5bc0e26", "services/dagfs/src/search.rs": "98e3807b60e039082e2f16cd975d4637520642b380ea5ee3b7a25a4b843de448", - "services/dagfs/src/service/contract.rs": "32219b508a02f9136416618d2e2ab2db1f3bda46f18f8654683caf43e7eb4a66", - "services/dagfs/src/service/mod.rs": "c815acf69647b61a3b907b19b46e1456894d19237d7f19206892f336c9b1b3e0", + "services/dagfs/src/service/contract.rs": "f978333c7da9cb7797e08b9c71690e09c5b1a58b7eedc3a48256671c7f7577d2", + "services/dagfs/src/service/mod.rs": "3403eaf8628cf0e0bc0ce6ff8d2d9ba7a80ea64d550e2822f680757a6d2672e2", "services/dagfs/src/slot.rs": "e8f8751c09a4e6006bcaccd2d189cbb6762da1be6c4ea56bd331fbdc7f5cf827", + "services/dagfs/src/watches.rs": "9ae8277cd4ea1eadbbc8cc992b53edb6e476f247cf208e43ea6e13af3702b74c", "services/dagfs/src/window.rs": "8be4bd5c8f1e51cfa5744801e12a9406e7dafb351d0340a625ca8b8bddeb3364", "services/dagfs/src/wire.rs": "3c85fd1cdf0e35bb3a85cef10093f1b0d08626a9198fe219ff68fe09ba64595b", "services/dagfs/tests/authority.rs": "1432f45aef2bdffd63b83a2e93481673ba3412b14d4dc2abc1ef91f105172641", @@ -343,10 +369,10 @@ "services/dagfs/tests/provider.rs": "ba4070af34ae034c109e391e97bcc4755c6bd70b8e2fb229aaf9d747fef71e2d", "services/dagfs/tests/query_experiment.rs": "6fe147810474b6754ef31ad0fb95493e0b3c0886fbbafd78deb3729d410e8155", "services/device/Cargo.toml": "8f35c9ddd701b58a538f151e95e93ce618f0b346960491adad6d843287165d1c", - "services/device/boot.toml": "18b640a44bda45b29f3fb381477500a989c9f94b77dcb69eac0ec79dddbeed36", + "services/device/boot.toml": "4a0c84d22eb06e749e5d0803d51b659e3f8af81c52acb14491f58cec4246e752", "services/device/src/client.rs": "71660deb4a9f51edb7a72fbb3edac94170bf4cccf9ce182cbd478ee3daf406eb", "services/device/src/lib.rs": "215c7c7e81d5d51664ac77b72b2edb0bab739d5a54b30bc9ad08f9dd3a3cfea4", - "services/device/src/service/mod.rs": "817a718222ab737712ba8223bdd5392d1ca59b02d1edcf119a5812d2e8f96aa4", + "services/device/src/service/mod.rs": "6038ba0b9375e3c6bbb25c49bf0849a5878a402f4a3a691dc9df1494849d252b", "services/device/src/wire.rs": "aaec2baa85f4d5ceab233302bd1ba8e54c158f56dcbdef3bbd59e0d99f449d18", "services/extract-worker-c/Cargo.toml": "0a84769f520d5f2735711a21efebfc93380d91724ba2ed690cfe0fe1529e1513", "services/extract-worker-c/build.rs": "66b66d671df56f116176896f7f6c9aaba008ada2d51d0b3dc1f6ca4db08e74bd", @@ -385,7 +411,7 @@ "services/extract/vendor/jsmn/jsmn.h": "babfcb7413354a43dcd539c600a1e00bd2d1579ad0a6f6c7d8b81dccedb16cd0", "services/extract/vendor/jsmn/kore_jsmn.c": "a666060a7af03c819d1faaccc0d6faa19b76bcfe78e4ef61aeab273ad052c95c", "services/mm/Cargo.toml": "25809b6a03916d96b8de64a5988d005c71c49314fafbe0d920c3ec06f2dc5951", - "services/mm/boot.toml": "d4794c49f41698733a0569a0dd0354d17238321b0048ad3f3a348612784f0cfc", + "services/mm/boot.toml": "8753f22f227455e5fff90674c7295a933b1d035ee6f3c27b7474ca9b99e149c4", "services/mm/src/client.rs": "c6dcc0dc3ed59e0a4cf7532fd306af4c1dc4d4e5ad1edd1d86887def791c5aa6", "services/mm/src/heap.rs": "e55210f3609f2e3e90f30d5497355ddab5801acc60ccbe3f8acbbffff71dbfde", "services/mm/src/lib.rs": "d3820740666ed993e547ee1b3b579231bf439a813d9766860e3b1b57fe815bd6", @@ -397,7 +423,7 @@ "services/output/src/service/mod.rs": "d80c3beb50979b3e1e500abdfa3c0b9d38e354c24e2e34aea443732abbead6f1", "services/output/src/wire.rs": "3752e994eb18714518facae1b78a9ee2b34f99aad8687653f0b89dc8fdeca621", "services/swap/Cargo.toml": "9c0af01403a6ad5a7f4ff69a02b18c8b1b2624a684814d6dc1ac07968c5bbe53", - "services/swap/boot.toml": "046fbdab1a4d21d6a2e623b15a52d38b841fbb44b0e4f70148c2b16356af93d3", + "services/swap/boot.toml": "bba1d75a7f4418ffa066453fa0b753eebc5ae9d7e69ac5dca2b2a0ed694479eb", "services/swap/src/client.rs": "04f735d059b491765d5ba63a4f5f640332735bf336e62dcbae86c17647583e2c", "services/swap/src/extent.rs": "d6705f77951df1c42cda51fc4526ba6262cab6af4a0545405ce06d2d2e1db7b4", "services/swap/src/lib.rs": "883684df54a87ae66aa7895ea19a872cabc8ab7fd4aded4ea65b7e3dd51fe0cf", diff --git a/native/services/block/boot.toml b/native/services/block/boot.toml index dc9f49e..65fdc1b 100644 --- a/native/services/block/boot.toml +++ b/native/services/block/boot.toml @@ -12,5 +12,5 @@ chain = true endowment = "kore_svc_block::BlockEndowment" protocol = "kore_svc_block::BlockService" endows = ["DMA_POOL@1"] -digest_x86_64 = "blake3:c852d96c707ed0e95aad161be935fcc8bee5310598ec94f0f3bc75849cf586b0" -digest_aarch64 = "blake3:98023e83300526b637f67f62bafb1b523cd8657dabe4c980f5133f282d6f3ed0" +digest_x86_64 = "blake3:d9f8829568372dbcdccc3028bd4b302850c814eb4d44233290c5bdb165c67cef" +digest_aarch64 = "blake3:d004acf08fb53d2d42656f0df24bf54318e3eda87ec79a6778c2e186b61ade79" diff --git a/native/services/dagfs/boot.toml b/native/services/dagfs/boot.toml index fe9b8b5..37866c1 100644 --- a/native/services/dagfs/boot.toml +++ b/native/services/dagfs/boot.toml @@ -47,5 +47,5 @@ exec_client = true # `dagfs_heap_supply_holds_every_session` holds this row to it; the rest is # room for the proofs open handles carry, allocated per open at their depth. heap_supply_bytes = 0x00D0_0000 -digest_x86_64 = "blake3:2f00411e60e4c43e73c1290e225c8d1908f27e3ff36bf1d3be7709200bb5c454" -digest_aarch64 = "blake3:d8ce59acf3bdf5bb66912e3be2477c3a1de50c97614f66c83138f3fce1995680" +digest_x86_64 = "blake3:dc2fe387db38e271eac4f5d93ac4b5d815060a82cee8cdc81c16bd363d6e9eed" +digest_aarch64 = "blake3:d27bf4cc470afee6a1b5c7c80127dc7e0f8366569b56b74b09c75719e45a3e23" diff --git a/native/services/dagfs/src/durable.rs b/native/services/dagfs/src/durable.rs index adf8b89..864a3bd 100644 --- a/native/services/dagfs/src/durable.rs +++ b/native/services/dagfs/src/durable.rs @@ -79,7 +79,8 @@ use kore_svc_block::{BlockClient, BlockRefusal, BlockService, ProvisionRequest, use crate::persistence::{PersistentFs, VolumeProbe, bootstrap_data_start}; use crate::{ - FsEndowment, FsError, FsRequest, FsResponse, FsService, ProviderError, RamService, SessionId, + FsEndowment, FsError, FsRequest, FsResponse, FsResult, FsService, Handle, ObjectId, + ProviderError, RamService, SessionId, persistence::{CommitStage, PersistenceError, PersistenceState, commit_engine}, provider::BlockProvider, router::{self, RoutedRequest, Store}, @@ -1248,6 +1249,65 @@ impl MountedEngine { } } + fn attach_watch_plane(&mut self, peer: PeerId, aux: &'static mut [u8]) -> bool { + match self { + Self::Durable { fs, .. } => fs.attach_watch_plane(peer, aux), + Self::Volatile(fs) => fs.attach_watch_plane(peer, aux), + } + } + + fn detach_watch_plane(&mut self, peer: PeerId) { + match self { + Self::Durable { fs, .. } => fs.detach_watch_plane(peer), + Self::Volatile(fs) => fs.detach_watch_plane(peer), + } + } + + fn watch_target(&self, session: SessionId, handle: Handle) -> FsResult<(ObjectId, u64)> { + match self { + Self::Durable { fs, .. } => fs.watch_target(session, handle), + Self::Volatile(fs) => fs.watch_target(session, handle), + } + } + + fn watch_epoch(&self, session: SessionId, handle: Handle, dir: ObjectId) -> FsResult { + match self { + Self::Durable { fs, .. } => fs.watch_epoch(session, handle, dir), + Self::Volatile(fs) => fs.watch_epoch(session, handle, dir), + } + } + + #[allow(clippy::too_many_arguments)] + fn arm_watch( + &mut self, + via: u16, + session: SessionId, + slot: u8, + tag: u32, + store: Store, + handle: Handle, + target: FsResult<(ObjectId, u64)>, + ) -> FsResponse { + match self { + Self::Durable { fs, .. } => fs.arm_watch(via, session, slot, tag, store, handle, target), + Self::Volatile(fs) => fs.arm_watch(via, session, slot, tag, store, handle, target), + } + } + + fn settle_watches(&mut self, system: impl FnMut(SessionId, Handle, ObjectId) -> FsResult) { + match self { + Self::Durable { fs, .. } => fs.settle_watches(system), + Self::Volatile(fs) => fs.settle_watches(system), + } + } + + fn owed_watch_wakes(&mut self, owed: &mut [Option]) -> usize { + match self { + Self::Durable { fs, .. } => fs.owed_watch_wakes(owed), + Self::Volatile(fs) => fs.owed_watch_wakes(owed), + } + } + async fn serve_request_async( &mut self, request: FsRequest, @@ -1417,6 +1477,16 @@ impl MountedFs { self.system.is_some() } + /// Publish what every armed watch saw: the primary frontend holds the + /// planes, and each watch is read through the engine of its own store. + fn settle_watches(&mut self) { + let system = self.system.as_ref(); + self.user.settle_watches(|session, handle, dir| match system { + Some(system) => system.watch_epoch(session, handle, dir), + None => Err(FsError::AccessLost), + }); + } + fn durable(&self) -> bool { self.user.durable() || self.system.as_ref().is_some_and(MountedEngine::durable) } @@ -1711,6 +1781,24 @@ impl MountedFs { } FsResponse::Done } + // A watch on the user volume is the primary frontend's own verb. + RoutedRequest::Watch(Store::User, request) => { + self.user.serve_request_async(request, sv).await + } + // One on the system volume is resolved by the system engine and + // recorded in the pair's own plane: the client holds one plane. + RoutedRequest::Watch(Store::System, FsRequest::Watch { handle, slot, tag }) => { + let Some(system) = self.system.as_ref() else { + return FsResponse::Refused(FsError::InvalidHandle); + }; + let Some(session) = system.session_for_via(sv.via()) else { + return FsResponse::Refused(FsError::AccessDenied); + }; + let target = system.watch_target(session, handle); + self.user + .arm_watch(sv.via(), session, slot, tag, Store::System, handle, target) + } + RoutedRequest::Watch(Store::System, _) => FsResponse::Refused(FsError::Unsupported), RoutedRequest::Both(_) => FsResponse::Refused(FsError::Unsupported), // `DurableService::serve_async` answers the ring verbs before the // engines are consulted; reaching here means this row has none. @@ -3319,6 +3407,28 @@ impl SystemService for DurableService { self.advance_stor(); } + /// A recyclable peer's pair carries its watch plane after the rings; the + /// primary frontend keeps it, whichever volume a watch names. + fn attach_peer_aux(&mut self, peer: PeerId, aux: &'static mut [u8]) -> bool { + self.fs + .as_mut() + .is_some_and(|fs| fs.user.attach_watch_plane(peer, aux)) + } + + fn peer_detaching(&mut self, peer: PeerId) { + if let Some(fs) = self.fs.as_mut() { + fs.user.detach_watch_plane(peer); + } + } + + fn take_owed_wakes(&mut self, owed: &mut [Option]) -> usize { + let Some(fs) = self.fs.as_mut() else { + return 0; + }; + fs.settle_watches(); + fs.user.owed_watch_wakes(owed) + } + fn peer_closed(&mut self, via: u16, reason: PeerClose) -> bool { // A departed session's ring goes first: its targets are client // memory mapped into the controller, and a command still in flight @@ -3481,6 +3591,14 @@ impl SvcProtocol for DurableService { const REPLY_GRANTS: usize = ::REPLY_GRANTS; const SERVE_PAIRS: usize = ::SERVE_PAIRS; + fn init_pair_window( + window: &mut [u8], + req_cap: u32, + rep_cap: u32, + ) -> Result<(), kore_ring::spsc::RingError> { + ::init_pair_window(window, req_cap, rep_cap) + } + fn reply_matches(request: &Self::ReqWire, response: &FsResponse, grants: u8) -> bool { ::reply_matches(request, response, grants) } diff --git a/native/services/dagfs/src/engine.rs b/native/services/dagfs/src/engine.rs index dd2adb0..66a635d 100644 --- a/native/services/dagfs/src/engine.rs +++ b/native/services/dagfs/src/engine.rs @@ -762,6 +762,33 @@ impl Engine

{ }) } + /// What a watch on `handle` follows: the directory it names, and that + /// directory's change epoch now. Watching is observing what a listing + /// shows, so it takes exactly a listing's authority — `READ` on a live + /// route to a directory. + pub(crate) fn watch_target(&self, session: SessionId, handle: Handle) -> FsResult<(ObjectId, u64)> { + let route = self.route(session, handle)?; + route.rights.require(Rights::READ)?; + if !self.ns.kind(route.object)?.is_dir() { + return Err(FsError::NotDirectory); + } + Ok((route.object, self.ns.change_epoch(route.object)?)) + } + + /// The change epoch of the directory a watch armed on `handle` follows, + /// read only through a route that still proves to that same directory: + /// a retired edge, a closed handle or a reclaimed directory is + /// `AccessLost`, and the watch ends instead of counting for a client + /// that could no longer list what it counts. + pub(crate) fn watch_epoch(&self, session: SessionId, handle: Handle, dir: ObjectId) -> FsResult { + let route = self.route(session, handle)?; + route.rights.require(Rights::READ)?; + if route.object != dir { + return Err(FsError::AccessLost); + } + self.ns.change_epoch(dir) + } + /// One page of a directory's entries. pub fn list(&self, session: SessionId, handle: Handle, cursor: u32) -> FsResult { let route = self.route(session, handle)?; diff --git a/native/services/dagfs/src/lib.rs b/native/services/dagfs/src/lib.rs index 0373d1c..ecc50b5 100644 --- a/native/services/dagfs/src/lib.rs +++ b/native/services/dagfs/src/lib.rs @@ -158,6 +158,7 @@ mod router; pub mod search; pub mod service; pub mod slot; +mod watches; pub mod window; pub mod wire; @@ -185,6 +186,9 @@ pub use kore_dagfs::DagfsProtocol; /// The storage-ring client (`res/STORAGE_TRANSPORT.md` §4), for rows that /// read through the filesystem's transport — exec's image loads. pub use kore_dagfs::stor; +/// The watch plane every filesystem pair carries after its rings +/// (`kore_dagfs::watch`), for the founders that format pair windows. +pub use kore_dagfs::watch; pub use namespace::{EdgeId, ListEntry, Name}; pub use object::{ANY_REVISION, ObjectId, ObjectKind, ObjectSubtype, Revision, StableObjectId}; pub use provider::{ diff --git a/native/services/dagfs/src/namespace.rs b/native/services/dagfs/src/namespace.rs index b5ac08d..8adf1db 100644 --- a/native/services/dagfs/src/namespace.rs +++ b/native/services/dagfs/src/namespace.rs @@ -33,7 +33,7 @@ //! while exact edge generations govern route validity. use alloc::{boxed::Box, vec::Vec}; -use core::sync::atomic::{AtomicU32, AtomicU64, Ordering}; +use core::sync::atomic::{AtomicU32, AtomicU64, Ordering, fence}; use crate::{ ServiceAlloc, @@ -312,13 +312,29 @@ struct ObjectSnapshotView { /// a metadata commit. A reader never spins behind a preempted writer: it /// returns typed [`FsError::Busy`] and lets the bounded request retry policy /// decide when to revisit the snapshot. +/// +/// `epoch` and `owed` ride the same line but outside the seqlock: each is one +/// word with its own atomic meaning. +/// +/// - `epoch` is a directory's **change epoch** — advanced after every change +/// to what a listing of it shows: an edge made, removed or renamed in it, +/// a child's length, revision or incoming count moved, or an entry made or +/// removed in a child folder (that folder's line: its count, and whether +/// it holds folders). It is never reset, so it stays monotone across slot +/// reuse; the generation is what tells occupants apart. Watches compare it +/// ([`Namespace::change_epoch`]). +/// - `owed` marks a change to what the object's parents show of it — a +/// file's content, a folder's entries — that could not be passed on +/// because another commit held the object's parent row; the holder settles +/// it on release ([`HeldParents`]). #[repr(align(64))] struct ObjectSnapshot { sequence: AtomicU64, generation: AtomicU32, - _pad: AtomicU32, + owed: AtomicU32, len: AtomicU64, dir: AtomicU64, + epoch: AtomicU64, } impl ObjectSnapshot { @@ -326,9 +342,10 @@ impl ObjectSnapshot { Self { sequence: AtomicU64::new(0), generation: AtomicU32::new(0), - _pad: AtomicU32::new(0), + owed: AtomicU32::new(0), len: AtomicU64::new(0), dir: AtomicU64::new(0), + epoch: AtomicU64::new(0), } } @@ -380,6 +397,64 @@ impl ObjectSnapshot { dir: (dir != 0).then(|| DirId(SlotKey::from_bits(dir))), }) } + + /// `id`'s change epoch, validated against the slot's occupant: a slot + /// reused by another object answers `AccessLost`, never its successor's + /// count. + fn read_epoch(&self, id: ObjectId) -> FsResult { + let before = self.sequence.load(Ordering::Acquire); + if before & 1 != 0 { + return Err(FsError::Busy); + } + let generation = self.generation.load(Ordering::Relaxed); + // Acquire: a caller that sees the epoch move sees the change behind + // it (`Namespace::touch` advances it with release order). + let epoch = self.epoch.load(Ordering::Acquire); + let after = self.sequence.load(Ordering::Acquire); + if before != after { + return Err(FsError::Busy); + } + if generation != id.key().generation() || generation == 0 { + return Err(FsError::AccessLost); + } + Ok(epoch) + } +} + +/// A claim on one object's parent row that settles, on release, a change +/// owed to the object's parents ([`Namespace::note_shown`]). +/// +/// Every row claim in this module goes through it, so whichever commit holds +/// the row when a change lands is the one that passes it on: the changer +/// marks the change owed and tries the row; a holder, after letting go, +/// looks for the mark. The two sides store before they look, across a +/// sequentially consistent fence each, so at least one of them sees the +/// other — the change is never lost and never waited for. +pub(crate) struct HeldParents<'a, K> { + ns: &'a Namespace, + object: ObjectId, + claim: Option>, +} + +impl<'a, K> core::ops::Deref for HeldParents<'a, K> { + type Target = ParentClaim<'a>; + fn deref(&self) -> &Self::Target { + self.claim.as_ref().expect("the row is held until the guard drops") + } +} + +impl core::ops::DerefMut for HeldParents<'_, K> { + fn deref_mut(&mut self) -> &mut Self::Target { + self.claim.as_mut().expect("the row is held until the guard drops") + } +} + +impl Drop for HeldParents<'_, K> { + fn drop(&mut self) { + drop(self.claim.take()); + fence(Ordering::SeqCst); + self.ns.settle_owed(self.object); + } } /// One vacant snapshot per object slot, heap-backed for the reason recorded on @@ -396,6 +471,87 @@ fn vacant_snapshots() -> Box<[ObjectSnapshot]> { snapshots.into_boxed_slice() } +/// Change epochs and the parent-row guard. Nothing here depends on the +/// provider's key type: it touches only the snapshot lines and the rows. +impl Namespace { + fn snapshot_line(&self, id: ObjectId) -> Option<&ObjectSnapshot> { + usize::try_from(id.key().index()) + .ok() + .and_then(|index| self.snapshots.get(index)) + } + + /// Advance `dir`'s change epoch, after the change it reports is + /// published (release order; [`ObjectSnapshot::read_epoch`] acquires). + fn touch(&self, dir: ObjectId) { + if let Some(line) = self.snapshot_line(dir) { + line.epoch.fetch_add(1, Ordering::Release); + } + } + + /// Advance the epoch of every directory that names the row's object: what + /// each shows of it — its length, revision, or folder count — moved. + fn touch_parents(&self, row: &ParentClaim<'_>) { + for edge in row.iter() { + self.touch(edge.parent); + } + } + + /// Claim `object`'s parent row through the guard that settles a content + /// change owed while it is held. + fn claim_parents(&self, object: ObjectId) -> FsResult> { + let claim = self.parents_of.claim(object)?; + Ok(HeldParents { + ns: self, + object, + claim: Some(claim), + }) + } + + /// What every directory naming `object` shows of it just changed — a + /// file's length and revision, or a folder's entries (its line's count, + /// and whether it holds folders). Marked owed first, so a commit holding + /// the row passes the change on when it lets go ([`HeldParents`]). + fn note_shown(&self, object: ObjectId) { + let Some(line) = self.snapshot_line(object) else { + return; + }; + line.owed.store(1, Ordering::Relaxed); + fence(Ordering::SeqCst); + self.settle_owed(object); + } + + /// Pass an owed change on to `object`'s parents, if the row is free; a + /// row another commit holds is that commit's to settle. + /// + /// Each pass ends by letting the row go and looking again, because a + /// writer that found the row held during the pass left its mark for this + /// side. Every pass that continues consumed such a mark, so the loop does + /// the writers' work and never waits for anything. + fn settle_owed(&self, object: ObjectId) { + let Some(line) = self.snapshot_line(object) else { + return; + }; + while line.owed.load(Ordering::Relaxed) != 0 { + let Ok(row) = self.parents_of.claim(object) else { + return; + }; + if line.owed.swap(0, Ordering::AcqRel) != 0 { + self.touch_parents(&row); + } + drop(row); + fence(Ordering::SeqCst); + } + } + + /// `dir`'s change epoch: equal readings bracket no change to what a + /// listing of it shows. `AccessLost` once the directory is gone. + pub(crate) fn change_epoch(&self, dir: ObjectId) -> FsResult { + self.snapshot_line(dir) + .ok_or(FsError::AccessLost)? + .read_epoch(dir) + } +} + impl Namespace { pub(crate) fn new() -> Self { // Bootstrap-scale and fixed after startup, exactly like @@ -454,6 +610,7 @@ impl Namespace { debug_assert_eq!(object.id(), id.key()); object.commit_len(len); self.snapshots[id.key().index() as usize].publish(id, len, object.dir()); + self.note_shown(id); object.revision() } @@ -1066,7 +1223,7 @@ impl Namespace { // for the edge, before publishing it, so contention cannot leave a // visible name without its matching count or reverse entry. let mut child_object = self.object_mut(child)?; - let mut parents = self.parents_of.claim(child)?; + let mut parents = self.claim_parents(child)?; parents.reserve()?; let entry = entries .insert(DirEntry { name: *name, child }) @@ -1075,6 +1232,12 @@ impl Namespace { child_object.incoming_inc(); let edge = EdgeId { parent, entry }; parents.add(edge); + // `parent` gained an entry, and every other folder holding the child + // shows one more folder count: the row names them all. `parent`'s own + // folders see its line change (one more entry, perhaps a first + // folder). + self.touch_parents(&parents); + self.note_shown(parent); Ok(edge) } @@ -1093,12 +1256,15 @@ impl Namespace { .ok_or(FsError::NotFound)?; let edge = EdgeId { parent, entry }; let mut child_object = self.object_mut(child)?; - let mut parents = self.parents_of.claim(child)?; + let mut parents = self.claim_parents(child)?; entries .try_remove(edge.entry) .map_err(|error| fs_access(error, FsError::AccessLost))?; child_object.incoming_dec(); parents.remove(edge); + self.touch(parent); + self.touch_parents(&parents); + self.note_shown(parent); Ok((edge, child)) } @@ -1130,7 +1296,7 @@ impl Namespace { return Err(FsError::NotEmpty); } let mut child_object = self.object_mut(child)?; - let mut parents = self.parents_of.claim(child)?; + let mut parents = self.claim_parents(child)?; claims .get_mut(parent)? .try_remove(entry) @@ -1138,6 +1304,9 @@ impl Namespace { child_object.incoming_dec(); let edge = EdgeId { parent, entry }; parents.remove(edge); + self.touch(parent); + self.touch_parents(&parents); + self.note_shown(parent); Ok((edge, child)) } @@ -1166,6 +1335,9 @@ impl Namespace { .try_write(edge.entry) .map_err(|error| fs_access(error, FsError::AccessLost))?; entry.name = *new; + // Only `parent` spells the edge: the child's other folders show it + // under their own names, unchanged. + self.touch(parent); Ok(edge) } @@ -1243,7 +1415,7 @@ impl Namespace { // The child's parent row moves with the edge; it is claimed on its // own, so a file mid-transaction can still be moved. - let mut parents = self.parents_of.claim(child)?; + let mut parents = self.claim_parents(child)?; // Commit: retire the source edge, then create the destination edge. claims @@ -1261,6 +1433,16 @@ impl Namespace { // is deliberately not touched here; the parent row swaps in place. let dst_edge = EdgeId { parent: dst, entry }; parents.replace(src_edge, dst_edge); + // The source lost an entry; the row now names the destination and + // every other folder, whose "located in" for the child moved. + self.touch(src); + self.touch_parents(&parents); + // Across folders, each one's line moved (one entry fewer, one more); + // within one folder only a spelling changed. + if src != dst { + self.note_shown(src); + self.note_shown(dst); + } Ok((src_edge, dst_edge)) } @@ -1360,7 +1542,7 @@ impl Namespace { /// `object`'s incoming edges, copied out of its parent row so no claim /// is held while they are walked. fn incoming_edges(&self, object: ObjectId) -> FsResult> { - let row = self.parents_of.claim(object)?; + let row = self.claim_parents(object)?; let mut edges = BoundedVec::try_new_in(row.len(), crate::service_alloc()) .map_err(|_| FsError::OutOfObjects)?; for edge in row.iter() { @@ -1486,7 +1668,7 @@ impl Namespace { child_ids.as_mut_slice().sort_unstable_by_key(|id| id.key()); let mut children = heapless::Vec::< - (ObjectId, ObjectWrite<'_, K>, ParentClaim<'_>), + (ObjectId, ObjectWrite<'_, K>, HeldParents<'_, K>), MAX_DIR_ENTRIES, >::new(); for child in child_ids { @@ -1494,14 +1676,14 @@ impl Namespace { .objects .try_write(child.key()) .map_err(|error| fs_access(error, FsError::AccessLost))?; - let parents = self.parents_of.claim(child)?; + let parents = self.claim_parents(child)?; children .push((child, claimed, parents)) .map_err(|_| FsError::OutOfObjects)?; } // The reclaimed object names nothing once it is gone; its own row // is emptied with it (it is already empty: nothing names it). - let mut own_parents = self.parents_of.claim(id)?; + let mut own_parents = self.claim_parents(id)?; // All-or-nothing commit: decrement once per removed edge, and drop // its reverse entry, then record only the child generations that @@ -1521,12 +1703,14 @@ impl Namespace { } own_parents.clear(); let mut became_reclaimable = heapless::Vec::new(); - for (id, child, _) in children.iter() { + for (id, child, parents) in children.iter() { if child.is_reclaimable() { became_reclaimable .push(*id) .map_err(|_| FsError::OutOfObjects)?; } + // A child that keeps a folder elsewhere shows one fewer there. + self.touch_parents(parents); } if let Some(entries) = directory.take() { @@ -2743,6 +2927,211 @@ mod tests { assert_eq!(ns.leaked_placements(), 2); } + // ── change epochs ─────────────────────────────────────────────────────── + + /// Whether `dir`'s change epoch moved since `before`, and the new reading. + fn moved(ns: &Ns, dir: ObjectId, before: &mut u64) -> bool { + let now = ns.change_epoch(dir).unwrap(); + let moved = now != *before; + *before = now; + moved + } + + #[test] + fn every_change_a_listing_or_a_folder_line_shows_moves_its_epoch_and_nothing_else_does() { + let (mut ns, root) = ns_with_root(); + let a = ns.create_dir_object().unwrap(); + let b = ns.create_dir_object().unwrap(); + ns.link(root, &name("a"), a).unwrap(); + ns.link(root, &name("b"), b).unwrap(); + let file = ns.create_file_object().unwrap(); + let (mut at_a, mut at_b, mut at_root) = ( + ns.change_epoch(a).unwrap(), + ns.change_epoch(b).unwrap(), + ns.change_epoch(root).unwrap(), + ); + + // An entry made in `a`: `a` shows it, and `root` shows `a`'s line + // with one more entry. `b` shows nothing new. + ns.link(a, &name("f"), file).unwrap(); + assert!(moved(&ns, a, &mut at_a)); + assert!(moved(&ns, root, &mut at_root)); + assert!(!moved(&ns, b, &mut at_b)); + + // A second folder for the file: `b` gains an entry (and its line in + // `root` moves), and `a` shows one more folder count for it. + ns.link(b, &name("g"), file).unwrap(); + assert!(moved(&ns, b, &mut at_b)); + assert!(moved(&ns, a, &mut at_a)); + assert!(moved(&ns, root, &mut at_root)); + + // A rename is one folder's spelling only: its line's count holds. + ns.rename(b, &name("g"), &name("h")).unwrap(); + assert!(moved(&ns, b, &mut at_b)); + assert!(!moved(&ns, a, &mut at_a)); + assert!(!moved(&ns, root, &mut at_root)); + + // A content commit reaches every folder naming the file, and no + // folder line: the folders' entries are unchanged. + { + let mut object = ns.object_mut(file).unwrap(); + ns.commit_file_len(file, &mut object, 12); + } + assert!(moved(&ns, a, &mut at_a)); + assert!(moved(&ns, b, &mut at_b)); + assert!(!moved(&ns, root, &mut at_root)); + + // A move: the source loses an entry, the destination gains one, the + // file's other folder sees its location move, and both folders' + // lines move in `root`. + let c = ns.create_dir_object().unwrap(); + ns.link(root, &name("c"), c).unwrap(); + assert!(moved(&ns, root, &mut at_root)); + let mut at_c = ns.change_epoch(c).unwrap(); + ns.relocate(b, &name("h"), c, &name("h")).unwrap(); + assert!(moved(&ns, b, &mut at_b)); + assert!(moved(&ns, c, &mut at_c)); + assert!(moved(&ns, a, &mut at_a)); + assert!(moved(&ns, root, &mut at_root)); + + // A move within one folder is a spelling: its line holds. + ns.relocate(c, &name("h"), c, &name("i")).unwrap(); + assert!(moved(&ns, c, &mut at_c)); + assert!(!moved(&ns, root, &mut at_root)); + + // Removing a name: the folder loses it, the other shows one fewer, + // and the folder's line moves. + ns.detach(c, &name("i")).unwrap(); + assert!(moved(&ns, c, &mut at_c)); + assert!(moved(&ns, a, &mut at_a)); + assert!(moved(&ns, root, &mut at_root)); + ns.unlink_empty(a, &name("f")).unwrap(); + assert!(moved(&ns, a, &mut at_a)); + assert!(moved(&ns, root, &mut at_root)); + + // A refused change moves nothing. + assert_eq!(ns.rename(a, &name("absent"), &name("x")).unwrap_err(), FsError::NotFound); + assert_eq!(ns.detach(a, &name("absent")).unwrap_err(), FsError::NotFound); + assert!(!moved(&ns, a, &mut at_a)); + assert!(!moved(&ns, root, &mut at_root)); + } + + #[test] + fn a_folder_line_change_meeting_its_held_parent_row_is_passed_on_by_the_holder() { + let (mut ns, root) = ns_with_root(); + let a = ns.create_dir_object().unwrap(); + ns.link(root, &name("a"), a).unwrap(); + let file = ns.create_file_object().unwrap(); + let mut at_root = ns.change_epoch(root).unwrap(); + + // A commit holds `a`'s own row (a move of `a`, a parents walk) while + // an entry lands in `a`: the linker cannot reach `a`'s folders. + let held = ns.claim_parents(a).unwrap(); + ns.link(a, &name("f"), file).unwrap(); + assert!(!moved(&ns, root, &mut at_root), "the linker left the change owed"); + drop(held); + assert!(moved(&ns, root, &mut at_root), "the holder passed it on"); + drop(ns.claim_parents(a).unwrap()); + assert!(!moved(&ns, root, &mut at_root), "owed once, settled once"); + } + + #[test] + fn a_reclaimed_directory_answers_access_lost_and_its_children_s_other_folders_move() { + let (mut ns, root) = ns_with_root(); + let doomed = ns.create_dir_object().unwrap(); + let kept = ns.create_dir_object().unwrap(); + let file = ns.create_file_object().unwrap(); + ns.link(root, &name("doomed"), doomed).unwrap(); + ns.link(root, &name("kept"), kept).unwrap(); + ns.link(doomed, &name("f"), file).unwrap(); + ns.link(kept, &name("f"), file).unwrap(); + let mut at_kept = ns.change_epoch(kept).unwrap(); + + let (_, child) = ns.detach(root, &name("doomed")).unwrap(); + let mut placements: BoundedVec = + BoundedVec::try_new_in(MAX_OBJECTS, crate::service_alloc()).unwrap(); + let mut objects: BoundedVec = + BoundedVec::try_new_in(MAX_OBJECTS, crate::service_alloc()).unwrap(); + assert_eq!(ns.reclaim(child, &mut placements, &mut objects), 1); + assert_eq!(ns.change_epoch(doomed), Err(FsError::AccessLost)); + assert!(moved(&ns, kept, &mut at_kept), "the file is in one folder fewer"); + + // A slot reused by another directory never answers for the old one. + let successor = ns.create_dir_object().unwrap(); + assert_eq!(ns.change_epoch(doomed), Err(FsError::AccessLost)); + assert!(ns.change_epoch(successor).is_ok()); + } + + #[test] + fn a_content_change_meeting_a_held_parent_row_is_passed_on_by_the_holder() { + let (mut ns, root) = ns_with_root(); + let a = ns.create_dir_object().unwrap(); + ns.link(root, &name("a"), a).unwrap(); + let file = ns.create_file_object().unwrap(); + ns.link(a, &name("f"), file).unwrap(); + let mut at_a = ns.change_epoch(a).unwrap(); + + // A commit holds the file's row (a move, a parents walk) while the + // file's bytes commit: the writer cannot touch the parents itself. + let held = ns.claim_parents(file).unwrap(); + { + let mut object = ns.object_mut(file).unwrap(); + ns.commit_file_len(file, &mut object, 3); + } + assert!(!moved(&ns, a, &mut at_a), "the writer left the change owed"); + drop(held); + assert!(moved(&ns, a, &mut at_a), "the holder passed it on"); + // Owed once, settled once. + drop(ns.claim_parents(file).unwrap()); + assert!(!moved(&ns, a, &mut at_a)); + } + + #[test] + fn concurrent_writers_and_row_holders_never_lose_a_content_change() { + use std::sync::Arc; + use std::sync::atomic::AtomicBool; + + let (mut ns, root) = ns_with_root(); + let a = ns.create_dir_object().unwrap(); + ns.link(root, &name("a"), a).unwrap(); + let file = ns.create_file_object().unwrap(); + ns.link(a, &name("f"), file).unwrap(); + let ns = Arc::new(ns); + let stop = Arc::new(AtomicBool::new(false)); + // A holder thread keeps taking and letting go of the row, as parents + // walks and moves do. + let holder = { + let (ns, stop) = (Arc::clone(&ns), Arc::clone(&stop)); + std::thread::spawn(move || { + while !stop.load(Ordering::Relaxed) { + if let Ok(row) = ns.claim_parents(file) { + core::hint::spin_loop(); + drop(row); + } + } + }) + }; + for len in 0..2_000u64 { + let before = ns.change_epoch(a).unwrap(); + { + let mut object = ns.object_mut(file).unwrap(); + ns.commit_file_len(file, &mut object, len); + } + // Either side settles it; once both have let go, the change is in. + let mut settled = false; + for _ in 0..1_000_000 { + if ns.change_epoch(a).unwrap() != before { + settled = true; + break; + } + std::thread::yield_now(); + } + assert!(settled, "write {len} was never passed on"); + } + stop.store(true, Ordering::Relaxed); + holder.join().unwrap(); + } + /// Write `value` as decimal ASCII into `buf`, returning the byte count. /// Test-local so name generation needs no formatting machinery. fn write_index(buf: &mut [u8; 8], value: usize) -> usize { diff --git a/native/services/dagfs/src/router.rs b/native/services/dagfs/src/router.rs index 8540f77..89c7e10 100644 --- a/native/services/dagfs/src/router.rs +++ b/native/services/dagfs/src/router.rs @@ -58,6 +58,10 @@ pub(crate) enum RoutedRequest { /// A storage-ring verb: neither engine's; served by the row that owns /// the controller (`res/STORAGE_TRANSPORT.md` §4). Stor(FsRequest), + /// Arm a watch on a directory of one store. The request's handle is + /// native to that store; the plane it publishes into is the pair's one, + /// which the primary frontend holds. + Watch(Store, FsRequest), } #[inline] @@ -435,6 +439,13 @@ pub(crate) fn route(request: FsRequest, has_system: bool) -> Result { + let (store, handle) = split_handle(handle); + RoutedRequest::Watch(store, Watch { handle, slot, tag }) + } + // Disarming names no handle: the pair's one plane, in the primary + // frontend, holds every slot, whichever store the slot watched. + Unwatch { .. } => RoutedRequest::One(Store::User, request), LendVersion { handle, window_offset, @@ -541,7 +552,11 @@ pub(crate) fn route(request: FsRequest, has_system: bool) -> Result Option<&'static (dyn core::alloc::Allocator + Sync)> { Some(crate::service_alloc()) } + + /// A recyclable peer's pair carries its watch plane after the rings. + fn attach_peer_aux(&mut self, peer: PeerId, aux: &'static mut [u8]) -> bool { + self.attach_watch_plane(peer, aux) + } + + fn peer_detaching(&mut self, peer: PeerId) { + self.detach_watch_plane(peer); + } + + /// Publish what every armed watch saw since the last unit of work. A lone + /// engine has no system volume, so nothing can be watched on one. + fn take_owed_wakes(&mut self, owed: &mut [Option]) -> usize { + self.settle_watches(|_, _, _| Err(FsError::AccessLost)); + self.owed_watch_wakes(owed) + } } impl FsService

{ @@ -429,6 +445,19 @@ impl FsService

{ Err(error) => FsResponse::Refused(error), }; } + FsRequest::Watch { handle, slot, tag } => { + let target = self.watch_target(session, handle); + self.arm_watch( + sv.via(), + session, + slot, + tag, + crate::router::Store::User, + handle, + target, + ) + } + FsRequest::Unwatch { slot } => self.disarm_watch(sv.via(), session, slot), FsRequest::MatchView { handle, announced } => { let Some(mut input) = sv.returned_input(announced) else { return FsResponse::Refused(FsError::AccessDenied); @@ -501,6 +530,14 @@ impl SvcProtocol for RamService { const REPLY_GRANTS: usize = ::REPLY_GRANTS; const SERVE_PAIRS: usize = ::SERVE_PAIRS; + fn init_pair_window( + window: &mut [u8], + req_cap: u32, + rep_cap: u32, + ) -> Result<(), kore_ring::spsc::RingError> { + DagfsProtocol::init_pair_window(window, req_cap, rep_cap) + } + fn reply_matches(request: &FsReqWire, response: &FsResponse, grants: u8) -> bool { DagfsProtocol::reply_matches(request, response, grants) } diff --git a/native/services/dagfs/src/service/mod.rs b/native/services/dagfs/src/service/mod.rs index 123f0c9..51e04a5 100644 --- a/native/services/dagfs/src/service/mod.rs +++ b/native/services/dagfs/src/service/mod.rs @@ -65,7 +65,7 @@ use kore_rt::{ use kore_svc_extract::schema::InputIdentity; use crate::{ - ServiceAlloc, + ObjectId, ServiceAlloc, authority::{GrantId, Handle, Retired, Rights, SessionId}, block_on, bounds::{FIRST_TASK_VIEW_SESSION, MAX_SESSIONS}, @@ -369,6 +369,9 @@ pub struct FsService { /// immediately before delegating a status request; the default zeros /// honestly describe the bare volatile posture every other harness runs. backend: FsBackendStatus, + /// Which slots' watch planes have a watch armed or owe a wake + /// ([`crate::watches`]); the planes live in the slots' own records. + watches: crate::watches::WatchIndex, } /// What the mounted backend reports about itself for `SystemStatus` — set @@ -484,6 +487,11 @@ struct SessionSlot { /// The session's one private provider placement. The service already /// serializes dispatch; no additional lock or namespace object. replacement: Option>, + /// The recyclable peer's watch plane and its armed watches + /// ([`crate::watches`]), adopted from the pair window's tail at admission + /// and dropped before the host unmaps it. Only the primary frontend's + /// slots carry one: a secondary engine's watches are recorded here too. + watch: Option, } impl SessionSlot { @@ -495,6 +503,7 @@ impl SessionSlot { task_view: None, deferring: false, replacement: None, + watch: None, } } } @@ -601,9 +610,151 @@ impl FsService

{ pending_reclaims: Vec::new_in(crate::service_alloc()), pending_aborts: Vec::new_in(crate::service_alloc()), backend: FsBackendStatus::default(), + watches: crate::watches::WatchIndex::new(), } } + /// Adopt `peer`'s watch plane from its pair window's tail into the slot's + /// record, claiming the record as admission does. Refused for a fixed + /// pair, a heap that cannot hold the record, or a slot whose previous + /// plane never detached — a host/service desync admission must not paper + /// over. A tail without a formatted plane admits the peer with none: the + /// plane is what watches publish into, not what the pair needs to serve, + /// so such a client's `Watch` answers `Unsupported`. + pub(crate) fn attach_watch_plane(&mut self, peer: PeerId, aux: &'static mut [u8]) -> bool { + let slot = peer.slot(); + if slot == 0 { + return false; + } + let Ok(state) = self.slots.claim(slot) else { + return false; + }; + if state.watch.is_some() { + return false; + } + state.watch = crate::watches::Plane::adopt(peer, aux); + true + } + + /// Drop `peer`'s watch plane before its window is unmapped. A late call + /// for an earlier occupant is inert. + pub(crate) fn detach_watch_plane(&mut self, peer: PeerId) { + let slot = peer.slot(); + if let Some(state) = self.slots.get_mut(slot) + && state.watch.as_ref().is_some_and(|plane| plane.peer() == peer) + { + state.watch = None; + self.watches.forget(slot); + } + } + + /// The plane of the pair that delivered a request (`via`) for `session`; + /// `Unsupported` for a fixed pair or a peer admitted without one. + fn watch_plane_mut(&mut self, via: u16, session: SessionId) -> FsResult<&mut crate::watches::Plane> { + self.slots + .get_mut(usize::from(session.index())) + .and_then(|state| state.watch.as_mut()) + .filter(|plane| plane.peer().arrival() == via) + .ok_or(FsError::Unsupported) + } + + /// Arm a watch on a directory of `store`, resolved by that store's engine + /// (`target`), into the plane of the pair that asked. + #[allow(clippy::too_many_arguments)] + pub(crate) fn arm_watch( + &mut self, + via: u16, + session: SessionId, + slot: u8, + tag: u32, + store: crate::router::Store, + handle: Handle, + target: FsResult<(ObjectId, u64)>, + ) -> FsResponse { + let armed = self + .watch_plane_mut(via, session) + .and_then(|plane| { + let owed = plane.arm(slot, tag, store, handle, target?)?; + Ok((plane.armed(), owed)) + }); + match armed { + Ok((armed, owed)) => { + self.watches.note(usize::from(session.index()), armed, owed); + FsResponse::Done + } + Err(error) => FsResponse::Refused(error), + } + } + + pub(crate) fn disarm_watch(&mut self, via: u16, session: SessionId, slot: u8) -> FsResponse { + let disarmed = self.watch_plane_mut(via, session).and_then(|plane| { + let owed = plane.disarm(slot)?; + Ok((plane.armed(), owed)) + }); + match disarmed { + Ok((armed, owed)) => { + self.watches.note(usize::from(session.index()), armed, owed); + FsResponse::Done + } + Err(error) => FsResponse::Refused(error), + } + } + + /// The change epoch a watch on `handle` follows, read through this + /// engine — the secondary half of a dual-router settle. + pub(crate) fn watch_epoch(&self, session: SessionId, handle: Handle, dir: ObjectId) -> FsResult { + self.engine.watch_epoch(session, handle, dir) + } + + /// What a watch on `handle` would follow in this engine. + pub(crate) fn watch_target(&self, session: SessionId, handle: Handle) -> FsResult<(ObjectId, u64)> { + self.engine.watch_target(session, handle) + } + + /// Publish every armed watch whose directory changed. This engine answers + /// for the user store; `system` for the dual router's system volume (a + /// lone engine passes one that refuses, which ends any such watch). + pub(crate) fn settle_watches( + &mut self, + mut system: impl FnMut(SessionId, Handle, ObjectId) -> FsResult, + ) { + if self.watches.idle() { + return; + } + let engine = &self.engine; + for slot in self.watches.armed() { + let Some(plane) = self.slots.get_mut(slot).and_then(|state| state.watch.as_mut()) else { + self.watches.forget(slot); + continue; + }; + let owed = plane.settle(SessionId::new(slot as u16), |store, session, handle, dir| { + match store { + crate::router::Store::User => engine.watch_epoch(session, handle, dir), + crate::router::Store::System => system(session, handle, dir), + } + }); + self.watches.note(slot, plane.armed(), owed); + } + } + + /// Peers owed a watch-plane wake, for the serving host: up to + /// `out.len()`, each handed over once. + pub(crate) fn owed_watch_wakes(&mut self, out: &mut [Option]) -> usize { + let slots = &self.slots; + let mut taken = 0; + self.watches.take_owed(|slot| { + if taken == out.len() { + return false; + } + if let Some(plane) = slots.get(slot).and_then(|state| state.watch.as_ref()) { + out[taken] = Some(plane.peer()); + taken += 1; + } + true + }); + taken + } + /// Replace the backend posture snapshot folded into `SystemStatus` /// answers. The mount owner calls this immediately before delegating a /// status request, so the record reads as one moment. @@ -1140,6 +1291,7 @@ impl FsService

{ if let Some(state) = self.slots.get_mut(slot) { *state = SessionSlot::vacant(); } + self.watches.forget(slot); true } @@ -2466,6 +2618,13 @@ impl FsService

{ self.engine.sync(session, handle).await?; FsResponse::Done } + // A watch publishes into the asking pair's plane, which only the + // pair-aware layer above names (`serve_request_async`, or the dual + // router); reaching the session-only dispatch is a composition + // that forwarded the verb past its owner. + FsRequest::Watch { .. } | FsRequest::Unwatch { .. } => { + return Err(FsError::Unsupported); + } FsRequest::LendVersion { handle, window_offset, @@ -3659,6 +3818,233 @@ mod recyclable_session_tests { assert_eq!(native.stable_object, ls.stable_object); } + /// A recyclable peer's watch plane as exec formats it after the rings, + /// leaked like the host's mapping of it, with the client's half beside + /// the service's. + fn watch_plane() -> ( + &'static mut [u8], + kore_dagfs::watch::WatchSink<'static>, + ) { + #[repr(align(64))] + struct Plane([u8; kore_dagfs::watch::PLANE_BYTES]); + let plane = &mut alloc::boxed::Box::leak(alloc::boxed::Box::new(Plane( + [0; kore_dagfs::watch::PLANE_BYTES], + ))) + .0; + kore_ring::event::init::( + plane, + kore_abi::svc::FS_WATCH_RECORDS, + ) + .unwrap(); + let (base, len) = (plane.as_mut_ptr(), plane.len()); + // SAFETY: the two halves alias the plane only through the channel + // protocol, as the client's and the service's mappings do on metal. + let client = unsafe { core::slice::from_raw_parts_mut(base, len) }; + (plane, kore_dagfs::watch::sink(client).unwrap()) + } + + /// Two peers of one task view: a watcher holding its root, and a writer + /// that changes it. + fn watched_view() -> (RamService, SessionId, SessionId) { + let mut service = RamService::default(); + assert!(service.establish_admin_source()); + service + .register_private_invocation_view( + 0xD0, + &Name::new("terminal-0").unwrap(), + Rights::ALL, + None, + ) + .unwrap(); + let watcher = admit_view(&mut service, 0x106, 0xD0, Rights::ALL); + let writer = admit_view(&mut service, 0x105, 0xD0, Rights::ALL); + (service, watcher, writer) + } + + fn settle(service: &mut RamService, owed: &mut [Option]) -> usize { + ::take_owed_wakes(service, owed) + } + + #[test] + fn a_watch_counts_another_peer_s_change_and_owes_a_parked_client_one_wake() { + use kore_dagfs::watch::{Cursor, Seen}; + let (mut service, watcher, writer) = watched_view(); + let peer = PeerId::from_arrival(0x106).unwrap(); + let (plane, sink) = watch_plane(); + assert!(service.attach_watch_plane(peer, plane)); + let root = service.engine.open_root(watcher, Rights::READ).unwrap(); + let target = service.watch_target(watcher, root.handle); + assert_eq!( + service.arm_watch(0x106, watcher, 0, 7, crate::router::Store::User, root.handle, target), + FsResponse::Done + ); + let mut cursor = Cursor::new(0, 7).unwrap(); + assert_eq!(cursor.observe(&sink.state().unwrap()), Seen::Same); + + // Nothing changed: nothing published, nobody owed. + let mut owed = [None; 4]; + assert_eq!(settle(&mut service, &mut owed), 0); + + // The client parks; the other peer makes a file in the shared view. + let seen = sink.seq(); + assert!(sink.park(seen)); + let writer_root = service.engine.open_root(writer, Rights::ALL).unwrap(); + crate::block_on(service.engine.create( + writer, + writer_root.handle, + &Name::new("new.txt").unwrap(), + ObjectKind::File, + )) + .unwrap(); + assert_eq!(settle(&mut service, &mut owed), 1); + assert_eq!(owed[0], Some(peer)); + assert_eq!(settle(&mut service, &mut owed), 0, "one change, one wake"); + assert_ne!(sink.seq(), seen); + assert_eq!(cursor.observe(&sink.state().unwrap()), Seen::Changed); + + // A running client is owed nothing for the next change; it reads it. + let created = crate::block_on(service.engine.create( + writer, + writer_root.handle, + &Name::new("more.txt").unwrap(), + ObjectKind::File, + )) + .unwrap(); + assert_eq!(settle(&mut service, &mut owed), 0); + assert_eq!(cursor.observe(&sink.state().unwrap()), Seen::Changed); + + // A child's bytes changing is a change to what the folder shows. + crate::block_on(service.engine.write_at( + writer, + created.handle, + 0, + b"bytes", + crate::ANY_REVISION, + )) + .unwrap(); + assert_eq!(settle(&mut service, &mut owed), 0); + assert_eq!(cursor.observe(&sink.state().unwrap()), Seen::Changed); + + // Closing the watched handle ends the watch: the plane stops + // counting for a route the client let go of. + service.engine.close(watcher, root.handle).unwrap(); + assert_eq!(settle(&mut service, &mut owed), 0); + assert_eq!(cursor.observe(&sink.state().unwrap()), Seen::Ended); + assert_eq!(service.watches.armed_count(), 0); + } + + /// A plane lives in its serving slot's own record, so a peer far past + /// the host's bootstrap table — the seventh program reading files, and + /// the two-hundredth — watches as the first does. + #[test] + fn a_peer_in_a_high_slot_has_its_own_plane() { + use kore_dagfs::watch::{Cursor, Seen}; + let (mut service, _, writer) = watched_view(); + let high = admit_view(&mut service, 0x1C8, 0xD0, Rights::ALL); + assert_eq!(high, SessionId::new(0xC8)); + let peer = PeerId::from_arrival(0x1C8).unwrap(); + let (plane, sink) = watch_plane(); + assert!(service.attach_watch_plane(peer, plane)); + let root = service.engine.open_root(high, Rights::READ).unwrap(); + let target = service.watch_target(high, root.handle); + assert_eq!( + service.arm_watch(0x1C8, high, 3, 11, crate::router::Store::User, root.handle, target), + FsResponse::Done + ); + let mut cursor = Cursor::new(3, 11).unwrap(); + let writer_root = service.engine.open_root(writer, Rights::ALL).unwrap(); + crate::block_on(service.engine.create( + writer, + writer_root.handle, + &Name::new("far.txt").unwrap(), + ObjectKind::File, + )) + .unwrap(); + let mut owed = [None; 4]; + settle(&mut service, &mut owed); + assert_eq!(cursor.observe(&sink.state().unwrap()), Seen::Changed); + // Closing the peer wipes the slot, plane and index together. + service.detach_watch_plane(peer); + assert!(service.peer_closed(0x1C8, PeerClose::Orderly)); + assert_eq!(service.watches.armed_count(), 0); + } + + #[test] + fn a_watch_takes_a_listing_s_authority_and_a_plane_of_the_asking_pair() { + let (mut service, watcher, _) = watched_view(); + let peer = PeerId::from_arrival(0x106).unwrap(); + let root = service.engine.open_root(watcher, Rights::ALL).unwrap(); + let file = crate::block_on(service.engine.create( + watcher, + root.handle, + &Name::new("f").unwrap(), + ObjectKind::File, + )) + .unwrap(); + let user = crate::router::Store::User; + + // No plane attached yet: nothing to publish into. + let target = service.watch_target(watcher, root.handle); + assert_eq!( + service.arm_watch(0x106, watcher, 0, 1, user, root.handle, target), + FsResponse::Refused(FsError::Unsupported) + ); + let (plane, sink) = watch_plane(); + assert!(service.attach_watch_plane(peer, plane)); + // A second plane for the occupied slot is refused. + let (again, _) = watch_plane(); + assert!(!service.attach_watch_plane(peer, again)); + // A tail that carries no plane admits its peer without one. + let bare: &'static mut [u8] = alloc::boxed::Box::leak(alloc::boxed::Box::new([0u8; 64])); + assert!(service.attach_watch_plane(PeerId::from_arrival(0x105).unwrap(), bare)); + // Another generation of the slot names no plane of its own. + let target = service.watch_target(watcher, root.handle); + assert_eq!( + service.arm_watch(0x206, watcher, 0, 1, user, root.handle, target), + FsResponse::Refused(FsError::Unsupported) + ); + // A file is not watched: a listing is what a watch follows. + let target = service.watch_target(watcher, file.handle); + assert_eq!( + service.arm_watch(0x106, watcher, 0, 1, user, file.handle, target), + FsResponse::Refused(FsError::NotDirectory) + ); + // A slot past the plane, or the zero tag, is out of range. + for (slot, tag) in [(kore_abi::svc::FS_WATCH_SLOTS as u8, 1), (0, 0)] { + let target = service.watch_target(watcher, root.handle); + assert_eq!( + service.arm_watch(0x106, watcher, slot, tag, user, root.handle, target), + FsResponse::Refused(FsError::Range) + ); + } + assert_eq!(service.watches.armed_count(), 0); + + // Re-arming a slot replaces its watch; disarming clears its tag. + for tag in [3, 4] { + let target = service.watch_target(watcher, root.handle); + assert_eq!( + service.arm_watch(0x106, watcher, 1, tag, user, root.handle, target), + FsResponse::Done + ); + } + assert_eq!(service.watches.armed_count(), 1); + assert_eq!(sink.state().unwrap().tag[1], 4); + assert_eq!(service.disarm_watch(0x106, watcher, 1), FsResponse::Done); + assert_eq!(sink.state().unwrap().tag[1], 0); + assert_eq!(service.watches.armed_count(), 0); + + // Detaching the plane before the window goes drops its watches. + let target = service.watch_target(watcher, root.handle); + assert_eq!( + service.arm_watch(0x106, watcher, 2, 9, user, root.handle, target), + FsResponse::Done + ); + service.detach_watch_plane(PeerId::from_arrival(0x206).unwrap()); + assert_eq!(service.watches.armed_count(), 1, "a late detach for another generation is inert"); + service.detach_watch_plane(peer); + assert_eq!(service.watches.armed_count(), 0); + } + /// A dead writer's staged replacement is released on the service's next /// dispatch, never inside the synchronous peer-death hook. A release the /// provider refuses stays pending — custody retained — while the dead diff --git a/native/services/dagfs/src/watches.rs b/native/services/dagfs/src/watches.rs new file mode 100644 index 0000000..c7800aa --- /dev/null +++ b/native/services/dagfs/src/watches.rs @@ -0,0 +1,310 @@ +//! Directory watches: the service half of the watch plane +//! (`kore_dagfs::watch`, `res/DAGFS.md` §4). +//! +//! A session arms up to [`FS_WATCH_SLOTS`] watches, each on a directory +//! handle it holds. After every unit of service work each armed directory's +//! change epoch ([`Namespace::change_epoch`]) is read **through the handle's +//! route** — revalidated as every operation's is — and the plane republished +//! when one moved. Only a client parked on its plane is owed a wake; the +//! serving host sends each one notice (`SystemService::take_owed_wakes`). +//! +//! Nothing here is global. A peer's [`Plane`] lives in its serving slot's own +//! record, beside the slot's window and staged replacement, allocated with it +//! and wiped with it; the service keeps only [`WatchIndex`] — which slots have +//! a watch armed, and which owe a wake — so a settle visits exactly the slots +//! that need it, however large the slot space grows. A watch on the dual +//! router's system volume is recorded in the same plane with the store it +//! lives in, so the one plane a client holds carries both. +//! +//! [`Namespace::change_epoch`]: crate::namespace::Namespace::change_epoch + +use kore_abi::svc::{FS_WATCH_SLOTS, FsWatchState}; +use kore_dagfs::watch::{self, WatchSource, state}; +use kore_rt::svc::PeerId; + +use crate::{ + ObjectId, + authority::{Handle, SessionId}, + bounds::PAIR_SESSIONS, + error::{FsError, FsResult}, + router::Store, +}; + +/// One armed slot. +#[derive(Clone, Copy, Debug)] +struct Watch { + store: Store, + handle: Handle, + dir: ObjectId, + /// The epoch the plane last reported for this watch. + seen: u64, +} + +/// One recyclable peer's watch plane and the watches published into it. +pub(crate) struct Plane { + peer: PeerId, + source: WatchSource<'static>, + slots: [Option; FS_WATCH_SLOTS], + /// The plane's state changed since its last commit. + dirty: bool, +} + +impl Plane { + /// `peer`'s plane over its pair window's tail (`aux`, as the host hands it + /// to `attach_peer_aux`), or `None` for a tail without a formatted plane. + /// Starts from nothing armed, whatever the window last carried. + pub(crate) fn adopt(peer: PeerId, aux: &'static mut [u8]) -> Option { + let mut source = watch::source(aux).ok()?; + if source.replace_state(FsWatchState::default()) { + let _ = source.commit(); + } + Some(Self { + peer, + source, + slots: [None; FS_WATCH_SLOTS], + dirty: false, + }) + } + + pub(crate) const fn peer(&self) -> PeerId { + self.peer + } + + /// Whether any slot is armed. + pub(crate) fn armed(&self) -> bool { + self.slots.iter().any(Option::is_some) + } + + fn edit(&mut self, edit: impl FnOnce(&mut FsWatchState)) { + self.source.state(edit); + self.dirty = true; + } + + /// Publish a batch: `true` when its commit found the client parked, and + /// so owes it one notice. + fn commit(&mut self) -> bool { + core::mem::take(&mut self.dirty) && self.source.commit() + } + + /// Arm `slot` under `tag` on the directory `dir` at epoch `seen` (its + /// baseline: only a later change counts), replacing what the slot held. + /// Answers whether a parked client is owed a wake. + pub(crate) fn arm( + &mut self, + slot: u8, + tag: u32, + store: Store, + handle: Handle, + (dir, seen): (ObjectId, u64), + ) -> FsResult { + let slot = usize::from(slot); + if slot >= FS_WATCH_SLOTS || tag == 0 { + return Err(FsError::Range); + } + self.slots[slot] = Some(Watch { + store, + handle, + dir, + seen, + }); + self.edit(|s| state::arm(s, slot, tag)); + Ok(self.commit()) + } + + /// Disarm `slot`, armed or not. Answers whether a parked client is owed + /// a wake. + pub(crate) fn disarm(&mut self, slot: u8) -> FsResult { + let slot = usize::from(slot); + if slot >= FS_WATCH_SLOTS { + return Err(FsError::Range); + } + self.slots[slot] = None; + self.edit(|s| state::disarm(s, slot)); + Ok(self.commit()) + } + + /// Read every armed watch's epoch through `epoch` — the engine of its + /// store, revalidating its route — and publish what moved. A watch whose + /// route no longer proves ends; one whose directory was mid-publish + /// (`Busy`) counts as changed and keeps its baseline, so the next reading + /// reports the change again rather than losing it. Answers whether a + /// parked client is owed a wake. + pub(crate) fn settle( + &mut self, + session: SessionId, + mut epoch: impl FnMut(Store, SessionId, Handle, ObjectId) -> FsResult, + ) -> bool { + for slot in 0..FS_WATCH_SLOTS { + let Some(watch) = self.slots[slot].as_mut() else { + continue; + }; + match epoch(watch.store, session, watch.handle, watch.dir) { + Ok(now) if now == watch.seen => {} + Ok(now) => { + watch.seen = now; + self.edit(|s| state::changed(s, slot)); + } + Err(FsError::Busy) => self.edit(|s| state::changed(s, slot)), + Err(_) => { + self.slots[slot] = None; + self.edit(|s| state::end(s, slot)); + } + } + } + self.commit() + } +} + +/// Words of a set over every pair session's slot. +const WORDS: usize = PAIR_SESSIONS.div_ceil(64); + +/// A set of pair-session slots: one bit each, scanned word by word. +#[derive(Clone, Copy, Debug, Default)] +struct SlotSet([u64; WORDS]); + +impl SlotSet { + fn insert(&mut self, slot: usize) { + if let Some(word) = self.0.get_mut(slot / 64) { + *word |= 1 << (slot % 64); + } + } + + fn remove(&mut self, slot: usize) { + if let Some(word) = self.0.get_mut(slot / 64) { + *word &= !(1 << (slot % 64)); + } + } + + fn is_empty(&self) -> bool { + self.0.iter().all(|word| *word == 0) + } + + /// The members, lowest first, as of the call. + fn members(&self) -> impl Iterator + use<> { + let words = self.0; + (0..WORDS).flat_map(move |at| { + let mut word = words[at]; + core::iter::from_fn(move || { + (word != 0).then(|| { + let bit = word.trailing_zeros() as usize; + word &= word - 1; + at * 64 + bit + }) + }) + }) + } +} + +/// Which slots' planes have a watch armed, and which owe their client a wake. +/// Derived from the planes and kept beside them: a settle and a wake +/// collection cost the slots they name, and nothing when the sets are empty. +#[derive(Debug, Default)] +pub(crate) struct WatchIndex { + armed: SlotSet, + owed: SlotSet, +} + +impl WatchIndex { + pub(crate) const fn new() -> Self { + Self { + armed: SlotSet([0; WORDS]), + owed: SlotSet([0; WORDS]), + } + } + + /// Record what an edit of `slot`'s plane left: whether a watch is still + /// armed there, and whether its client is owed a wake. + pub(crate) fn note(&mut self, slot: usize, armed: bool, owed: bool) { + if armed { + self.armed.insert(slot); + } else { + self.armed.remove(slot); + } + if owed { + self.owed.insert(slot); + } + } + + /// Forget `slot`: its plane is gone. + pub(crate) fn forget(&mut self, slot: usize) { + self.armed.remove(slot); + self.owed.remove(slot); + } + + /// The slots with a watch armed. + pub(crate) fn armed(&self) -> impl Iterator + use<> { + self.armed.members() + } + + pub(crate) fn idle(&self) -> bool { + self.armed.is_empty() + } + + /// Hand `out` each slot owing a wake, lowest first, taking it off the set; + /// `out` answers `false` when its batch is full, and the rest stay owed. + pub(crate) fn take_owed(&mut self, mut out: impl FnMut(usize) -> bool) { + for slot in self.owed.members() { + if !out(slot) { + return; + } + self.owed.remove(slot); + } + } + + #[cfg(test)] + pub(crate) fn armed_count(&self) -> usize { + self.armed.members().count() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_slot_set_names_its_members_lowest_first_across_words() { + let mut set = SlotSet::default(); + for slot in [PAIR_SESSIONS - 1, 3, 64, 65, 0] { + set.insert(slot); + } + set.insert(PAIR_SESSIONS); // past the space: ignored + let members: std::vec::Vec = set.members().collect(); + assert_eq!(members, [0, 3, 64, 65, PAIR_SESSIONS - 1]); + set.remove(64); + set.remove(0); + assert_eq!(set.members().collect::>(), [3, 65, PAIR_SESSIONS - 1]); + assert!(!set.is_empty()); + for slot in [3, 65, PAIR_SESSIONS - 1] { + set.remove(slot); + } + assert!(set.is_empty()); + } + + #[test] + fn owed_wakes_are_taken_once_and_a_full_batch_keeps_the_rest() { + let mut index = WatchIndex::new(); + for slot in [2, 7, 9] { + index.note(slot, true, true); + } + let mut taken = std::vec::Vec::new(); + index.take_owed(|slot| { + if taken.len() == 2 { + return false; + } + taken.push(slot); + true + }); + assert_eq!(taken, [2, 7]); + taken.clear(); + index.take_owed(|slot| { + taken.push(slot); + true + }); + assert_eq!(taken, [9], "the batch's leftover, and nothing twice"); + index.forget(9); + assert_eq!(index.armed().collect::>(), [2, 7]); + index.note(2, false, false); + index.note(7, false, false); + assert!(index.idle()); + } +} diff --git a/native/services/device/boot.toml b/native/services/device/boot.toml index 80737c4..96c7f02 100644 --- a/native/services/device/boot.toml +++ b/native/services/device/boot.toml @@ -47,5 +47,5 @@ protocol = "kore_svc_device::DeviceBroker" # row's own claimant band. instances = 8 endows = ["ROOT_COMPLEX@0", "PLATFORM_BUS@0", "FACE_SUPPLY@0"] -digest_x86_64 = "blake3:6eb5411353e5cf48b9c9497d3b4868207bb8bbade4827537be4a7af3492d4a74" # res/x86_64/; make boot-digests -digest_aarch64 = "blake3:cd930094ae3f3fe8145f9a8fd61e5c522bc39b4956f4f45aadd1a66726aa3098" # res/aarch64/; make boot-digests-aarch64 +digest_x86_64 = "blake3:5a3ff2252bef814e79727f873c89de62565ab491f8065599ada4db348c829223" # res/x86_64/; make boot-digests +digest_aarch64 = "blake3:f00fd8b3aad6308541c51f785e3651dc793b60652a821ffee9aa7d79b49eb615" # res/aarch64/; make boot-digests-aarch64 diff --git a/native/services/device/src/service/mod.rs b/native/services/device/src/service/mod.rs index 3e1cf26..4e0e6b1 100644 --- a/native/services/device/src/service/mod.rs +++ b/native/services/device/src/service/mod.rs @@ -3,7 +3,7 @@ use kore_abi::device::{DeviceRecord, DeviceSpecies}; use kore_abi::session::SlotRef; -use kore_abi::task::{FACE_RING_STRIDE, FACE_SUPPLY_BYTES, role}; +use kore_abi::task::{CLAIMANT_FACE_SUPPLY_BYTES, FACE_RING_STRIDE, role}; use kore_rt::cap::{PlatformBusToken, RootComplexToken, SupplyToken, verbs}; use kore_rt::entry::FixedSlotBand; use kore_rt::service::{ @@ -34,8 +34,11 @@ pub const DEVICE_GRANT_RIGHTS: Rights = verbs::DEVICE_OWNER_RIGHTS; /// Claimant faces the broker's [`role::FACE_SUPPLY`] band strides into — /// the geometry contract's own quotient, so the count can never disagree -/// with the span the founder mints. -pub const FACE_COUNT: usize = (FACE_SUPPLY_BYTES / FACE_RING_STRIDE) as usize; +/// with the span the founder mints: one face per driver row +/// ([`MAX_CLAIMANT_FACES`](kore_abi::device::MAX_CLAIMANT_FACES)). +pub const FACE_COUNT: usize = (CLAIMANT_FACE_SUPPLY_BYTES / FACE_RING_STRIDE) as usize; + +const _: () = assert!(FACE_COUNT == kore_abi::device::MAX_CLAIMANT_FACES); /// The first minted face's [`Serve::via`] identity. Minted vias start at /// [`MAX_SERVE_PAIRS`](kore_rt::svc::MAX_SERVE_PAIRS) — the host's rule @@ -59,6 +62,14 @@ const FACE_LAND_BASE: u32 = kore_abi::task::MAX_BOOT_HANDLES as u32 + kore_rt::entry::SERVE_SCRATCH_SLOTS as u32 + 2; const FACE_LAND_END: u32 = FACE_LAND_BASE + 2 * FACE_COUNT as u32; +// `claim_fixed_slot_band` admits nothing at or past the startup-link band, +// so a band that crosses it refuses readiness (detail 0x204) on every +// machine. Checked here, where the face count and the band meet. +const _: () = assert!( + FACE_LAND_END <= kore_rt::entry::STARTUP_LINK_SLOT_BASE, + "the claimant landing band must end below the startup-link band" +); + /// A conveyed face leg must at least be postable by the broker. const FACE_LEG_RIGHTS: Rights = Rights::ACCESS; @@ -915,11 +926,11 @@ impl SystemService for DeviceBroker { if ::SERVE_PAIRS > 1 { return Err(StartFailure::new(0x205)); } - if supply.len() != FACE_SUPPLY_BYTES { + if supply.len() != CLAIMANT_FACE_SUPPLY_BYTES { return Err(StartFailure::new(0x200)); } let win = verbs::map_held_rw(supply.slot()).ok_or_else(|| StartFailure::new(0x201))?; - if win.len() < FACE_SUPPLY_BYTES as usize { + if win.len() < CLAIMANT_FACE_SUPPLY_BYTES as usize { return Err(StartFailure::new(0x202)); } let band = rt @@ -1178,7 +1189,7 @@ impl SystemService for DeviceBroker { // image face table's recycled-slot hygiene. // SAFETY: the view is live for the task's life // (mapped once at start, length gated ≥ - // FACE_SUPPLY_BYTES there); this face is not Live, + // CLAIMANT_FACE_SUPPLY_BYTES there); this face is not Live, // so no attached pair aliases this stride // (occupancy is the exclusion discipline); the // `&mut` ends before the spec is exposed. @@ -1370,11 +1381,15 @@ mod face_tests { } /// The band geometry quotient: the face count times the stride is - /// exactly the supply span the founder mints (`FACE_SUPPLY_BYTES`), - /// and the landing band holds exactly two slots per face. + /// exactly the supply span the founder mints + /// (`CLAIMANT_FACE_SUPPLY_BYTES`), and the landing band holds exactly + /// two slots per face. #[test] fn the_face_geometry_is_the_contracts_quotient() { - assert_eq!(FACE_COUNT as u64 * FACE_RING_STRIDE, FACE_SUPPLY_BYTES); + assert_eq!( + FACE_COUNT as u64 * FACE_RING_STRIDE, + CLAIMANT_FACE_SUPPLY_BYTES + ); assert_eq!(FACE_LAND_END - FACE_LAND_BASE, 2 * FACE_COUNT as u32); } } diff --git a/native/services/mm/boot.toml b/native/services/mm/boot.toml index 094ac5a..b6412e0 100644 --- a/native/services/mm/boot.toml +++ b/native/services/mm/boot.toml @@ -54,5 +54,5 @@ supply_quota_bytes = 27917287424 # 26 GiB endows = ["MEM_SUPPLY@0", "MEM_REGION@0", "SYSTEM_OBSERVE@0"] # review mirror; asserted === PROFILE.roles shell_client = true exec_client = true -digest_x86_64 = "blake3:0c893dbdf58c6c17dd66d424ad025ae0cc5b4eab41a718940f09a3595c32830a" # res/x86_64/; make boot-digests -digest_aarch64 = "blake3:f1ffc4cb9f02cc59ef78781fd8a1cba3722d9518b7abb0adb17bf3f7376b9f57" # res/aarch64/; make boot-digests-aarch64 +digest_x86_64 = "blake3:041a33515a21fff195c4cab320e9c8e64cbf775f04f01b7dbe40fa90be5a31a7" # res/x86_64/; make boot-digests +digest_aarch64 = "blake3:e17d59e57476b1db9554bc889640ffd3f02b6e2c98d509d93dea1c7efadb2a2a" # res/aarch64/; make boot-digests-aarch64 diff --git a/native/services/swap/boot.toml b/native/services/swap/boot.toml index b433dd4..0a1028a 100644 --- a/native/services/swap/boot.toml +++ b/native/services/swap/boot.toml @@ -9,5 +9,5 @@ image = "kore-svc-swap" endowment = "kore_svc_swap::ZramEndowment" protocol = "kore_svc_swap::ZramSwap" endows = ["MEM_SUPPLY@0"] -digest_x86_64 = "blake3:5f77376810266176274a0af803d6b16bb9d750d97780b171e5fba376fcbb58fc" # res/x86_64/; make boot-digests -digest_aarch64 = "blake3:b939b2b8bd153674b2bf76dfa83e0941cc3f16139c721c5271f6a75b6fea0874" # res/aarch64/; make boot-digests-aarch64 +digest_x86_64 = "blake3:bc3ff6fbb123b3b44645e959346e320519b9ab5f43b5c999d0685ee9c7fed141" # res/x86_64/; make boot-digests +digest_aarch64 = "blake3:6495b930db0d4e4ca6ce41b0d19580ea1d816748358cc9c8bceef150ec0ccc30" # res/aarch64/; make boot-digests-aarch64 diff --git a/tools/check-bindings.py b/tools/check-bindings.py index b389550..2c6c9a6 100644 --- a/tools/check-bindings.py +++ b/tools/check-bindings.py @@ -21,7 +21,8 @@ for arch, c_target, zig_target in TARGETS: subprocess.run([os.environ.get('CLANG', 'clang'), '-target', c_target, '-ffreestanding', '-std=c11', '-Werror', '-fsyntax-only', '-I', 'extern/c/include', - 'crates/ffi/tests/consumer.c', 'crates/ffi/tests/alloc.c'], cwd=ROOT, check=True) + 'crates/ffi/tests/consumer.c', 'crates/ffi/tests/alloc.c', + 'crates/c/tests/program.c'], cwd=ROOT, check=True) translated = destination / f'{arch}-c.zig' with translated.open('w') as output: subprocess.run([zig, 'translate-c', '-target', zig_target, 'extern/c/include/kore.h'], diff --git a/tools/check-program.py b/tools/check-program.py new file mode 100755 index 0000000..c6b0a1b --- /dev/null +++ b/tools/check-program.py @@ -0,0 +1,145 @@ +#!/usr/bin/env python3 +"""Link libkore.so for every Kore ISA and check the image a C program binds. + +The image is a distribution's one-line cdylib over kore-c (Kore's is +user/runtime/libkore), linked with the contract kore-c documents; this builds +that crate in a temporary workspace, as check-entry.py builds its consumers. +The library is what an independent C program on Kore links with -lkore, so the +checks are the ones its loader and its consumers rely on: + +- a shared object with no entry point: Kore's image gate classifies a module by + its entry, and a library must never register as a program; +- the soname consumers record, nothing it needs, nothing left undefined; +- static initial-exec TLS only (Kore's startup linker resolves no dynamic TLS); +- exactly the declared ABI exported: every function kore.h and kore/program.h + declare, the C names kore/stdlib.h and kore/pthread.h alias, the C + library's two lifecycle entry points, and the C library names Kore's + hosted headers declare and libkore.so implements; no other symbol. +""" +import argparse +import json +import os +from pathlib import Path +import re +import subprocess +import tempfile +import tomllib + +ROOT = Path(__file__).resolve().parents[1] +ARCHES = ('x86_64', 'aarch64', 'riscv64gc') +HEADERS = ('extern/c/include/kore.h', 'extern/c/include/kore/program.h') +# kore-ffi's `c-stdlib` names: allocation, the pthread lifecycle, and the +# compiler ABI's thread-exit destructor hooks. +C_STDLIB = {'malloc', 'calloc', 'realloc', 'free', 'pthread_create', 'pthread_join', + 'pthread_detach', 'pthread_cancel', 'pthread_key_create', 'pthread_key_delete', + 'pthread_setspecific', 'pthread_getspecific', 'pthread_setcancelstate', + 'pthread_setcanceltype', 'pthread_testcancel', 'pthread_exit', + '__cxa_thread_atexit', '__cxa_thread_atexit_impl'} +# The C library's start and exit call these when a program links libkore. +LIFECYCLE = {'__kore_program_start', '__kore_program_exit', '__kore_program_arguments'} +# The rest of the C library a program's headers declare (Kore's TCC +# `include/`: , , , ), which libkore.so +# implements: kore-c-shim's printf family, streams, parsers, string routines +# and per-thread errno, and kore-c's own abort; and , and +# 's open, close, mmap and munmap over the capabilities the +# program holds. The program's libc.a holds the rest: exit, atexit and the +# routines a compiler assumes. +C_LIBRARY = {'printf', 'fprintf', 'sprintf', 'snprintf', 'vprintf', 'vfprintf', 'vsprintf', + 'vsnprintf', 'fputc', 'putc', 'putchar', 'fputs', 'puts', 'fwrite', 'fflush', + 'fclose', 'stdout', 'stderr', + 'strtol', 'strtoll', 'strtoul', 'strtoull', 'atoi', 'atol', 'strtod', 'strtof', + 'qsort', 'strerror', 'abort', + 'memchr', 'strchr', 'strrchr', 'strcmp', 'strncmp', 'strcpy', 'strncpy', 'strncat', + 'strstr', + '__errno_location', + 'open', 'close', 'mmap', 'munmap'} +DYNAMIC_TLS = re.compile(r'TLSDESC|DTPMOD|DTPOFF|TLSGD|TLSLD|TLS_DTP') + + +def readelf(*args): + return subprocess.check_output([os.environ.get('LLVM_READELF', 'llvm-readelf'), '-W', *args], + text=True) + + +def declared(): + names = set() + for header in HEADERS: + names |= set(re.findall(r'\b(kore_[a-z][a-z0-9_]*)\s*\(', (ROOT / header).read_text())) + if not names: + raise SystemExit('no foreign functions found in the generated headers') + return names + + +# The shared object as a distribution builds it: a cdylib named `kore` that +# names kore-c, linked with the soname and no undefined symbol. +WRAPPER_MANIFEST = ('[package]\nname = "libkore-so"\nversion = "0.0.0"\nedition = "2024"\n' + '[lib]\nname = "kore"\ncrate-type = ["cdylib"]\n[workspace]\n' + '[dependencies]\nkore-c = {{ path = {path} }}\n') +WRAPPER_SOURCE = '#![no_std]\n// The runtime and its exports come from kore-c.\nuse kore_c as _;\n' +LINK_CONTRACT = ['-C', 'link-arg=--soname=libkore.so', '-C', 'link-arg=--no-undefined'] + + +def build(arch): + channel = tomllib.loads((ROOT / 'rust-toolchain.toml').read_text())['toolchain']['channel'] + target_dir = ROOT / 'target/program-consumer' + env = dict(os.environ, RUSTUP_TOOLCHAIN=channel, CARGO_TARGET_DIR=str(target_dir), + CARGO_INCREMENTAL='0') + with tempfile.TemporaryDirectory(prefix='libkore-so-') as temporary: + work = Path(temporary) + (work / 'src').mkdir() + (work / 'Cargo.toml').write_text( + WRAPPER_MANIFEST.format(path=json.dumps(str(ROOT / 'crates/c')))) + (work / 'src/lib.rs').write_text(WRAPPER_SOURCE) + # Reuse the reviewed dependency versions while Cargo adds this fixture. + (work / 'Cargo.lock').write_bytes((ROOT / 'Cargo.lock').read_bytes()) + subprocess.run([ + 'cargo', 'rustc', '--release', '-Zjson-target-spec', + '--target', str(ROOT / f'targets/{arch}-unknown-kore-user.json'), + '-Zbuild-std=core,alloc,compiler_builtins', + '-Zbuild-std-features=compiler-builtins-mem', '--', *LINK_CONTRACT, + ], cwd=work, env=env, check=True) + return target_dir / f'{arch}-unknown-kore-user/release/libkore.so' + + +def check(arch, expected): + image = build(arch) + header = readelf('--file-header', str(image)) + if not re.search(r'Type:\s+DYN', header): + raise SystemExit(f'{arch}: libkore.so is not a shared object') + if not re.search(r'Entry point address:\s+0x0\b', header): + raise SystemExit(f'{arch}: libkore.so has an entry point; it would register as a program') + dynamic = readelf('--dynamic', str(image)) + if 'Library soname: [libkore.so]' not in dynamic: + raise SystemExit(f'{arch}: libkore.so does not carry its soname') + if needed := re.findall(r'\(NEEDED\)\s+Shared library: \[([^\]]+)\]', dynamic): + raise SystemExit(f'{arch}: libkore.so needs other libraries: {needed}') + exported, undefined = set(), set() + for line in readelf('--dyn-syms', str(image)).splitlines(): + fields = line.split() + if len(fields) < 8 or not fields[0].rstrip(':').isdigit(): + continue + kind, binding, section, name = fields[3], fields[4], fields[6], fields[7] + if section == 'UND': + if name: + undefined.add(name) + elif binding in ('GLOBAL', 'WEAK') and kind in ('FUNC', 'OBJECT', 'TLS'): + exported.add(name.split('@')[0]) + if undefined: + raise SystemExit(f'{arch}: undefined symbols: {sorted(undefined)}') + if missing := expected - exported: + raise SystemExit(f'{arch}: declared but not exported: {sorted(missing)}') + if extra := exported - expected: + raise SystemExit(f'{arch}: exported outside the declared ABI: {sorted(extra)}') + relocations = readelf('--relocations', str(image)) + if found := sorted(set(DYNAMIC_TLS.findall(relocations))): + raise SystemExit(f'{arch}: dynamic TLS relocations {found}; static initial-exec TLS required') + print(f'libkore.so: {arch}: {len(exported)} exported symbols, the declared ABI exactly', flush=True) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument('--arch', choices=ARCHES) + args = parser.parse_args() + expected = declared() | C_STDLIB | LIFECYCLE | C_LIBRARY + for arch in [args.arch] if args.arch else ARCHES: + check(arch, expected) diff --git a/tools/generate-bindings.py b/tools/generate-bindings.py index 2ba0782..04ee299 100644 --- a/tools/generate-bindings.py +++ b/tools/generate-bindings.py @@ -19,6 +19,14 @@ def generate(check=False): subprocess.run([generator, '--config', 'crates/ffi/cbindgen.toml', '--crate', 'kore-ffi', '--output', str(header)], cwd=ROOT, check=True) source = header.read_text() + # What libkore.so adds for an independent C program; it includes kore.h + # and declares none of its types again. + program_header = Path(directory) / 'program.h' + subprocess.run([generator, '--config', 'crates/c/cbindgen.toml', '--crate', 'kore-c', + '--output', str(program_header)], cwd=ROOT, check=True) + program = program_header.read_text() + if 'kore_fs_run' not in program: + raise SystemExit('missing program export: kore_fs_run') # cbindgen is the declaration authority. This step only groups exported # names; it never reproduces signatures or guesses C/Rust type layouts. functions = sorted(set(re.findall(r'\b(kore_[a-z][a-z0-9_]*)\s*\(', source))) @@ -72,7 +80,8 @@ def generate(check=False): pthread += '#define pthread_cleanup_pop(execute) kore_thread_cleanup_pop(&_kore_cleanup, (execute)); } while (0)\n' pthread += '#endif\n' for relative, data in [('extern/c/include/kore.h', source), ('extern/zig/src/root.zig', zig), - ('extern/c/include/kore/stdlib.h', stdlib), ('extern/c/include/kore/pthread.h', pthread)]: + ('extern/c/include/kore/stdlib.h', stdlib), ('extern/c/include/kore/pthread.h', pthread), + ('extern/c/include/kore/program.h', program)]: path = ROOT / relative if check: if not path.exists() or path.read_text() != data: