From 24c9eabc559c3244a0b3fcdae5ef10e5c72f10ac Mon Sep 17 00:00:00 2001 From: Jacob Stephens Date: Mon, 26 Oct 2026 05:00:54 -0400 Subject: [PATCH 1/9] thirdshift: failed ticket run (codex exited 1: Selected model is at capacity. Please try a different model.) 2026-10-26T09:00:54Z, host thirdshift. Uncommitted work at the time of failure is included in this commit. --- src/logs/reading/metrics_tests.rs | 88 +++++++++++++++++++++++++++++++ src/logs/reading/tests.rs | 4 ++ 2 files changed, 92 insertions(+) create mode 100644 src/logs/reading/metrics_tests.rs diff --git a/src/logs/reading/metrics_tests.rs b/src/logs/reading/metrics_tests.rs new file mode 100644 index 000000000..b353b2aea --- /dev/null +++ b/src/logs/reading/metrics_tests.rs @@ -0,0 +1,88 @@ +//! Observe actual trace opens at the Command-details and Snapshot seams. +use std::ffi::CString; +use std::io::Read; +use std::os::fd::FromRawFd; +use std::os::unix::ffi::OsStrExt; + +use super::*; + +struct Opens { + file: fs::File, + paths: BTreeMap, +} + +impl Opens { + fn new(paths: &[PathBuf]) -> Self { + // SAFETY: the successful descriptor is owned exactly once by File. + let fd = unsafe { libc::inotify_init1(libc::IN_NONBLOCK | libc::IN_CLOEXEC) }; + assert!(fd >= 0, "{}", std::io::Error::last_os_error()); + let file = unsafe { fs::File::from_raw_fd(fd) }; + let paths = paths + .iter() + .map(|path| { + let name = CString::new(path.as_os_str().as_bytes()).unwrap(); + // SAFETY: name is NUL terminated and fd remains owned above. + let wd = unsafe { libc::inotify_add_watch(fd, name.as_ptr(), libc::IN_OPEN) }; + assert!(wd >= 0, "{}", std::io::Error::last_os_error()); + (wd, path.clone()) + }) + .collect(); + Self { file, paths } + } + + fn take(&mut self) -> BTreeSet { + let mut opened = BTreeSet::new(); + let mut bytes = [0_u8; 8192]; + loop { + match self.file.read(&mut bytes) { + Ok(0) => break, + Ok(len) => { + let mut offset = 0; + while offset < len { + // SAFETY: inotify returns complete events; byte arrays + // need not have the alignment of inotify_event. + let event = unsafe { + bytes.as_ptr().add(offset).cast::().read_unaligned() + }; + if event.mask & libc::IN_OPEN != 0 { + opened.insert(self.paths[&event.wd].clone()); + } + offset += std::mem::size_of::() + event.len as usize; + } + } + Err(error) if error.kind() == std::io::ErrorKind::WouldBlock => break, + Err(error) => panic!("{error}"), + } + } + opened + } +} + +fn historical(root: &Path, number: u64) -> PathBuf { + let log = root.join(format!("acme/widgets/commands/issue/{number}-trace.logs.jsonl")); + fs::create_dir_all(log.parent().unwrap()).unwrap(); + fs::write(&log, "{\"resourceLogs\":[]}\n").unwrap(); + let mut past = record(number); + past["process_id"] = 0.into(); + past["outcome"] = "merged".into(); + past["ended_at"] = "2026-10-19T11:01:00-04:00".into(); + past["stages"] = json!([{ + "name":"implement", "began_at":"2026-10-19T11:00:00-04:00", "session_log":log + }]); + write(root, &format!("{number}-fixture.command.json"), &past); + log +} + +#[test] +fn command_details_only_read_selected_historical_session_traces() { + for count in [1, 2, 562] { + let dir = tempfile::tempdir().unwrap(); + let traces: Vec<_> = (1..=count).map(|n| historical(dir.path(), n)).collect(); + let mut opens = Opens::new(&traces); + let mut config = crate::config::UserConfig::parse("", &dir.path().join("config.toml"), dir.path()).unwrap(); + config.logs_dir = dir.path().into(); + let inspection = crate::logs::stop::inspect(&config, "widgets#1").unwrap(); + assert_eq!(inspection.details["run_metrics"]["session_signals"]["implement"]["observed_sessions"], 1); + assert_eq!(opens.take(), [traces[0].clone()].into(), "Command details read unrelated historical Session traces with {count} records"); + } +} diff --git a/src/logs/reading/tests.rs b/src/logs/reading/tests.rs index 0390e5d92..9b3d4b7ad 100644 --- a/src/logs/reading/tests.rs +++ b/src/logs/reading/tests.rs @@ -14,6 +14,10 @@ mod inventory_tests; #[path = "reader_tests.rs"] mod reader_tests; +#[cfg(target_os = "linux")] +#[path = "metrics_tests.rs"] +mod metrics_tests; + fn now() -> DateTime { DateTime::parse_from_rfc3339("2026-10-19T12:00:00-04:00") .unwrap() From f4e82745992e41779ff91fa69285aa7fea1434a7 Mon Sep 17 00:00:00 2001 From: Jacob Stephens Date: Mon, 26 Oct 2026 09:00:26 -0400 Subject: [PATCH 2/9] Limit Command details to selected Session signals and cache historical reads --- console/tests/inbox.test.cjs | 26 +++++ site/architecture/index.html | 1 + site/how-it-works/index.html | 1 + src/console/assets/console.js | 10 +- src/logs/reading.rs | 52 +++++++-- src/logs/reading/cache.rs | 57 +++++++--- src/logs/reading/metrics_tests.rs | 177 +++++++++++++++++++++++++++++- src/logs/status/inspect.rs | 3 +- src/logs/stop.rs | 3 +- 9 files changed, 299 insertions(+), 31 deletions(-) diff --git a/console/tests/inbox.test.cjs b/console/tests/inbox.test.cjs index 9e9c3b3cf..fdb16095e 100644 --- a/console/tests/inbox.test.cjs +++ b/console/tests/inbox.test.cjs @@ -66,6 +66,32 @@ async function pageFor(t, options) { return page; } +test('Command details times out a held request and can be opened again', async t => { + const page = await pageFor(t); + await page.clock.install(); + let release; + const held = new Promise(resolve => { release = resolve; }); + await page.route('**/peek?*', async route => { + await held; + await route.continue(); + }); + try { + await page.goto(`${origin}/?page=security`); + const requested = page.waitForRequest('**/peek?*'); + await page.locator('.peek-link').click(); + await requested; + await page.clock.fastForward(15000); + await page.getByText('Details unavailable. Reload to try again.', { exact: true }) + .waitFor({ timeout: 2000 }); + } finally { + release(); + await page.unrouteAll({ behavior: 'wait' }); + } + await page.locator('[data-close-peek]').click(); + await page.locator('.peek-link').click(); + await page.locator('#peek dt').first().waitFor(); +}); + test('Inbox row Dismiss works by keyboard without opening Command details', async t => { const page = await pageFor(t); const observer = await pageFor(t); diff --git a/site/architecture/index.html b/site/architecture/index.html index cc070589e..8fced47fb 100644 --- a/site/architecture/index.html +++ b/site/architecture/index.html @@ -219,6 +219,7 @@

Architecture

thirdshift experiment --release <commit> compares the content commit with the last release tag on touched suites. The release script waits for rechecks and asks permission for a fail or inconclusive Verdict before pushing. Reruns reuse the Experiment, including before opening a pushed branch's PR. It refuses different main content before pushing or merging and checks the merged tree before tagging, so the release cannot contain unmeasured changes. No touched suite or a last release without the Trial flag carries on with a note. Releasing.

Each Ticket run's own process saves its Run metrics at completion in its Command record, including Ticket runs and Base fixes: outcome, Repairs, CI rounds, Unaddressed findings by Standards, Spec and Security, stage times against their Usual ranges, and minutes by stage and in total. Unaddressed findings counts retain Delivery's final PR body and the Security review's count, so later edits do not change them. Each Session stage keeps the review's Standards and Spec finding counts from its summary, with missing counts left missing. thirdshift show <command> --json includes these retained facts. The Console reads the same details, showing each Ticket run's available metrics and marking unavailable signals as Missing. thirdshift metrics <command> [--json] reads a Ticket run with its children; without a Command it groups weekly trends by repository, Harness, requested Model and Effort, Ticket run kind and session kind. Week and repository filters narrow that view. Historical metrics are computed from Command records without writing back, and missing signals stay missing. No combined score is produced.

Run metrics and Experiments share one Session usage module for tokens by observed Model and Harness-reported cost. Its normalized-event and retained-span readers resolve call amendments and partial totals, retaining fields a total omits. Input includes cache reads and writes. Reported Session cost wins; otherwise each Model's reported cost replaces its call costs. The metrics reader separately counts tool errors, permission denials, killed work and repeated tool calls within that session. Completion metrics sum these by session kind, and weekly Session trends split by the Harness version from each trace's resource. Saved completion metrics stay fixed. Historical metrics also read converted Session traces, leaving missing traces and unreported measurements missing.

+

Command details reconstruct Session signals only for the selected Command and its nested Ticket runs and dispatches; all records still contribute to Usual ranges and health. Weekly metrics read signals for every Command. A bounded process-local cache retains derived signals by the resolved trace's path, length, modification time and file identity, checking the identity again after parsing. Appends and replacements invalidate it; unchanged traces can be reused across Console requests. Restarting the process clears the cache. Historical Command records are never rewritten.

Confirmed Bug origins are joined when metrics are read, from local Scoring checkpoints under ~/.thirdshift/evals/scoring/. The latest window's decision for each bug wins. Its retained local Ticket run identity selects the exact Ticket run; cross-host pull-request attribution selects the latest local Ticket run for that URL, as in Scoring, so Continuations never multiply a count. The Scoring data covers both hosts; metrics read no remote records. Late confirmations count in the Ticket run's start week, corrections move the count to the settled pull request, and rejected or unsettled origins count for nothing. Command details in show --json and the Console share this projection and reconstruct available historical metrics. Command records are never rewritten.

The same reader checks each session's shell commands against one code-owned destructive-command pattern list. Force pushes, shared-branch history rewrites and recursive deletes outside the Worktree are retained with their commands in Run metrics, with flag counts in weekly Session trends. This check is advisory: it never blocks or fails a Ticket run.

    diff --git a/site/how-it-works/index.html b/site/how-it-works/index.html index 8d969b319..7655bfc3e 100644 --- a/site/how-it-works/index.html +++ b/site/how-it-works/index.html @@ -160,6 +160,7 @@

    How thirdshift works

    Run metrics and Experiments share Session usage rules. Partial totals replace only reported fields, preserving known input, cache counts and cost. Input includes cache reads and writes. A reported Session cost wins; otherwise each Model's reported cost replaces its call costs. Missing measurements stay missing, and saved completion metrics stay fixed.

    CLI log following and the Console keep reading through the captured writer's exit, including final output written after a Command's outcome is recorded. An exact Session trace finishes when its writer exits or its owning record moves to a later Session.

    The Console shows this machine's Commands, opening on its Inbox. Beside the CLI, you can watch their stages and logs there. Open Command details for a Ticket run's metrics, including available historical signals and late Bug-origin confirmations; unavailable signals say Missing. Stages keeps each Spec run with its Ticket runs, newest-started groups first, filling the window's width beside any open Command details: scroll right for older runs, or switch from All runs to By repository for one grid per repository. When a later Command takes up an issue, the earlier row stays in Resolved for inbox.days from that Command's start, saying why it left.

    +

    Command details and thirdshift show <command> --json read historical Session signals only for that Command and its nested work. The Console reuses derived signals while each trace is unchanged, reading it again after an append or replacement. If a details request has not finished within 10 seconds, the pane says “Details unavailable. Reload to try again.”

    The Console reads ready rows' pull requests and Needs you rows' issues and pull requests from GitHub in the background once a minute, even with no pages open. A changed answer wakes open Inbox pages; an unchanged answer leaves them idle. thirdshift status uses the same kept answer and reads GitHub itself only when it is missing or five minutes old. An ended Needs you row leaves once its issue closes or its pull request merges; a Triage row also leaves when needs-triage is removed. Architect and Security runs use the issue and pull request of the Spec run or Ticket run they dispatched. A pull request closed unmerged leaves a failure waiting while its issue is open. Live Silent and Slow rows wait until their Commands end. A merged or closed pull request takes its row out of Ready for review. Each exit says why, with Resolved counted from when GitHub shows it settled. Rows GitHub can speak for keep waiting however old they are. Reopening brings back a row still in Done or Resolved; History keeps the last facts and is not read again. Dismissals hold through GitHub changes; Undo uses GitHub's latest kept facts. Rows with no issue or pull request, no Command record, no read yet or nothing GitHub can find still age out, straight to History.

    When GitHub cannot answer, each row keeps the group its last facts give it; a row never read still goes by age. After five minutes without a good read, the Console and status warn with the last-answer time and latest failure. A successful read clears that warning. With fewer than 500 GraphQL points or 500 REST requests left, both skip reads until that quota resets and say why, leaving the rest to the factory. status --json gives the last-answer time and latest failure. These limits are fixed, with no User config settings.

    status --json also gives each row's kept GitHub facts and when they were read. History keeps those observation times with its facts. A dismissed row stays in Resolved for inbox.days from its dismissal; a later GitHub settlement does not extend it.

    diff --git a/src/console/assets/console.js b/src/console/assets/console.js index 43b540f79..8c96de2d3 100644 --- a/src/console/assets/console.js +++ b/src/console/assets/console.js @@ -267,6 +267,11 @@ async function loadPeek(url) { pendingPeek?.abort(); const request = new AbortController(); pendingPeek = request; + let timedOut = false; + const timeout = setTimeout(() => { + timedOut = true; + request.abort(); + }, 10000); try { const response = await fetch(url, { signal: request.signal, cache: 'no-store' }); if (!response.ok) throw new Error('Details unavailable'); @@ -280,8 +285,11 @@ async function loadPeek(url) { if (content.querySelector('[data-live="true"]')) startUpdates(); else stopPeekUpdates(); } catch (error) { - if (!request.signal.aborted && error.name !== 'AbortError') content.textContent = 'Details unavailable. Reload to try again.'; + if (pendingPeek === request && (timedOut || (!request.signal.aborted && error.name !== 'AbortError'))) { + content.textContent = 'Details unavailable. Reload to try again.'; + } } finally { + clearTimeout(timeout); if (pendingPeek === request) pendingPeek = null; } } diff --git a/src/logs/reading.rs b/src/logs/reading.rs index c11be8992..4ab40fc22 100644 --- a/src/logs/reading.rs +++ b/src/logs/reading.rs @@ -62,15 +62,39 @@ impl Snapshot { } pub(super) fn load_metrics(root: &Path, now: DateTime) -> Result { - Self::load_metrics_with_silence(root, now, std::time::Duration::from_secs(600)) + let mut snapshot = Self::capture_records( + Arc::new(inventory::Native), + root, + now, + std::time::Duration::from_secs(600), + true, + )?; + for entry in &mut snapshot.commands { + if let Entry::Record(command) = entry { + command.add_session_signals(None); + } + } + Ok(snapshot) } - pub(super) fn load_metrics_with_silence( + pub(super) fn load_details( root: &Path, now: DateTime, silent: std::time::Duration, + input: &str, ) -> Result { - Self::capture_records(Arc::new(inventory::Native), root, now, silent, true) + let mut snapshot = + Self::capture_records(Arc::new(inventory::Native), root, now, silent, true)?; + if let Observation::Record(selected) = snapshot.resolve(input)?.observation { + let path = selected.record_path.clone(); + let name = selected.record.name.clone(); + for entry in &mut snapshot.commands { + if let Entry::Record(command) = entry { + command.add_session_signals(Some((&path, &name))); + } + } + } + Ok(snapshot) } fn capture( @@ -257,6 +281,22 @@ pub(super) struct CurrentStage { } impl Command { + /// Enrich only the selected Command tree; all records still contribute to + /// Usual ranges and health. None selects every tree for weekly metrics. + fn add_session_signals(&mut self, selected: Option<(&Path, &str)>) { + let selected = + selected.filter(|(path, name)| self.record_path != *path || self.record.name != *name); + if selected.is_none() + && let Some(mut metrics) = self.record.run_metrics.take() + { + metrics.add_session_signals(&self.record, cache::session_signals); + self.record.run_metrics = Some(metrics); + } + for child in self.dispatched.iter_mut().chain(&mut self.children) { + child.add_session_signals(selected); + } + } + fn new( inventory: &Arc, record: CommandRecord, @@ -328,12 +368,6 @@ fn read_records( if record.run_metrics.is_none() && record.outcome.is_some() { record.run_metrics = Some(super::run_metrics::RunMetrics::historical(record)); } - if let Some(mut metrics) = record.run_metrics.take() { - metrics.add_session_signals(record, |log| { - super::session_signals::read(log).ok().flatten() - }); - record.run_metrics = Some(metrics); - } } } let record = &mut tree.command; diff --git a/src/logs/reading/cache.rs b/src/logs/reading/cache.rs index 2e6d9868b..009281a1a 100644 --- a/src/logs/reading/cache.rs +++ b/src/logs/reading/cache.rs @@ -42,6 +42,7 @@ struct Retained { identity: Identity, value: T, used: Instant, + bytes: u64, } struct Cache { @@ -57,7 +58,12 @@ impl Cache { } } - fn read(&mut self, path: &Path, load: impl FnOnce(&Path) -> Result) -> Result { + fn read( + &mut self, + path: &Path, + load: impl FnOnce(&Path) -> Result, + retained_bytes: impl FnOnce(&T, u64) -> u64, + ) -> Result { let identity = Identity::read(path)?; if let Some(retained) = self.entries.get_mut(path) && retained.identity == identity @@ -66,28 +72,30 @@ impl Cache { return Ok(retained.value.clone()); } if let Some(old) = self.entries.remove(path) { - self.bytes -= old.identity.len; + self.bytes -= old.bytes; } let value = load(path)?; + let bytes = retained_bytes(&value, identity.len); // Don't retain a partial in-place write. A later request retries it. const BUDGET: u64 = 64 * 1024 * 1024; - if identity.len <= BUDGET && Identity::read(path)? == identity { - while self.bytes + identity.len > BUDGET || self.entries.len() >= 8192 { + if bytes <= BUDGET && Identity::read(path)? == identity { + while self.bytes + bytes > BUDGET || self.entries.len() >= 8192 { let oldest = self .entries .iter() .min_by_key(|(_, entry)| entry.used) .map(|(path, _)| path.clone()) .expect("nonempty cache over budget"); - self.bytes -= self.entries.remove(&oldest).unwrap().identity.len; + self.bytes -= self.entries.remove(&oldest).unwrap().bytes; } - self.bytes += identity.len; + self.bytes += bytes; self.entries.insert( path.to_path_buf(), Retained { identity, value: value.clone(), used: Instant::now(), + bytes, }, ); } @@ -101,19 +109,21 @@ pub(super) fn record(path: &Path) -> Result> { RECORDS .lock() .unwrap() - .read(path, |path| { - Ok(serde_json::from_slice(&fs::read(path)?).ok()) - }) + .read( + path, + |path| Ok(serde_json::from_slice(&fs::read(path)?).ok()), + |_, source_bytes| source_bytes, + ) .with_context(|| format!("could not read {}", path.display())) } pub(super) fn text(path: &Path) -> Result> { static TEXT: LazyLock>>> = LazyLock::new(|| Mutex::new(Cache::new())); - match TEXT - .lock() - .unwrap() - .read(path, |path| Ok(fs::read_to_string(path)?.into())) - { + match TEXT.lock().unwrap().read( + path, + |path| Ok(fs::read_to_string(path)?.into()), + |_, source_bytes| source_bytes, + ) { Ok(text) => Ok(text), Err(error) if error @@ -125,3 +135,22 @@ pub(super) fn text(path: &Path) -> Result> { Err(error) => Err(error).with_context(|| format!("could not read {}", path.display())), } } + +/// Retain only the derived signals, keyed by the resolved Session trace's +/// identity. Large source traces must not consume the summary cache's budget. +pub(super) fn session_signals(log: &Path) -> Option { + use crate::logs::session_signals::{SessionSignals, read}; + + static SIGNALS: LazyLock>>> = + LazyLock::new(|| Mutex::new(Cache::new())); + SIGNALS + .lock() + .unwrap() + .read(&crate::logs::session::path(log), read, |signals, _| { + serde_json::to_vec(signals) + .expect("Session signals serialize") + .len() as u64 + }) + .ok() + .flatten() +} diff --git a/src/logs/reading/metrics_tests.rs b/src/logs/reading/metrics_tests.rs index b353b2aea..76f0c7a03 100644 --- a/src/logs/reading/metrics_tests.rs +++ b/src/logs/reading/metrics_tests.rs @@ -42,7 +42,11 @@ impl Opens { // SAFETY: inotify returns complete events; byte arrays // need not have the alignment of inotify_event. let event = unsafe { - bytes.as_ptr().add(offset).cast::().read_unaligned() + bytes + .as_ptr() + .add(offset) + .cast::() + .read_unaligned() }; if event.mask & libc::IN_OPEN != 0 { opened.insert(self.paths[&event.wd].clone()); @@ -59,7 +63,9 @@ impl Opens { } fn historical(root: &Path, number: u64) -> PathBuf { - let log = root.join(format!("acme/widgets/commands/issue/{number}-trace.logs.jsonl")); + let log = root.join(format!( + "acme/widgets/commands/issue/{number}-trace.logs.jsonl" + )); fs::create_dir_all(log.parent().unwrap()).unwrap(); fs::write(&log, "{\"resourceLogs\":[]}\n").unwrap(); let mut past = record(number); @@ -79,10 +85,171 @@ fn command_details_only_read_selected_historical_session_traces() { let dir = tempfile::tempdir().unwrap(); let traces: Vec<_> = (1..=count).map(|n| historical(dir.path(), n)).collect(); let mut opens = Opens::new(&traces); - let mut config = crate::config::UserConfig::parse("", &dir.path().join("config.toml"), dir.path()).unwrap(); + let mut config = + crate::config::UserConfig::parse("", &dir.path().join("config.toml"), dir.path()) + .unwrap(); config.logs_dir = dir.path().into(); let inspection = crate::logs::stop::inspect(&config, "widgets#1").unwrap(); - assert_eq!(inspection.details["run_metrics"]["session_signals"]["implement"]["observed_sessions"], 1); - assert_eq!(opens.take(), [traces[0].clone()].into(), "Command details read unrelated historical Session traces with {count} records"); + assert_eq!( + inspection.details["run_metrics"]["session_signals"]["implement"]["observed_sessions"], + 1 + ); + assert_eq!( + opens.take(), + [traces[0].clone()].into(), + "Command details read unrelated historical Session traces with {count} records" + ); } } + +#[test] +fn metrics_reuse_unchanged_session_traces_and_refresh_changed_traces() { + use std::io::Write; + + let dir = tempfile::tempdir().unwrap(); + let traces: Vec<_> = (1..=4).map(|n| historical(dir.path(), n)).collect(); + // The source can exceed the entire cache budget; only its compact derived + // signals should count toward that budget. + let mut large = fs::File::create(&traces[3]).unwrap(); + let padding = vec![b' '; 1024 * 1024]; + for _ in 0..65 { + large.write_all(&padding).unwrap(); + } + large.write_all(b"{\"resourceLogs\":[]}\n").unwrap(); + drop(large); + let mut opens = Opens::new(&traces); + let first = Snapshot::load_metrics(dir.path(), now()).unwrap(); + assert_eq!(opens.take(), traces.iter().cloned().collect()); + let second = Snapshot::load_metrics(dir.path(), now()).unwrap(); + assert!( + opens.take().is_empty(), + "unchanged Session traces were reparsed" + ); + assert_eq!( + serde_json::to_value(first.resolve("widgets#1").unwrap().observation).unwrap(), + serde_json::to_value(second.resolve("widgets#1").unwrap().observation).unwrap() + ); + + let amendment = |version: &str| { + serde_json::to_string(&json!({ + "resourceLogs": [{"resource": {"attributes": [ + {"key": "thirdshift.harness.version", "value": {"stringValue": version}} + ]}, "scopeLogs": []}] + })) + .unwrap() + + "\n" + }; + fs::OpenOptions::new() + .append(true) + .open(&traces[0]) + .unwrap() + .write_all(amendment("1.0").as_bytes()) + .unwrap(); + opens.take(); + let appended = Snapshot::load_metrics(dir.path(), now()).unwrap(); + assert_eq!(opens.take(), [traces[0].clone()].into()); + let signals = |snapshot: &Snapshot| { + serde_json::to_value(snapshot.resolve("widgets#1").unwrap().observation).unwrap() + ["run_metrics"]["stages"][0]["session_signals"].clone() + }; + assert_eq!(signals(&appended)["harness_version"], "1.0"); + + // Same length and mtime, but a new inode must invalidate the old signals. + let modified = fs::metadata(&traces[0]).unwrap().modified().unwrap(); + let replacement = dir.path().join("replacement.logs.jsonl"); + fs::write( + &replacement, + "{\"resourceLogs\":[]}\n".to_owned() + &amendment("2.0"), + ) + .unwrap(); + fs::File::open(&replacement) + .unwrap() + .set_times(fs::FileTimes::new().set_modified(modified)) + .unwrap(); + fs::rename(replacement, &traces[0]).unwrap(); + let mut replaced_opens = Opens::new(&traces); + let replaced = Snapshot::load_metrics(dir.path(), now()).unwrap(); + assert_eq!(replaced_opens.take(), [traces[0].clone()].into()); + assert_eq!(signals(&replaced)["harness_version"], "2.0"); +} + +#[test] +fn command_details_keep_dispatch_and_child_signals_and_other_commands_health() { + for (selected, expected) in [("widgets#1", vec![1, 2, 3]), ("widgets#2", vec![2, 3])] { + let dir = tempfile::tempdir().unwrap(); + let traces: Vec<_> = (1..=5).map(|n| historical(dir.path(), n)).collect(); + let path = |n| { + dir.path().join(format!( + "acme/widgets/commands/issue/{n}-fixture.command.json" + )) + }; + let read = |n| serde_json::from_slice::(&fs::read(path(n)).unwrap()).unwrap(); + let mut outer = read(1); + outer["dispatched"] = json!([read(2)]); + fs::write(path(1), serde_json::to_vec(&outer).unwrap()).unwrap(); + fs::remove_file(path(2)).unwrap(); + let mut child = read(3); + child["parent_record"] = json!(path(1)); + child["parent_dispatch"] = "https://github.com/acme/widgets/issues/2".into(); + fs::write(path(3), serde_json::to_vec(&child).unwrap()).unwrap(); + let mut live = record(5); + live["stages"] = json!([{"name": "implement", "began_at": "2026-10-19T11:00:00-04:00"}]); + fs::write(path(5), serde_json::to_vec(&live).unwrap()).unwrap(); + + let before = Snapshot::load(dir.path(), now()).unwrap(); + let mut opens = Opens::new(&traces); + let details = Snapshot::load_details( + dir.path(), + now(), + std::time::Duration::from_secs(600), + selected, + ) + .unwrap(); + assert_eq!( + opens.take(), + expected.iter().map(|n| traces[n - 1].clone()).collect() + ); + for number in [1, 2, 3] { + let value = serde_json::to_value( + details + .resolve(&format!("widgets#{number}")) + .unwrap() + .observation, + ) + .unwrap(); + if expected.contains(&number) { + assert_eq!( + value["run_metrics"]["session_signals"]["implement"]["observed_sessions"], + 1 + ); + } + } + let observation = |snapshot: &Snapshot| { + serde_json::to_value(snapshot.resolve("widgets#5").unwrap().observation).unwrap() + }; + assert_eq!( + observation(&before)["health"], + observation(&details)["health"] + ); + assert_eq!( + observation(&before)["usual_ranges"], + observation(&details)["usual_ranges"] + ); + assert!(!observation(&before)["usual_ranges"]["implement"].is_null()); + } +} + +#[test] +fn saved_session_signals_do_not_read_the_historical_trace() { + let dir = tempfile::tempdir().unwrap(); + let trace = historical(dir.path(), 1); + let snapshot = Snapshot::load_metrics(dir.path(), now()).unwrap(); + let value = serde_json::to_value(snapshot.resolve("widgets#1").unwrap().observation).unwrap(); + write(dir.path(), "1-fixture.command.json", &value); + fs::write(&trace, "invalid trace, saved signals must win\n").unwrap(); + let mut opens = Opens::new(&[trace]); + let snapshot = Snapshot::load_metrics(dir.path(), now()).unwrap(); + assert!(opens.take().is_empty()); + let saved = serde_json::to_value(snapshot.resolve("widgets#1").unwrap().observation).unwrap(); + assert_eq!(saved["run_metrics"], value["run_metrics"]); +} diff --git a/src/logs/status/inspect.rs b/src/logs/status/inspect.rs index 391afd81a..7cf646c19 100644 --- a/src/logs/status/inspect.rs +++ b/src/logs/status/inspect.rs @@ -96,10 +96,11 @@ fn print_details(command: &Command, indent: &str) -> Result<()> { pub fn show(input: &str, json: bool) -> Result<()> { let config = UserConfig::load()?; - let snapshot = Snapshot::load_metrics_with_silence( + let snapshot = Snapshot::load_details( &config.logs_dir, Local::now(), config.silent_interval, + input, )?; let command = snapshot.resolve(input)?.observation; if json { diff --git a/src/logs/stop.rs b/src/logs/stop.rs index 532c0d30b..629c79166 100644 --- a/src/logs/stop.rs +++ b/src/logs/stop.rs @@ -118,10 +118,11 @@ pub(crate) struct Inspection { /// Availability follows the CLI's child refusal and captured process identity. pub(crate) fn inspect(config: &UserConfig, input: &str) -> Result { - let snapshot = Snapshot::load_metrics_with_silence( + let snapshot = Snapshot::load_details( &config.logs_dir, chrono::Local::now(), config.silent_interval, + input, )?; let selected = snapshot.resolve(input)?; let pending = selected From 02d65d521bd8590eab518dd1cc4fc41476af0812 Mon Sep 17 00:00:00 2001 From: Jacob Stephens Date: Mon, 26 Oct 2026 09:09:53 -0400 Subject: [PATCH 3/9] Keep cold Session trace parsing from blocking unrelated cached details --- console/README.md | 4 + .../reproductions/peek-cache-contention.py | 99 +++++++++++++ src/logs/reading/cache.rs | 138 ++++++++++++------ 3 files changed, 200 insertions(+), 41 deletions(-) create mode 100644 console/reproductions/peek-cache-contention.py diff --git a/console/README.md b/console/README.md index e95442554..050bb98fa 100644 --- a/console/README.md +++ b/console/README.md @@ -286,6 +286,10 @@ That Peek's Copy CLI copies its `thirdshift show '' --json` line; if the browser refuses the clipboard, the line is selected to copy by keyboard, and stays selected through live refresh. +After a build on Linux, `python3 console/reproductions/peek-cache-contention.py` +checks that parsing another Command's cold Session trace does not delay cached +Command details. + Zoom fits a parent span or a padded window around a call. The sticky axis keeps the zoom notice and Zoom out control visible; the slowest calls re-rank within the window. Folds, selection, zoom and Peek disclosures survive live refresh. diff --git a/console/reproductions/peek-cache-contention.py b/console/reproductions/peek-cache-contention.py new file mode 100644 index 000000000..a996328f4 --- /dev/null +++ b/console/reproductions/peek-cache-contention.py @@ -0,0 +1,99 @@ +#!/usr/bin/env python3 +"""After cargo build, check cached details during another Command's cold parse. + +Linux only: IN_OPEN pins the overlapping requests to actual Session trace IO. +The 128 MiB OTLP fixture makes lock contention visible even on a fast machine. +""" + +import concurrent.futures +import ctypes +import json +import os +import pathlib +import select +import subprocess +import tempfile +import time +import urllib.request + + +def check(): + binary = str(pathlib.Path(__file__).resolve().parents[2] / 'target/debug/thirdshift') + with tempfile.TemporaryDirectory(prefix='thirdshift-peek-contention-') as directory: + task_home = pathlib.Path(directory) + config = task_home / '.thirdshift' + records = config / 'logs/acme/widgets/commands/issue' + records.mkdir(parents=True) + (config / 'config.toml').write_text('[console]\nport = 0\n') + for number in [1, 2]: + trace = records / f'{number}-fixture.logs.jsonl' + if number == 1: + trace.write_text('{"resourceLogs":[]}\n') + else: + line = json.dumps({'resourceLogs': [{'scopeLogs': [{'logRecords': [ + {'body': {'stringValue': 'x' * (8 * 1024 * 1024)}} + ]}]}]}) + '\n' + with trace.open('w') as output: + for _ in range(16): + output.write(line) + record = dict( + version=1, kind='Run', repository='acme/widgets', + name=f'acme/widgets#{number}', + issue=dict(number=number, url=f'https://github.com/acme/widgets/issues/{number}', title='Widgets'), + hostname='factory', process_id=0, process_start='gone', harness='claude', + started_at='2026-10-19T11:00:00-04:00', ended_at='2026-10-19T11:01:00-04:00', + stages=[dict(name='implement', began_at='2026-10-19T11:00:00-04:00', session_log=str(trace))], + outcome='merged', + ) + (records / f'{number}-fixture.command.json').write_text(json.dumps(record)) + env = dict(os.environ, HOME=directory) + server = subprocess.Popen( + [binary, 'console', '--no-sign-in'], env=env, + stdout=subprocess.PIPE, text=True, + ) + try: + assert select.select([server.stdout], [], [], 10)[0], 'Console startup timed out' + origin = server.stdout.readline().strip() + assert origin.startswith('http://127.0.0.1:'), origin + + def peek(number): + began = time.monotonic() + with urllib.request.urlopen( + origin + '/peek?command=widgets%23' + str(number), timeout=45, + ) as response: + assert response.status == 200 + response.read() + return time.monotonic() - began + + print('warm selected details seconds', peek(1), flush=True) + libc = ctypes.CDLL(None, use_errno=True) + descriptor = libc.inotify_init1(os.O_NONBLOCK | os.O_CLOEXEC) + assert descriptor >= 0 + try: + watched = libc.inotify_add_watch( + descriptor, os.fsencode(records / '2-fixture.logs.jsonl'), 0x20, # IN_OPEN + ) + assert watched >= 0 + with concurrent.futures.ThreadPoolExecutor() as executor: + cold = executor.submit(peek, 2) + assert select.select([descriptor], [], [], 15)[0], 'cold reader did not open trace' + os.read(descriptor, 8192) + warm_seconds = peek(1) + cold_seconds = cold.result(timeout=45) + print('cold details seconds', cold_seconds, flush=True) + print('cached unrelated details seconds', warm_seconds, flush=True) + assert warm_seconds < 1, f'cached details took {warm_seconds:.3f}s during an unrelated parse' + finally: + os.close(descriptor) + finally: + server.terminate() + try: + server.wait(timeout=10) + except subprocess.TimeoutExpired: + server.kill() + server.wait() + server.stdout.close() + + +if __name__ == '__main__': + check() diff --git a/src/logs/reading/cache.rs b/src/logs/reading/cache.rs index 009281a1a..ade8577ba 100644 --- a/src/logs/reading/cache.rs +++ b/src/logs/reading/cache.rs @@ -3,12 +3,13 @@ use std::collections::BTreeMap; use std::fs; use std::path::{Path, PathBuf}; -use std::sync::{Arc, LazyLock, Mutex}; +use std::sync::{Arc, LazyLock, Mutex, Weak}; use std::time::{Instant, SystemTime}; use anyhow::{Context, Result}; use crate::logs::command_record::CommandTree; +use crate::logs::session_signals::SessionSignals; #[derive(PartialEq)] struct Identity { @@ -65,41 +66,58 @@ impl Cache { retained_bytes: impl FnOnce(&T, u64) -> u64, ) -> Result { let identity = Identity::read(path)?; + if let Some(value) = self.get(path, &identity) { + return Ok(value); + } + let value = load(path)?; + let bytes = retained_bytes(&value, identity.len); + // Don't retain a partial in-place write. A later request retries it. + if Identity::read(path)? == identity { + self.insert(path, identity, value.clone(), bytes); + } + Ok(value) + } + + fn get(&mut self, path: &Path, identity: &Identity) -> Option { if let Some(retained) = self.entries.get_mut(path) - && retained.identity == identity + && retained.identity == *identity { retained.used = Instant::now(); - return Ok(retained.value.clone()); + return Some(retained.value.clone()); } if let Some(old) = self.entries.remove(path) { self.bytes -= old.bytes; } - let value = load(path)?; - let bytes = retained_bytes(&value, identity.len); - // Don't retain a partial in-place write. A later request retries it. + None + } + + fn insert(&mut self, path: &Path, identity: Identity, value: T, bytes: u64) { const BUDGET: u64 = 64 * 1024 * 1024; - if bytes <= BUDGET && Identity::read(path)? == identity { - while self.bytes + bytes > BUDGET || self.entries.len() >= 8192 { - let oldest = self - .entries - .iter() - .min_by_key(|(_, entry)| entry.used) - .map(|(path, _)| path.clone()) - .expect("nonempty cache over budget"); - self.bytes -= self.entries.remove(&oldest).unwrap().bytes; - } - self.bytes += bytes; - self.entries.insert( - path.to_path_buf(), - Retained { - identity, - value: value.clone(), - used: Instant::now(), - bytes, - }, - ); + if bytes > BUDGET { + return; } - Ok(value) + if let Some(old) = self.entries.remove(path) { + self.bytes -= old.bytes; + } + while self.bytes + bytes > BUDGET || self.entries.len() >= 8192 { + let oldest = self + .entries + .iter() + .min_by_key(|(_, entry)| entry.used) + .map(|(path, _)| path.clone()) + .expect("nonempty cache over budget"); + self.bytes -= self.entries.remove(&oldest).unwrap().bytes; + } + self.bytes += bytes; + self.entries.insert( + path.to_path_buf(), + Retained { + identity, + value, + used: Instant::now(), + bytes, + }, + ); } } @@ -138,19 +156,57 @@ pub(super) fn text(path: &Path) -> Result> { /// Retain only the derived signals, keyed by the resolved Session trace's /// identity. Large source traces must not consume the summary cache's budget. -pub(super) fn session_signals(log: &Path) -> Option { - use crate::logs::session_signals::{SessionSignals, read}; - - static SIGNALS: LazyLock>>> = - LazyLock::new(|| Mutex::new(Cache::new())); - SIGNALS - .lock() - .unwrap() - .read(&crate::logs::session::path(log), read, |signals, _| { - serde_json::to_vec(signals) - .expect("Session signals serialize") - .len() as u64 +pub(super) fn session_signals(log: &Path) -> Option { + struct Signals { + cache: Cache>, + readers: BTreeMap>>, + } + static SIGNALS: LazyLock> = LazyLock::new(|| { + Mutex::new(Signals { + cache: Cache::new(), + readers: BTreeMap::new(), }) - .ok() - .flatten() + }); + let path = crate::logs::session::path(log); + let identity = Identity::read(&path).ok()?; + let reader = { + let mut signals = SIGNALS.lock().unwrap(); + if let Some(value) = signals.cache.get(&path, &identity) { + return value; + } + // Coordinate readers of this trace only. Parsing a cold trace must not + // block cached details for unrelated Commands. Weak entries disappear + // on the next miss once their readers have finished. + signals + .readers + .retain(|_, reader| reader.strong_count() > 0); + match signals.readers.get(&path).and_then(Weak::upgrade) { + Some(reader) => reader, + None => { + let reader = Arc::new(Mutex::new(())); + signals + .readers + .insert(path.clone(), Arc::downgrade(&reader)); + reader + } + } + }; + let _reading = reader.lock().unwrap(); + // A preceding reader or writer may have changed the facts while we waited. + let identity = Identity::read(&path).ok()?; + if let Some(value) = SIGNALS.lock().unwrap().cache.get(&path, &identity) { + return value; + } + let value = crate::logs::session_signals::read(&path).ok()?; + if Identity::read(&path).ok().as_ref() == Some(&identity) { + let bytes = serde_json::to_vec(&value) + .expect("Session signals serialize") + .len() as u64; + SIGNALS + .lock() + .unwrap() + .cache + .insert(&path, identity, value.clone(), bytes); + } + value } From c6a07a820fdb9aa1298f49655a4d7341830de487 Mon Sep 17 00:00:00 2001 From: Jacob Stephens Date: Mon, 26 Oct 2026 09:14:37 -0400 Subject: [PATCH 4/9] thirdshift: failed ticket run (codex exited 1: Selected model is at capacity. Please try a different model.) 2026-10-26T13:14:37Z, host thirdshift. Uncommitted work at the time of failure is included in this commit. From 78dfb4bcb0cd1d9383a88c2fa49cc9f6d208a9ea Mon Sep 17 00:00:00 2001 From: Jacob Stephens Date: Mon, 26 Oct 2026 10:39:40 -0400 Subject: [PATCH 5/9] Name the selected Command when enriching Session signals Command details still read only that Command and its nested work. The record cache no longer takes a byte-size closure, and the site names Session traces. --- site/architecture/index.html | 2 +- site/how-it-works/index.html | 2 +- src/logs/reading.rs | 17 +++++++++-------- src/logs/reading/cache.rs | 32 ++++++++++++++------------------ 4 files changed, 25 insertions(+), 28 deletions(-) diff --git a/site/architecture/index.html b/site/architecture/index.html index 8fced47fb..947fbd240 100644 --- a/site/architecture/index.html +++ b/site/architecture/index.html @@ -219,7 +219,7 @@

    Architecture

    thirdshift experiment --release <commit> compares the content commit with the last release tag on touched suites. The release script waits for rechecks and asks permission for a fail or inconclusive Verdict before pushing. Reruns reuse the Experiment, including before opening a pushed branch's PR. It refuses different main content before pushing or merging and checks the merged tree before tagging, so the release cannot contain unmeasured changes. No touched suite or a last release without the Trial flag carries on with a note. Releasing.

    Each Ticket run's own process saves its Run metrics at completion in its Command record, including Ticket runs and Base fixes: outcome, Repairs, CI rounds, Unaddressed findings by Standards, Spec and Security, stage times against their Usual ranges, and minutes by stage and in total. Unaddressed findings counts retain Delivery's final PR body and the Security review's count, so later edits do not change them. Each Session stage keeps the review's Standards and Spec finding counts from its summary, with missing counts left missing. thirdshift show <command> --json includes these retained facts. The Console reads the same details, showing each Ticket run's available metrics and marking unavailable signals as Missing. thirdshift metrics <command> [--json] reads a Ticket run with its children; without a Command it groups weekly trends by repository, Harness, requested Model and Effort, Ticket run kind and session kind. Week and repository filters narrow that view. Historical metrics are computed from Command records without writing back, and missing signals stay missing. No combined score is produced.

    Run metrics and Experiments share one Session usage module for tokens by observed Model and Harness-reported cost. Its normalized-event and retained-span readers resolve call amendments and partial totals, retaining fields a total omits. Input includes cache reads and writes. Reported Session cost wins; otherwise each Model's reported cost replaces its call costs. The metrics reader separately counts tool errors, permission denials, killed work and repeated tool calls within that session. Completion metrics sum these by session kind, and weekly Session trends split by the Harness version from each trace's resource. Saved completion metrics stay fixed. Historical metrics also read converted Session traces, leaving missing traces and unreported measurements missing.

    -

    Command details reconstruct Session signals only for the selected Command and its nested Ticket runs and dispatches; all records still contribute to Usual ranges and health. Weekly metrics read signals for every Command. A bounded process-local cache retains derived signals by the resolved trace's path, length, modification time and file identity, checking the identity again after parsing. Appends and replacements invalidate it; unchanged traces can be reused across Console requests. Restarting the process clears the cache. Historical Command records are never rewritten.

    +

    Command details reconstruct Session signals only for the selected Command and its nested Ticket runs and dispatches; all records still contribute to Usual ranges and health. Weekly metrics read signals for every Command. A bounded process-local cache retains derived signals by the resolved Session trace's path, length, modification time and file identity, checking the identity again after parsing. Appends and replacements invalidate it; unchanged Session traces can be reused across Console requests. Restarting the process clears the cache. Historical Command records are never rewritten.

    Confirmed Bug origins are joined when metrics are read, from local Scoring checkpoints under ~/.thirdshift/evals/scoring/. The latest window's decision for each bug wins. Its retained local Ticket run identity selects the exact Ticket run; cross-host pull-request attribution selects the latest local Ticket run for that URL, as in Scoring, so Continuations never multiply a count. The Scoring data covers both hosts; metrics read no remote records. Late confirmations count in the Ticket run's start week, corrections move the count to the settled pull request, and rejected or unsettled origins count for nothing. Command details in show --json and the Console share this projection and reconstruct available historical metrics. Command records are never rewritten.

    The same reader checks each session's shell commands against one code-owned destructive-command pattern list. Force pushes, shared-branch history rewrites and recursive deletes outside the Worktree are retained with their commands in Run metrics, with flag counts in weekly Session trends. This check is advisory: it never blocks or fails a Ticket run.

      diff --git a/site/how-it-works/index.html b/site/how-it-works/index.html index 7655bfc3e..6ff5882af 100644 --- a/site/how-it-works/index.html +++ b/site/how-it-works/index.html @@ -160,7 +160,7 @@

      How thirdshift works

      Run metrics and Experiments share Session usage rules. Partial totals replace only reported fields, preserving known input, cache counts and cost. Input includes cache reads and writes. A reported Session cost wins; otherwise each Model's reported cost replaces its call costs. Missing measurements stay missing, and saved completion metrics stay fixed.

      CLI log following and the Console keep reading through the captured writer's exit, including final output written after a Command's outcome is recorded. An exact Session trace finishes when its writer exits or its owning record moves to a later Session.

      The Console shows this machine's Commands, opening on its Inbox. Beside the CLI, you can watch their stages and logs there. Open Command details for a Ticket run's metrics, including available historical signals and late Bug-origin confirmations; unavailable signals say Missing. Stages keeps each Spec run with its Ticket runs, newest-started groups first, filling the window's width beside any open Command details: scroll right for older runs, or switch from All runs to By repository for one grid per repository. When a later Command takes up an issue, the earlier row stays in Resolved for inbox.days from that Command's start, saying why it left.

      -

      Command details and thirdshift show <command> --json read historical Session signals only for that Command and its nested work. The Console reuses derived signals while each trace is unchanged, reading it again after an append or replacement. If a details request has not finished within 10 seconds, the pane says “Details unavailable. Reload to try again.”

      +

      Command details and thirdshift show <command> --json read historical Session signals only for that Command and its nested work. The Console reuses derived signals while each Session trace is unchanged, reading it again after an append or replacement. If a details request has not finished within 10 seconds, the pane says “Details unavailable. Reload to try again.”

      The Console reads ready rows' pull requests and Needs you rows' issues and pull requests from GitHub in the background once a minute, even with no pages open. A changed answer wakes open Inbox pages; an unchanged answer leaves them idle. thirdshift status uses the same kept answer and reads GitHub itself only when it is missing or five minutes old. An ended Needs you row leaves once its issue closes or its pull request merges; a Triage row also leaves when needs-triage is removed. Architect and Security runs use the issue and pull request of the Spec run or Ticket run they dispatched. A pull request closed unmerged leaves a failure waiting while its issue is open. Live Silent and Slow rows wait until their Commands end. A merged or closed pull request takes its row out of Ready for review. Each exit says why, with Resolved counted from when GitHub shows it settled. Rows GitHub can speak for keep waiting however old they are. Reopening brings back a row still in Done or Resolved; History keeps the last facts and is not read again. Dismissals hold through GitHub changes; Undo uses GitHub's latest kept facts. Rows with no issue or pull request, no Command record, no read yet or nothing GitHub can find still age out, straight to History.

      When GitHub cannot answer, each row keeps the group its last facts give it; a row never read still goes by age. After five minutes without a good read, the Console and status warn with the last-answer time and latest failure. A successful read clears that warning. With fewer than 500 GraphQL points or 500 REST requests left, both skip reads until that quota resets and say why, leaving the rest to the factory. status --json gives the last-answer time and latest failure. These limits are fixed, with no User config settings.

      status --json also gives each row's kept GitHub facts and when they were read. History keeps those observation times with its facts. A dismissed row stays in Resolved for inbox.days from its dismissal; a later GitHub settlement does not extend it.

      diff --git a/src/logs/reading.rs b/src/logs/reading.rs index 4ab40fc22..097fec80e 100644 --- a/src/logs/reading.rs +++ b/src/logs/reading.rs @@ -281,19 +281,20 @@ pub(super) struct CurrentStage { } impl Command { - /// Enrich only the selected Command tree; all records still contribute to - /// Usual ranges and health. None selects every tree for weekly metrics. + /// Enrich this Command when `selected` is every Command (`None`), or this + /// Command's own record path and name. A match also enriches nested Ticket + /// runs and dispatches. Path alone is not enough: a dispatch shares its + /// parent's record path. Usual ranges and health already include every record. fn add_session_signals(&mut self, selected: Option<(&Path, &str)>) { - let selected = - selected.filter(|(path, name)| self.record_path != *path || self.record.name != *name); - if selected.is_none() - && let Some(mut metrics) = self.record.run_metrics.take() - { + let enrich = selected + .is_none_or(|(path, name)| self.record_path == path && self.record.name == name); + if enrich && let Some(mut metrics) = self.record.run_metrics.take() { metrics.add_session_signals(&self.record, cache::session_signals); self.record.run_metrics = Some(metrics); } + let nested = if enrich { None } else { selected }; for child in self.dispatched.iter_mut().chain(&mut self.children) { - child.add_session_signals(selected); + child.add_session_signals(nested); } } diff --git a/src/logs/reading/cache.rs b/src/logs/reading/cache.rs index ade8577ba..d7b5f0b90 100644 --- a/src/logs/reading/cache.rs +++ b/src/logs/reading/cache.rs @@ -1,5 +1,6 @@ //! Bounded, process-local caches of retained facts. Liveness and health are -//! sampled afresh; only file contents/parsed records survive between requests. +//! sampled afresh; file contents, parsed records and derived Session signals +//! survive between requests. use std::collections::BTreeMap; use std::fs; use std::path::{Path, PathBuf}; @@ -59,25 +60,22 @@ impl Cache { } } - fn read( - &mut self, - path: &Path, - load: impl FnOnce(&Path) -> Result, - retained_bytes: impl FnOnce(&T, u64) -> u64, - ) -> Result { + fn read(&mut self, path: &Path, load: impl FnOnce(&Path) -> Result) -> Result { let identity = Identity::read(path)?; if let Some(value) = self.get(path, &identity) { return Ok(value); } let value = load(path)?; - let bytes = retained_bytes(&value, identity.len); // Don't retain a partial in-place write. A later request retries it. if Identity::read(path)? == identity { + let bytes = identity.len; self.insert(path, identity, value.clone(), bytes); } Ok(value) } + /// Return a cached value. A different identity for this path is forgotten + /// so the caller can load and insert the current one. fn get(&mut self, path: &Path, identity: &Identity) -> Option { if let Some(retained) = self.entries.get_mut(path) && retained.identity == *identity @@ -127,21 +125,19 @@ pub(super) fn record(path: &Path) -> Result> { RECORDS .lock() .unwrap() - .read( - path, - |path| Ok(serde_json::from_slice(&fs::read(path)?).ok()), - |_, source_bytes| source_bytes, - ) + .read(path, |path| { + Ok(serde_json::from_slice(&fs::read(path)?).ok()) + }) .with_context(|| format!("could not read {}", path.display())) } pub(super) fn text(path: &Path) -> Result> { static TEXT: LazyLock>>> = LazyLock::new(|| Mutex::new(Cache::new())); - match TEXT.lock().unwrap().read( - path, - |path| Ok(fs::read_to_string(path)?.into()), - |_, source_bytes| source_bytes, - ) { + match TEXT + .lock() + .unwrap() + .read(path, |path| Ok(fs::read_to_string(path)?.into())) + { Ok(text) => Ok(text), Err(error) if error From a0b9bf7fd71a1a7b4b2ec1b4963bb65b00bb0fed Mon Sep 17 00:00:00 2001 From: Jacob Stephens Date: Mon, 26 Oct 2026 10:40:09 -0400 Subject: [PATCH 6/9] thirdshift: failed ticket run (bug diagnosis ended without a reproduced or not reproduced completion line) 2026-10-26T14:40:09Z, host thirdshift. Uncommitted work at the time of failure is included in this commit. From a16834294c4fec6fca80e3766c526387370f9e1c Mon Sep 17 00:00:00 2001 From: Jacob Stephens Date: Mon, 26 Oct 2026 16:15:46 -0400 Subject: [PATCH 7/9] Reuse historical Session signals across CLI processes --- console/README.md | 6 ++ site/architecture/index.html | 2 +- site/how-it-works/index.html | 2 +- src/logs/reading/cache.rs | 23 ++++-- src/logs/reading/cache/retained_signals.rs | 82 ++++++++++++++++++++ tests/metrics.rs | 87 ++++++++++++++++++++++ 6 files changed, 195 insertions(+), 7 deletions(-) create mode 100644 src/logs/reading/cache/retained_signals.rs diff --git a/console/README.md b/console/README.md index 050bb98fa..5f169e3cb 100644 --- a/console/README.md +++ b/console/README.md @@ -290,6 +290,12 @@ After a build on Linux, `python3 console/reproductions/peek-cache-contention.py` checks that parsing another Command's cold Session trace does not delay cached Command details. +Historical Session signals are cached in memory and in disposable +`*.signals-cache.json` files beside their Session traces, with per-trace +`*.signals-cache.lock` files coordinating CLI processes. Appends, replacements +and binary changes invalidate saved signals. A read-only logs directory still +works by parsing without saving a cache. Command records are never rewritten. + Zoom fits a parent span or a padded window around a call. The sticky axis keeps the zoom notice and Zoom out control visible; the slowest calls re-rank within the window. Folds, selection, zoom and Peek disclosures survive live refresh. diff --git a/site/architecture/index.html b/site/architecture/index.html index 947fbd240..70347f3ce 100644 --- a/site/architecture/index.html +++ b/site/architecture/index.html @@ -219,7 +219,7 @@

      Architecture

      thirdshift experiment --release <commit> compares the content commit with the last release tag on touched suites. The release script waits for rechecks and asks permission for a fail or inconclusive Verdict before pushing. Reruns reuse the Experiment, including before opening a pushed branch's PR. It refuses different main content before pushing or merging and checks the merged tree before tagging, so the release cannot contain unmeasured changes. No touched suite or a last release without the Trial flag carries on with a note. Releasing.

      Each Ticket run's own process saves its Run metrics at completion in its Command record, including Ticket runs and Base fixes: outcome, Repairs, CI rounds, Unaddressed findings by Standards, Spec and Security, stage times against their Usual ranges, and minutes by stage and in total. Unaddressed findings counts retain Delivery's final PR body and the Security review's count, so later edits do not change them. Each Session stage keeps the review's Standards and Spec finding counts from its summary, with missing counts left missing. thirdshift show <command> --json includes these retained facts. The Console reads the same details, showing each Ticket run's available metrics and marking unavailable signals as Missing. thirdshift metrics <command> [--json] reads a Ticket run with its children; without a Command it groups weekly trends by repository, Harness, requested Model and Effort, Ticket run kind and session kind. Week and repository filters narrow that view. Historical metrics are computed from Command records without writing back, and missing signals stay missing. No combined score is produced.

      Run metrics and Experiments share one Session usage module for tokens by observed Model and Harness-reported cost. Its normalized-event and retained-span readers resolve call amendments and partial totals, retaining fields a total omits. Input includes cache reads and writes. Reported Session cost wins; otherwise each Model's reported cost replaces its call costs. The metrics reader separately counts tool errors, permission denials, killed work and repeated tool calls within that session. Completion metrics sum these by session kind, and weekly Session trends split by the Harness version from each trace's resource. Saved completion metrics stay fixed. Historical metrics also read converted Session traces, leaving missing traces and unreported measurements missing.

      -

      Command details reconstruct Session signals only for the selected Command and its nested Ticket runs and dispatches; all records still contribute to Usual ranges and health. Weekly metrics read signals for every Command. A bounded process-local cache retains derived signals by the resolved Session trace's path, length, modification time and file identity, checking the identity again after parsing. Appends and replacements invalidate it; unchanged Session traces can be reused across Console requests. Restarting the process clears the cache. Historical Command records are never rewritten.

      +

      Command details reconstruct Session signals only for the selected Command and its nested Ticket runs and dispatches; all records still contribute to Usual ranges and health. Weekly metrics read signals for every Command. A bounded memory cache retains derived signals by the resolved Session trace's path, length, modification time and file identity, checking the identity again after parsing. Disposable *.signals-cache.json files beside each Session trace also retain them across CLI processes and Console restarts. Per-trace locks coordinate readers. Appends, replacements and a different binary invalidate the saved signals; an unwritable cache falls back to parsing. Historical Command records are never rewritten.

      Confirmed Bug origins are joined when metrics are read, from local Scoring checkpoints under ~/.thirdshift/evals/scoring/. The latest window's decision for each bug wins. Its retained local Ticket run identity selects the exact Ticket run; cross-host pull-request attribution selects the latest local Ticket run for that URL, as in Scoring, so Continuations never multiply a count. The Scoring data covers both hosts; metrics read no remote records. Late confirmations count in the Ticket run's start week, corrections move the count to the settled pull request, and rejected or unsettled origins count for nothing. Command details in show --json and the Console share this projection and reconstruct available historical metrics. Command records are never rewritten.

      The same reader checks each session's shell commands against one code-owned destructive-command pattern list. Force pushes, shared-branch history rewrites and recursive deletes outside the Worktree are retained with their commands in Run metrics, with flag counts in weekly Session trends. This check is advisory: it never blocks or fails a Ticket run.

        diff --git a/site/how-it-works/index.html b/site/how-it-works/index.html index 6ff5882af..de5842308 100644 --- a/site/how-it-works/index.html +++ b/site/how-it-works/index.html @@ -160,7 +160,7 @@

        How thirdshift works

        Run metrics and Experiments share Session usage rules. Partial totals replace only reported fields, preserving known input, cache counts and cost. Input includes cache reads and writes. A reported Session cost wins; otherwise each Model's reported cost replaces its call costs. Missing measurements stay missing, and saved completion metrics stay fixed.

        CLI log following and the Console keep reading through the captured writer's exit, including final output written after a Command's outcome is recorded. An exact Session trace finishes when its writer exits or its owning record moves to a later Session.

        The Console shows this machine's Commands, opening on its Inbox. Beside the CLI, you can watch their stages and logs there. Open Command details for a Ticket run's metrics, including available historical signals and late Bug-origin confirmations; unavailable signals say Missing. Stages keeps each Spec run with its Ticket runs, newest-started groups first, filling the window's width beside any open Command details: scroll right for older runs, or switch from All runs to By repository for one grid per repository. When a later Command takes up an issue, the earlier row stays in Resolved for inbox.days from that Command's start, saying why it left.

        -

        Command details and thirdshift show <command> --json read historical Session signals only for that Command and its nested work. The Console reuses derived signals while each Session trace is unchanged, reading it again after an append or replacement. If a details request has not finished within 10 seconds, the pane says “Details unavailable. Reload to try again.”

        +

        Command details and thirdshift show <command> --json read historical Session signals only for that Command and its nested work. The CLI and Console reuse derived signals while each Session trace is unchanged, including across process restarts, reading it again after an append, replacement or binary update. If a details request has not finished within 10 seconds, the pane says “Details unavailable. Reload to try again.”

        The Console reads ready rows' pull requests and Needs you rows' issues and pull requests from GitHub in the background once a minute, even with no pages open. A changed answer wakes open Inbox pages; an unchanged answer leaves them idle. thirdshift status uses the same kept answer and reads GitHub itself only when it is missing or five minutes old. An ended Needs you row leaves once its issue closes or its pull request merges; a Triage row also leaves when needs-triage is removed. Architect and Security runs use the issue and pull request of the Spec run or Ticket run they dispatched. A pull request closed unmerged leaves a failure waiting while its issue is open. Live Silent and Slow rows wait until their Commands end. A merged or closed pull request takes its row out of Ready for review. Each exit says why, with Resolved counted from when GitHub shows it settled. Rows GitHub can speak for keep waiting however old they are. Reopening brings back a row still in Done or Resolved; History keeps the last facts and is not read again. Dismissals hold through GitHub changes; Undo uses GitHub's latest kept facts. Rows with no issue or pull request, no Command record, no read yet or nothing GitHub can find still age out, straight to History.

        When GitHub cannot answer, each row keeps the group its last facts give it; a row never read still goes by age. After five minutes without a good read, the Console and status warn with the last-answer time and latest failure. A successful read clears that warning. With fewer than 500 GraphQL points or 500 REST requests left, both skip reads until that quota resets and say why, leaving the rest to the factory. status --json gives the last-answer time and latest failure. These limits are fixed, with no User config settings.

        status --json also gives each row's kept GitHub facts and when they were read. History keeps those observation times with its facts. A dismissed row stays in Resolved for inbox.days from its dismissal; a later GitHub settlement does not extend it.

        diff --git a/src/logs/reading/cache.rs b/src/logs/reading/cache.rs index d7b5f0b90..842237de4 100644 --- a/src/logs/reading/cache.rs +++ b/src/logs/reading/cache.rs @@ -1,6 +1,6 @@ -//! Bounded, process-local caches of retained facts. Liveness and health are -//! sampled afresh; file contents, parsed records and derived Session signals -//! survive between requests. +//! Bounded caches of retained facts. Liveness and health are sampled afresh; +//! derived Session signals also survive between CLI processes. +mod retained_signals; use std::collections::BTreeMap; use std::fs; use std::path::{Path, PathBuf}; @@ -8,11 +8,12 @@ use std::sync::{Arc, LazyLock, Mutex, Weak}; use std::time::{Instant, SystemTime}; use anyhow::{Context, Result}; +use serde::{Deserialize, Serialize}; use crate::logs::command_record::CommandTree; use crate::logs::session_signals::SessionSignals; -#[derive(PartialEq)] +#[derive(PartialEq, Serialize, Deserialize)] struct Identity { len: u64, modified: Option, @@ -193,8 +194,20 @@ pub(super) fn session_signals(log: &Path) -> Option { if let Some(value) = SIGNALS.lock().unwrap().cache.get(&path, &identity) { return value; } - let value = crate::logs::session_signals::read(&path).ok()?; + // The file lock also coordinates cold readers in separate CLI processes. + // An unwritable cache is only a missed optimization, never missing evidence. + let retained = retained_signals::RetainedSignals::lock(&path).ok(); + let identity = Identity::read(&path).ok()?; + let cached = retained.as_ref().and_then(|cache| cache.read(&identity)); + let persist = cached.is_none(); + let value = match cached { + Some(value) => value, + None => crate::logs::session_signals::read(&path).ok()?, + }; if Identity::read(&path).ok().as_ref() == Some(&identity) { + if persist && let Some(retained) = retained { + let _ = retained.write(&identity, &value); + } let bytes = serde_json::to_vec(&value) .expect("Session signals serialize") .len() as u64; diff --git a/src/logs/reading/cache/retained_signals.rs b/src/logs/reading/cache/retained_signals.rs new file mode 100644 index 000000000..5683a107d --- /dev/null +++ b/src/logs/reading/cache/retained_signals.rs @@ -0,0 +1,82 @@ +//! Disposable derived evidence beside a Session trace, never in its Command +//! record. Atomic replacement and a per-trace lock keep CLI readers coordinated. +use std::fs::{self, File}; +use std::io; +use std::path::{Path, PathBuf}; +use std::sync::LazyLock; + +use serde::{Deserialize, Serialize}; + +use super::Identity; +use crate::logs::session_signals::SessionSignals; + +#[derive(PartialEq, Serialize, Deserialize)] +struct Executable { + path: PathBuf, + identity: Identity, +} + +#[derive(Serialize, Deserialize)] +struct Saved { + executable: Executable, + trace: Identity, + signals: Option, +} + +pub(super) struct RetainedSignals { + path: PathBuf, + // Locks a separate stable inode, since the saved value is replaced atomically. + _lock: File, +} + +impl RetainedSignals { + pub(super) fn lock(trace: &Path) -> io::Result { + let lock = File::options() + .read(true) + .write(true) + .create(true) + .truncate(false) + .open(trace.with_extension("signals-cache.lock"))?; + lock.lock()?; + Ok(Self { + path: trace.with_extension("signals-cache.json"), + _lock: lock, + }) + } + + pub(super) fn read(&self, identity: &Identity) -> Option> { + let executable = executable()?; + let saved: Saved = serde_json::from_slice(&fs::read(&self.path).ok()?).ok()?; + (saved.trace == *identity && saved.executable == *executable).then_some(saved.signals) + } + + pub(super) fn write( + &self, + identity: &Identity, + signals: &Option, + ) -> anyhow::Result<()> { + // Include the exact executable identity so a change to signal semantics + // cannot silently reuse values derived by a previous binary. + let Some(executable) = executable() else { + return Ok(()); + }; + let temporary = tempfile::NamedTempFile::new_in(self.path.parent().unwrap())?; + serde_json::to_writer( + &temporary, + &serde_json::json!({ + "executable": executable, "trace": identity, "signals": signals + }), + )?; + temporary.persist(&self.path)?; + Ok(()) + } +} + +fn executable() -> Option<&'static Executable> { + static EXECUTABLE: LazyLock> = LazyLock::new(|| { + let path = std::env::current_exe().ok()?; + let identity = Identity::read(&path).ok()?; + Some(Executable { path, identity }) + }); + EXECUTABLE.as_ref() +} diff --git a/tests/metrics.rs b/tests/metrics.rs index ee7722618..7decb9fc7 100644 --- a/tests/metrics.rs +++ b/tests/metrics.rs @@ -7,6 +7,93 @@ use support::Scenario; const LOGS: &str = "home/.thirdshift/logs/acme/widgets/commands/issue"; +#[cfg(target_os = "linux")] +#[test] +fn unchanged_session_signals_survive_between_cli_processes() { + use std::ffi::CString; + use std::io::{Read, Write}; + use std::os::fd::FromRawFd; + use std::os::unix::ffi::OsStrExt; + + let scenario = Scenario::new(); + let log = trace(&scenario, "historical", "claude", "1.0", json!([])); + let mut past = record(7); + past["stages"][0]["session_log"] = json!(log); + let record = write(&scenario, "7-fixture.command.json", &past); + let original_record = fs::read(&record).unwrap(); + let original_trace = fs::read_to_string(&log).unwrap(); + // SAFETY: File owns the successful descriptor; the path is NUL terminated. + let fd = unsafe { libc::inotify_init1(libc::IN_NONBLOCK | libc::IN_CLOEXEC) }; + assert!(fd >= 0); + let mut watcher = unsafe { fs::File::from_raw_fd(fd) }; + let name = CString::new(log.as_os_str().as_bytes()).unwrap(); + assert!(unsafe { libc::inotify_add_watch(fd, name.as_ptr(), libc::IN_OPEN) } >= 0); + let opened = |watcher: &mut fs::File| { + let mut buffer = [0; 1024]; + match watcher.read(&mut buffer) { + Ok(len) => usize::from(len > 0), + Err(error) if error.kind() == std::io::ErrorKind::WouldBlock => 0, + Err(error) => panic!("{error}"), + } + }; + let first = metrics(&scenario, &["metrics", "widgets#7", "--json"]); + assert_eq!(opened(&mut watcher), 1); + let second = metrics(&scenario, &["metrics", "widgets#7", "--json"]); + assert_eq!( + opened(&mut watcher), + 0, + "a second CLI process reparsed an unchanged historical Session trace" + ); + assert_eq!(first, second); + + let amendment = |version: &str| { + serde_json::to_string(&json!({"resourceLogs": [{"resource": {"attributes": [ + {"key": "thirdshift.harness.version", "value": {"stringValue": version}} + ]}, "scopeLogs": []}]})) + .unwrap() + + "\n" + }; + fs::OpenOptions::new() + .append(true) + .open(&log) + .unwrap() + .write_all(amendment("2.0").as_bytes()) + .unwrap(); + opened(&mut watcher); + let appended = metrics(&scenario, &["metrics", "widgets#7", "--json"]); + assert_eq!(opened(&mut watcher), 1); + assert_eq!( + appended["command"]["metrics"]["stages"][0]["session_signals"]["harness_version"], + "2.0" + ); + + // Replacing a trace with the same length and mtime must still refresh it. + let modified = fs::metadata(&log).unwrap().modified().unwrap(); + let replacement = log.with_extension("replacement"); + fs::write(&replacement, original_trace + &amendment("3.0")).unwrap(); + fs::File::open(&replacement) + .unwrap() + .set_times(fs::FileTimes::new().set_modified(modified)) + .unwrap(); + fs::rename(replacement, &log).unwrap(); + opened(&mut watcher); + // SAFETY: the descriptor is still owned by watcher and name is NUL terminated. + assert!(unsafe { libc::inotify_add_watch(fd, name.as_ptr(), libc::IN_OPEN) } >= 0); + let replaced = metrics(&scenario, &["metrics", "widgets#7", "--json"]); + assert_eq!(opened(&mut watcher), 1); + assert_eq!( + replaced["command"]["metrics"]["stages"][0]["session_signals"]["harness_version"], + "3.0" + ); + let shown = metrics(&scenario, &["show", "widgets#7", "--json"]); + assert_eq!(opened(&mut watcher), 0); + assert_eq!( + shown["command"]["run_metrics"]["session_signals"], + replaced["command"]["metrics"]["session_signals"] + ); + assert_eq!(fs::read(record).unwrap(), original_record); +} + fn trace( scenario: &Scenario, name: &str, From 88b61cd0eb26c7cff926e5e9999e74ffa1a644cf Mon Sep 17 00:00:00 2001 From: Jacob Stephens Date: Mon, 26 Oct 2026 16:23:08 -0400 Subject: [PATCH 8/9] Cache Missing Session signals and use exact Command identities --- src/logs/reading.rs | 12 ++--- src/logs/reading/attribution.rs | 4 ++ src/logs/reading/cache.rs | 4 +- tests/metrics.rs | 96 +++++++++++++++++++++++---------- 4 files changed, 79 insertions(+), 37 deletions(-) diff --git a/src/logs/reading.rs b/src/logs/reading.rs index 097fec80e..f550fbd11 100644 --- a/src/logs/reading.rs +++ b/src/logs/reading.rs @@ -86,11 +86,10 @@ impl Snapshot { let mut snapshot = Self::capture_records(Arc::new(inventory::Native), root, now, silent, true)?; if let Observation::Record(selected) = snapshot.resolve(input)?.observation { - let path = selected.record_path.clone(); - let name = selected.record.name.clone(); + let identity = CommandIdentity::of(selected); for entry in &mut snapshot.commands { if let Entry::Record(command) = entry { - command.add_session_signals(Some((&path, &name))); + command.add_session_signals(Some(&identity)); } } } @@ -282,12 +281,11 @@ pub(super) struct CurrentStage { impl Command { /// Enrich this Command when `selected` is every Command (`None`), or this - /// Command's own record path and name. A match also enriches nested Ticket + /// Command's exact identity. A match also enriches nested Ticket /// runs and dispatches. Path alone is not enough: a dispatch shares its /// parent's record path. Usual ranges and health already include every record. - fn add_session_signals(&mut self, selected: Option<(&Path, &str)>) { - let enrich = selected - .is_none_or(|(path, name)| self.record_path == path && self.record.name == name); + fn add_session_signals(&mut self, selected: Option<&CommandIdentity>) { + let enrich = selected.is_none_or(|identity| identity.matches(self)); if enrich && let Some(mut metrics) = self.record.run_metrics.take() { metrics.add_session_signals(&self.record, cache::session_signals); self.record.run_metrics = Some(metrics); diff --git a/src/logs/reading/attribution.rs b/src/logs/reading/attribution.rs index 4d0b769fa..89dd4c768 100644 --- a/src/logs/reading/attribution.rs +++ b/src/logs/reading/attribution.rs @@ -15,6 +15,10 @@ pub(in crate::logs) struct CommandIdentity { } impl CommandIdentity { + pub(in crate::logs) fn matches(&self, command: &Command) -> bool { + self.record_path == command.record_path && self.name == command.record.name + } + pub(in crate::logs) fn of(command: &Command) -> Self { Self { record_path: command.record_path.clone(), diff --git a/src/logs/reading/cache.rs b/src/logs/reading/cache.rs index 842237de4..4586cd731 100644 --- a/src/logs/reading/cache.rs +++ b/src/logs/reading/cache.rs @@ -202,7 +202,9 @@ pub(super) fn session_signals(log: &Path) -> Option { let persist = cached.is_none(); let value = match cached { Some(value) => value, - None => crate::logs::session_signals::read(&path).ok()?, + // Readers expose a parse error as Missing. Retain that observation too; + // repairing the trace changes its identity and retries the parser. + None => crate::logs::session_signals::read(&path).ok().flatten(), }; if Identity::read(&path).ok().as_ref() == Some(&identity) { if persist && let Some(retained) = retained { diff --git a/tests/metrics.rs b/tests/metrics.rs index 7decb9fc7..84aec09ad 100644 --- a/tests/metrics.rs +++ b/tests/metrics.rs @@ -7,13 +7,67 @@ use support::Scenario; const LOGS: &str = "home/.thirdshift/logs/acme/widgets/commands/issue"; +#[cfg(target_os = "linux")] +struct TraceOpens(fs::File); + +#[cfg(target_os = "linux")] +impl TraceOpens { + fn new(log: &std::path::Path) -> Self { + use std::os::fd::FromRawFd; + use std::os::unix::ffi::OsStrExt; + // SAFETY: File owns the successful descriptor; name is NUL terminated. + let fd = unsafe { libc::inotify_init1(libc::IN_NONBLOCK | libc::IN_CLOEXEC) }; + assert!(fd >= 0); + let file = unsafe { fs::File::from_raw_fd(fd) }; + let name = std::ffi::CString::new(log.as_os_str().as_bytes()).unwrap(); + assert!(unsafe { libc::inotify_add_watch(fd, name.as_ptr(), libc::IN_OPEN) } >= 0); + Self(file) + } + + fn take(&mut self) -> bool { + use std::io::Read; + let mut buffer = [0; 1024]; + match self.0.read(&mut buffer) { + Ok(len) => len > 0, + Err(error) if error.kind() == std::io::ErrorKind::WouldBlock => false, + Err(error) => panic!("{error}"), + } + } +} + +#[cfg(target_os = "linux")] +#[test] +fn unchanged_malformed_session_signals_are_not_reparsed_and_repair_refreshes_them() { + let scenario = Scenario::new(); + let log = trace(&scenario, "malformed", "claude", "1.0", json!([])); + fs::write(&log, "{\"resourceLogs\":[]}\n{invalid}\n").unwrap(); + let mut past = record(7); + past["stages"][0]["session_log"] = json!(log); + write(&scenario, "7-fixture.command.json", &past); + let mut opens = TraceOpens::new(&log); + let first = metrics(&scenario, &["metrics", "widgets#7", "--json"]); + assert!(first["command"]["metrics"]["stages"][0]["session_signals"].is_null()); + assert!(opens.take()); + let second = metrics(&scenario, &["metrics", "widgets#7", "--json"]); + assert!( + !opens.take(), + "unchanged malformed Session trace was reparsed" + ); + assert_eq!(first, second); + trace(&scenario, "malformed", "claude", "2.0", json!([])); + opens.take(); + let repaired = metrics(&scenario, &["metrics", "widgets#7", "--json"]); + assert!(opens.take()); + assert_eq!( + repaired["command"]["metrics"]["stages"][0]["session_signals"]["harness_version"], + "2.0" + ); +} + #[cfg(target_os = "linux")] #[test] fn unchanged_session_signals_survive_between_cli_processes() { - use std::ffi::CString; - use std::io::{Read, Write}; - use std::os::fd::FromRawFd; - use std::os::unix::ffi::OsStrExt; + use std::io::Write; let scenario = Scenario::new(); let log = trace(&scenario, "historical", "claude", "1.0", json!([])); @@ -22,26 +76,12 @@ fn unchanged_session_signals_survive_between_cli_processes() { let record = write(&scenario, "7-fixture.command.json", &past); let original_record = fs::read(&record).unwrap(); let original_trace = fs::read_to_string(&log).unwrap(); - // SAFETY: File owns the successful descriptor; the path is NUL terminated. - let fd = unsafe { libc::inotify_init1(libc::IN_NONBLOCK | libc::IN_CLOEXEC) }; - assert!(fd >= 0); - let mut watcher = unsafe { fs::File::from_raw_fd(fd) }; - let name = CString::new(log.as_os_str().as_bytes()).unwrap(); - assert!(unsafe { libc::inotify_add_watch(fd, name.as_ptr(), libc::IN_OPEN) } >= 0); - let opened = |watcher: &mut fs::File| { - let mut buffer = [0; 1024]; - match watcher.read(&mut buffer) { - Ok(len) => usize::from(len > 0), - Err(error) if error.kind() == std::io::ErrorKind::WouldBlock => 0, - Err(error) => panic!("{error}"), - } - }; + let mut watcher = TraceOpens::new(&log); let first = metrics(&scenario, &["metrics", "widgets#7", "--json"]); - assert_eq!(opened(&mut watcher), 1); + assert!(watcher.take()); let second = metrics(&scenario, &["metrics", "widgets#7", "--json"]); - assert_eq!( - opened(&mut watcher), - 0, + assert!( + !watcher.take(), "a second CLI process reparsed an unchanged historical Session trace" ); assert_eq!(first, second); @@ -59,9 +99,9 @@ fn unchanged_session_signals_survive_between_cli_processes() { .unwrap() .write_all(amendment("2.0").as_bytes()) .unwrap(); - opened(&mut watcher); + watcher.take(); let appended = metrics(&scenario, &["metrics", "widgets#7", "--json"]); - assert_eq!(opened(&mut watcher), 1); + assert!(watcher.take()); assert_eq!( appended["command"]["metrics"]["stages"][0]["session_signals"]["harness_version"], "2.0" @@ -76,17 +116,15 @@ fn unchanged_session_signals_survive_between_cli_processes() { .set_times(fs::FileTimes::new().set_modified(modified)) .unwrap(); fs::rename(replacement, &log).unwrap(); - opened(&mut watcher); - // SAFETY: the descriptor is still owned by watcher and name is NUL terminated. - assert!(unsafe { libc::inotify_add_watch(fd, name.as_ptr(), libc::IN_OPEN) } >= 0); + watcher = TraceOpens::new(&log); let replaced = metrics(&scenario, &["metrics", "widgets#7", "--json"]); - assert_eq!(opened(&mut watcher), 1); + assert!(watcher.take()); assert_eq!( replaced["command"]["metrics"]["stages"][0]["session_signals"]["harness_version"], "3.0" ); let shown = metrics(&scenario, &["show", "widgets#7", "--json"]); - assert_eq!(opened(&mut watcher), 0); + assert!(!watcher.take()); assert_eq!( shown["command"]["run_metrics"]["session_signals"], replaced["command"]["metrics"]["session_signals"] From 142bc2fc890a3e9185c29c61dddf01fdce3804e1 Mon Sep 17 00:00:00 2001 From: Jacob Stephens Date: Mon, 26 Oct 2026 16:57:13 -0400 Subject: [PATCH 9/9] Consume skill fixture prompts before emitting partial output --- tests/skill.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/skill.rs b/tests/skill.rs index 6bc9a8aaa..7e15999e5 100644 --- a/tests/skill.rs +++ b/tests/skill.rs @@ -413,7 +413,7 @@ fn partial_muse_and_opencode_messages_are_not_success() { fs::remove_file(&executable).unwrap(); fs::write( &executable, - format!("#!/bin/sh\ncat <<'JSON'\n{event}\nJSON\nexit 0\n"), + format!("#!/bin/sh\ncat > /dev/null\ncat <<'JSON'\n{event}\nJSON\nexit 0\n"), ) .unwrap(); fs::set_permissions(&executable, fs::Permissions::from_mode(0o755)).unwrap();