Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 10 additions & 0 deletions console/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -340,6 +340,16 @@ line, cut off at the pane's edge, with the whole command on hover and in the
copy. Session trace links also stay on one line, trimmed from the start so the
end of each path, which tells them apart, still shows.

After a build on Linux, `python3 console/reproductions/peek-cache-contention.py`
checks that parsing another Command's cold Session trace does not delay cached
Command details.

Historical Session signals are cached in memory and in disposable
`*.signals-cache.json` files beside their Session traces, with per-trace
`*.signals-cache.lock` files coordinating CLI processes. Appends, replacements
and binary changes invalidate saved signals. A read-only logs directory still
works by parsing without saving a cache. Command records are never rewritten.

Zoom fits a parent span or a padded window around a call. The sticky axis keeps
the zoom notice and Zoom out control visible; the slowest calls re-rank within
the window. Folds, selection, zoom and Peek disclosures survive live refresh.
Expand Down
99 changes: 99 additions & 0 deletions console/reproductions/peek-cache-contention.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,99 @@
#!/usr/bin/env python3
"""After cargo build, check cached details during another Command's cold parse.

Linux only: IN_OPEN pins the overlapping requests to actual Session trace IO.
The 128 MiB OTLP fixture makes lock contention visible even on a fast machine.
"""

import concurrent.futures
import ctypes
import json
import os
import pathlib
import select
import subprocess
import tempfile
import time
import urllib.request


def check():
binary = str(pathlib.Path(__file__).resolve().parents[2] / 'target/debug/thirdshift')
with tempfile.TemporaryDirectory(prefix='thirdshift-peek-contention-') as directory:
task_home = pathlib.Path(directory)
config = task_home / '.thirdshift'
records = config / 'logs/acme/widgets/commands/issue'
records.mkdir(parents=True)
(config / 'config.toml').write_text('[console]\nport = 0\n')
for number in [1, 2]:
trace = records / f'{number}-fixture.logs.jsonl'
if number == 1:
trace.write_text('{"resourceLogs":[]}\n')
else:
line = json.dumps({'resourceLogs': [{'scopeLogs': [{'logRecords': [
{'body': {'stringValue': 'x' * (8 * 1024 * 1024)}}
]}]}]}) + '\n'
with trace.open('w') as output:
for _ in range(16):
output.write(line)
record = dict(
version=1, kind='Run', repository='acme/widgets',
name=f'acme/widgets#{number}',
issue=dict(number=number, url=f'https://github.com/acme/widgets/issues/{number}', title='Widgets'),
hostname='factory', process_id=0, process_start='gone', harness='claude',
started_at='2026-10-19T11:00:00-04:00', ended_at='2026-10-19T11:01:00-04:00',
stages=[dict(name='implement', began_at='2026-10-19T11:00:00-04:00', session_log=str(trace))],
outcome='merged',
)
(records / f'{number}-fixture.command.json').write_text(json.dumps(record))
env = dict(os.environ, HOME=directory)
server = subprocess.Popen(
[binary, 'console', '--no-sign-in'], env=env,
stdout=subprocess.PIPE, text=True,
)
try:
assert select.select([server.stdout], [], [], 10)[0], 'Console startup timed out'
origin = server.stdout.readline().strip()
assert origin.startswith('http://127.0.0.1:'), origin

def peek(number):
began = time.monotonic()
with urllib.request.urlopen(
origin + '/peek?command=widgets%23' + str(number), timeout=45,
) as response:
assert response.status == 200
response.read()
return time.monotonic() - began

print('warm selected details seconds', peek(1), flush=True)
libc = ctypes.CDLL(None, use_errno=True)
descriptor = libc.inotify_init1(os.O_NONBLOCK | os.O_CLOEXEC)
assert descriptor >= 0
try:
watched = libc.inotify_add_watch(
descriptor, os.fsencode(records / '2-fixture.logs.jsonl'), 0x20, # IN_OPEN
)
assert watched >= 0
with concurrent.futures.ThreadPoolExecutor() as executor:
cold = executor.submit(peek, 2)
assert select.select([descriptor], [], [], 15)[0], 'cold reader did not open trace'
os.read(descriptor, 8192)
warm_seconds = peek(1)
cold_seconds = cold.result(timeout=45)
print('cold details seconds', cold_seconds, flush=True)
print('cached unrelated details seconds', warm_seconds, flush=True)
assert warm_seconds < 1, f'cached details took {warm_seconds:.3f}s during an unrelated parse'
finally:
os.close(descriptor)
finally:
server.terminate()
try:
server.wait(timeout=10)
except subprocess.TimeoutExpired:
server.kill()
server.wait()
server.stdout.close()


if __name__ == '__main__':
check()
26 changes: 26 additions & 0 deletions console/tests/inbox.test.cjs
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,32 @@ async function pageFor(t, options) {
return page;
}

test('Command details times out a held request and can be opened again', async t => {
const page = await pageFor(t);
await page.clock.install();
let release;
const held = new Promise(resolve => { release = resolve; });
await page.route('**/peek?*', async route => {
await held;
await route.continue();
});
try {
await page.goto(`${origin}/?page=security`);
const requested = page.waitForRequest('**/peek?*');
await page.locator('.command .state').first().click();
await requested;
await page.clock.fastForward(15000);
await page.getByText('Details unavailable. Reload to try again.', { exact: true })
.waitFor({ timeout: 2000 });
} finally {
release();
await page.unrouteAll({ behavior: 'wait' });
}
await page.locator('[data-close-peek]').click();
await page.locator('.command .state').first().click();
await page.locator('#peek dt').first().waitFor();
});

// The Inbox shows one section, a folder, at a time; Details opens from a row.
const folder = (page, group) => page.locator(`#inbox .folders a[data-group="${group}"]`);
const shownGroup = page => page.locator('#inbox > .group:not([hidden])');
Expand Down
1 change: 1 addition & 0 deletions site/architecture/index.html
Original file line number Diff line number Diff line change
Expand Up @@ -219,6 +219,7 @@ <h1>Architecture</h1>
<p><code>thirdshift experiment <wbr>--release <wbr>&lt;commit&gt;</code> compares the content commit with the last release tag on touched suites. The release script waits for rechecks and asks permission for a fail or inconclusive Verdict before pushing. Reruns reuse the Experiment, including before opening a pushed branch's PR. It refuses different main content before pushing or merging and checks the merged tree before tagging, so the release cannot contain unmeasured changes. No touched suite or a last release without the Trial flag carries on with a note. <a href="/docs/#releasing">Releasing</a>.</p>
<p>Each Ticket run's own process saves its <strong>Run metrics</strong> at completion in its Command record, including Ticket runs and Base fixes: outcome, Repairs, CI rounds, Unaddressed findings by Standards, Spec and Security, stage times against their Usual ranges, and minutes by stage and in total. Unaddressed findings counts retain Delivery's final PR body and the Security review's count, so later edits do not change them. Each Session stage keeps the review's Standards and Spec finding counts from its summary, with missing counts left missing. <code>thirdshift show &lt;command&gt; --json</code> includes these retained facts. The Console reads the same details, showing each Ticket run's available metrics and marking unavailable signals as Missing. <code>thirdshift metrics &lt;command&gt; [--json]</code> reads a Ticket run with its children; without a Command it groups weekly trends by repository, Harness, requested Model and Effort, Ticket run kind and session kind. Week and repository filters narrow that view. Historical metrics are computed from Command records without writing back, and missing signals stay missing. No combined score is produced.</p>
<p>Run metrics and Experiments share one Session usage module for tokens by observed Model and Harness-reported cost. Its normalized-event and retained-span readers resolve call amendments and partial totals, retaining fields a total omits. Input includes cache reads and writes. Reported Session cost wins; otherwise each Model's reported cost replaces its call costs. The metrics reader separately counts tool errors, permission denials, killed work and repeated tool calls within that session. Completion metrics sum these by session kind, and weekly Session trends split by the Harness version from each trace's resource. Saved completion metrics stay fixed. Historical metrics also read converted Session traces, leaving missing traces and unreported measurements missing.</p>
<p>Command details reconstruct Session signals only for the selected Command and its nested Ticket runs and dispatches; all records still contribute to Usual ranges and health. Weekly metrics read signals for every Command. A bounded memory cache retains derived signals by the resolved Session trace's path, length, modification time and file identity, checking the identity again after parsing. Disposable <code>*.signals-cache.json</code> files beside each Session trace also retain them across CLI processes and Console restarts. Per-trace locks coordinate readers. Appends, replacements and a different binary invalidate the saved signals; an unwritable cache falls back to parsing. Historical Command records are never rewritten.</p>
<p>Confirmed Bug origins are joined when metrics are read, from local Scoring checkpoints under <code>~/.thirdshift/evals/scoring/</code>. The latest window's decision for each bug wins. Its retained local Ticket run identity selects the exact Ticket run; cross-host pull-request attribution selects the latest local Ticket run for that URL, as in Scoring, so Continuations never multiply a count. The Scoring data covers both hosts; metrics read no remote records. Late confirmations count in the Ticket run's start week, corrections move the count to the settled pull request, and rejected or unsettled origins count for nothing. Command details in <code>show --json</code> and the Console share this projection and reconstruct available historical metrics. Command records are never rewritten.</p>
<p>The same reader checks each session's shell commands against one code-owned destructive-command pattern list. Force pushes, shared-branch history rewrites and recursive deletes outside the Worktree are retained with their commands in Run metrics, with flag counts in weekly Session trends. This check is advisory: it never blocks or fails a Ticket run.</p>
<ol class="arch-stages">
Expand Down
1 change: 1 addition & 0 deletions site/how-it-works/index.html
Original file line number Diff line number Diff line change
Expand Up @@ -160,6 +160,7 @@ <h1>How thirdshift works</h1>
<p>Run metrics and Experiments share Session usage rules. Partial totals replace only reported fields, preserving known input, cache counts and cost. Input includes cache reads and writes. A reported Session cost wins; otherwise each Model's reported cost replaces its call costs. Missing measurements stay missing, and saved completion metrics stay fixed.</p>
<p>CLI log following and the Console keep reading through the captured writer's exit, including final output written after a Command's outcome is recorded. An exact Session trace finishes when its writer exits or its owning record moves to a later Session.</p>
<p>The <strong>Console</strong> shows this machine's Commands, opening on its <strong>Inbox</strong>. Beside the CLI, you can watch their stages and logs there. Open Command details for a Ticket run's metrics, including available historical signals and late Bug-origin confirmations; unavailable signals say Missing. Stages keeps each Spec run with its Ticket runs, newest-started groups first, filling the window's width beside any open Command details: scroll right for older runs, or switch from All runs to By repository for one grid per repository, each folded to its name and run count until opened. When a later Command takes up an issue, the earlier row stays in Resolved for <code>inbox.days</code> from that Command's start, saying why it left.</p>
<p>Command details and <code>thirdshift show &lt;command&gt; --json</code> read historical Session signals only for that Command and its nested work. The CLI and Console reuse derived signals while each Session trace is unchanged, including across process restarts, reading it again after an append, replacement or binary update. If a details request has not finished within 10 seconds, the pane says “Details unavailable. Reload to try again.”</p>
<p>The Console reads ready rows' pull requests and Needs you rows' issues and pull requests from GitHub in the background once a minute, even with no pages open. A changed answer wakes open Inbox pages; an unchanged answer leaves them idle. <code>thirdshift status</code> uses the same kept answer and reads GitHub itself only when it is missing or five minutes old. An ended Needs you row leaves once its issue closes or its pull request merges; a Triage row also leaves when <code>needs-triage</code> is removed. Architect and Security runs use the issue and pull request of the Spec run or Ticket run they dispatched. A pull request closed unmerged leaves a failure waiting while its issue is open. Live Silent and Slow rows wait until their Commands end. A merged or closed pull request takes its row out of Ready for review. Each exit says why, with Resolved counted from when GitHub shows it settled. Rows GitHub can speak for keep waiting however old they are. Reopening brings back a row still in Done or Resolved; History keeps the last facts and is not read again. Dismissals hold through GitHub changes; Undo uses GitHub's latest kept facts. Rows with no issue or pull request, no Command record, no read yet or nothing GitHub can find still age out, straight to History.</p>
<p>When GitHub cannot answer, each row keeps the group its last facts give it; a row never read still goes by age. After five minutes without a good read, the Console and <code>status</code> warn with the last-answer time and latest failure. A successful read clears that warning. With fewer than 500 GraphQL points or 500 REST requests left, both skip reads until that quota resets and say why, leaving the rest to the factory. <code>status --json</code> gives the last-answer time and latest failure. These limits are fixed, with no User config settings.</p>
<p><code>status --json</code> also gives each row's kept GitHub facts and when they were read. History keeps those observation times with its facts. A dismissed row stays in Resolved for <code>inbox.days</code> from its dismissal; a later GitHub settlement does not extend it.</p>
Expand Down
10 changes: 9 additions & 1 deletion src/console/assets/console.js
Original file line number Diff line number Diff line change
Expand Up @@ -334,6 +334,11 @@ async function loadPeek(url) {
pendingPeek?.abort();
const request = new AbortController();
pendingPeek = request;
let timedOut = false;
const timeout = setTimeout(() => {
timedOut = true;
request.abort();
}, 10000);
try {
const response = await fetch(url, { signal: request.signal, cache: 'no-store' });
if (!response.ok) throw new Error('Details unavailable');
Expand All @@ -349,8 +354,11 @@ async function loadPeek(url) {
if (content.querySelector('[data-live="true"]')) startUpdates();
else stopPeekUpdates();
} catch (error) {
if (!request.signal.aborted && error.name !== 'AbortError') content.textContent = 'Details unavailable. Reload to try again.';
if (pendingPeek === request && (timedOut || (!request.signal.aborted && error.name !== 'AbortError'))) {
content.textContent = 'Details unavailable. Reload to try again.';
}
} finally {
clearTimeout(timeout);
if (pendingPeek === request) pendingPeek = null;
}
}
Expand Down
51 changes: 42 additions & 9 deletions src/logs/reading.rs
Original file line number Diff line number Diff line change
Expand Up @@ -62,15 +62,38 @@ impl Snapshot {
}

pub(super) fn load_metrics(root: &Path, now: DateTime<Local>) -> Result<Self> {
Self::load_metrics_with_silence(root, now, std::time::Duration::from_secs(600))
let mut snapshot = Self::capture_records(
Arc::new(inventory::Native),
root,
now,
std::time::Duration::from_secs(600),
true,
)?;
for entry in &mut snapshot.commands {
if let Entry::Record(command) = entry {
command.add_session_signals(None);
}
}
Ok(snapshot)
}

pub(super) fn load_metrics_with_silence(
pub(super) fn load_details(
root: &Path,
now: DateTime<Local>,
silent: std::time::Duration,
input: &str,
) -> Result<Self> {
Self::capture_records(Arc::new(inventory::Native), root, now, silent, true)
let mut snapshot =
Self::capture_records(Arc::new(inventory::Native), root, now, silent, true)?;
if let Observation::Record(selected) = snapshot.resolve(input)?.observation {
let identity = CommandIdentity::of(selected);
for entry in &mut snapshot.commands {
if let Entry::Record(command) = entry {
command.add_session_signals(Some(&identity));
}
}
}
Ok(snapshot)
}

fn capture(
Expand Down Expand Up @@ -257,6 +280,22 @@ pub(super) struct CurrentStage {
}

impl Command {
/// Enrich this Command when `selected` is every Command (`None`), or this
/// Command's exact identity. A match also enriches nested Ticket
/// runs and dispatches. Path alone is not enough: a dispatch shares its
/// parent's record path. Usual ranges and health already include every record.
fn add_session_signals(&mut self, selected: Option<&CommandIdentity>) {
let enrich = selected.is_none_or(|identity| identity.matches(self));
if enrich && let Some(mut metrics) = self.record.run_metrics.take() {
metrics.add_session_signals(&self.record, cache::session_signals);
self.record.run_metrics = Some(metrics);
}
let nested = if enrich { None } else { selected };
for child in self.dispatched.iter_mut().chain(&mut self.children) {
child.add_session_signals(nested);
}
}

fn new(
inventory: &Arc<Inventory>,
record: CommandRecord,
Expand Down Expand Up @@ -328,12 +367,6 @@ fn read_records(
if record.run_metrics.is_none() && record.outcome.is_some() {
record.run_metrics = Some(super::run_metrics::RunMetrics::historical(record));
}
if let Some(mut metrics) = record.run_metrics.take() {
metrics.add_session_signals(record, |log| {
super::session_signals::read(log).ok().flatten()
});
record.run_metrics = Some(metrics);
}
}
}
let record = &mut tree.command;
Expand Down
4 changes: 4 additions & 0 deletions src/logs/reading/attribution.rs
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,10 @@ pub(in crate::logs) struct CommandIdentity {
}

impl CommandIdentity {
pub(in crate::logs) fn matches(&self, command: &Command) -> bool {
self.record_path == command.record_path && self.name == command.record.name
}

pub(in crate::logs) fn of(command: &Command) -> Self {
Self {
record_path: command.record_path.clone(),
Expand Down
Loading
Loading