diff --git a/apps/presentation/dashboard/src/features/personal-workspace/goal-research-progress.css b/apps/presentation/dashboard/src/features/personal-workspace/goal-research-progress.css index e0136d6caf..4e75c69842 100644 --- a/apps/presentation/dashboard/src/features/personal-workspace/goal-research-progress.css +++ b/apps/presentation/dashboard/src/features/personal-workspace/goal-research-progress.css @@ -53,7 +53,7 @@ .research-progress-basis { border-block-start: 1px solid var(--pw-line); margin-block-start: 12px; } .research-progress-basis > summary { font-size: var(--text-body); } .research-progress-basis > summary > span { color: var(--pw-muted); font-size: var(--text-caption); } -.research-usage h4 { font-size: var(--text-body); margin-block: 16px 8px; } +.research-usage h4 { font-size: var(--text-body); font-weight: 500; margin: 0; } .research-effect-evidence h4 { font-size: var(--text-caption); line-height: var(--leading-prose); margin: 8px 0; } .research-effect-citations { display: flex; flex-wrap: wrap; gap: 4px 20px; } .research-progress .research-effect-citations button { display: inline-flex; align-items: center; gap: 4px; min-height: 44px; font: inherit; font-size: var(--text-body); background: transparent; border: 0; border-radius: 0; padding: 4px 0; color: var(--pw-text); cursor: pointer; } @@ -72,14 +72,26 @@ .research-native-values strong { font-size: var(--text-body); font-weight: 600; font-variant-numeric: tabular-nums; } .research-native-values small { display: block; } .research-native-basis { font-size: var(--text-caption); } -.research-progress-basis > dl, .research-usage-basis { display: grid; grid-template-columns: 120px minmax(0, 1fr); gap: 8px 16px; font-size: var(--text-caption); } +.research-usage-basis { display: grid; grid-template-columns: 140px minmax(0, 1fr); gap: 12px 24px; margin-block: 8px 20px; font-size: var(--text-body); line-height: var(--leading-prose); } .research-progress-basis dt, .research-usage-basis dt { color: var(--pw-muted); } .research-progress-basis dd, .research-usage-basis dd { margin: 0; } .research-progress code { font: inherit; font-family: var(--font-mono); } .research-progress-values { display: grid; grid-template-columns: repeat(3, minmax(0, 1fr)); gap: 16px; margin-block: 16px; } .research-progress-values > div { min-width: 0; } -.research-progress-values dt { font-size: var(--text-caption); color: var(--pw-muted); } -.research-progress-values dd { margin: 4px 0; font-size: var(--text-body); font-weight: 600; font-variant-numeric: tabular-nums; } +.research-progress-values dt { font-size: var(--text-body); color: var(--pw-muted); } +.research-progress-values dd { margin: 8px 0; font-size: var(--text-title); font-weight: 600; font-variant-numeric: tabular-nums; } +.research-progress-values dd > span { font-size: var(--text-body); font-weight: 400; color: var(--pw-muted); } +.research-usage { padding-block: 12px 8px; } +.research-usage-heading { display: flex; align-items: center; gap: 4px; } +.research-progress .research-usage p, .research-progress .research-source-detail p { font-size: var(--text-body); margin-block: 8px; } +.research-progress .research-usage-scope { margin-block: 4px 20px; } +.research-usage-tail { border-inline-start: 2px solid var(--pw-caution); padding-inline-start: 16px; margin-block: 20px; } +.research-usage-tail strong { font-weight: 500; color: var(--pw-text); } +.research-analysis-cost { border-block-start: 1px solid var(--pw-line); padding-block-start: 16px; margin-block-start: 16px; } +.research-usage-detail > summary, .research-source-detail > summary { font-size: var(--text-body); } +.research-usage-basis code { display: block; margin-block-start: 4px; } +.research-source-detail { border-block-start: 1px solid var(--pw-line); margin-block-start: 16px; } +.research-progress-basis summary:focus-visible { outline: 2px solid var(--pw-blue); outline-offset: 2px; } .research-progress-warning { border-inline-start: 2px solid var(--pw-caution); padding-inline-start: 12px; } @media (max-width: 1200px) { .research-effect-metrics { grid-template-columns: repeat(3, minmax(0, 1fr)); } } @media (max-width: 640px) { @@ -89,7 +101,7 @@ .research-effect-metrics { grid-template-columns: repeat(2, minmax(0, 1fr)); gap: 8px 16px; } .research-assessment > summary { gap: 8px; } .research-native-values { grid-template-columns: repeat(2, minmax(0, 1fr)); } - .research-progress-values { grid-template-columns: 1fr; } - .research-progress-basis > dl, .research-usage-basis { grid-template-columns: 1fr; gap: 4px; } + .research-progress-values { grid-template-columns: 1fr; gap: 16px; } + .research-usage-basis { grid-template-columns: 1fr; gap: 4px; } .research-progress-basis dd, .research-usage-basis dd { margin-bottom: 8px; } } diff --git a/apps/presentation/dashboard/src/features/personal-workspace/goal-research-progress.tsx b/apps/presentation/dashboard/src/features/personal-workspace/goal-research-progress.tsx index fa171e9962..582f503e92 100644 --- a/apps/presentation/dashboard/src/features/personal-workspace/goal-research-progress.tsx +++ b/apps/presentation/dashboard/src/features/personal-workspace/goal-research-progress.tsx @@ -85,6 +85,7 @@ function NativeResults({native, zh, onCitation}: {native: ResearchProgressPack[" export function ResearchProgress({pack: p, packs, selectedId, zh, onSelect, onCitation}: {pack: ResearchProgressPack | null; packs: Pick[]; selectedId: string; zh: boolean; onSelect: (id: string) => void; onCitation: (citation: AssessmentCitation, origin?: HTMLButtonElement) => void}) { const measured = p?.solver_usage, usage = measured?.projection, result = p?.result; + const incomplete = usage?.incomplete_turn; const number = (n: number) => new Intl.NumberFormat(zh ? "zh-CN" : "en", {maximumFractionDigits: 1}).format(n); const percent = (n: number) => new Intl.NumberFormat(zh ? "zh-CN" : "en", {style: "percent", maximumFractionDigits: 0}).format(n); const duration = (s: number) => `${number(s / 60)} ${zh ? "分钟" : "min"}`; @@ -146,35 +147,51 @@ export function ResearchProgress({pack: p, packs, selectedId, zh, onSelect, onCi })} :

{zh ? "所选范围尚未分析" : "No analysis for the selected scope"}

} -
{zh ? "来源与用量" : "Sources and usage"}{p.native_outcomes ? "" : (zh ? " · 原生测量未采集" : " · Native measurements uncaptured")}{zh ? " · 完整成本未知" : " · Total cost unknown"}
-
{zh ? "题目 / 运行" : "Case / run"}
{p.scope.case_id}
{p.scope.run_id}
{zh ? "证据窗口" : "Evidence window"}
{p.scope.window}
{zh ? "取样规则" : "Selection rule"}
{p.selection.rule}
-
{zh ? "总体研究片段数" : "Population of episodes"}
{p.selection.population_episodes == null ? (zh ? "未知" : "Unknown") : number(p.selection.population_episodes)}
- {result ? <>
{zh ? "分析方法" : "Analysis method"}
{result.method_version} · {zh ? "Agent 判读未校准" : "Uncalibrated Agent judgment"}
{zh ? "结果标识" : "Result key"}
{result.analysis_key}
{zh ? "分析限制" : "Limitations"}
{result.limitations.map((line, i) =>

{line}

)}
: null} -
{zh ? "来源版本" : "Source revision"}
{p.scope.revision}
-
- {p.native_outcomes ? : null} -

{zh ? "用量与成本" : "Usage and cost"}{measured?.applicability === "historical" ? (zh ? " · 历史捕获" : " · Historical capture") : ""}

-

{zh ? "只读捕获，窗口可能与以上判读片段及下方 Explore 历史不同。" : "Read-only capture; its window can differ from the judgments above and Explore history below."}

-
-
{zh ? "分析尝试开销" : "Analysis attempt cost"}
{p.job?.usage ? <> - {zh ? "第" : "Attempt"} {p.job.attempt ?? "—"} · {zh ? "输入" : "input"} {p.job.usage.input_tokens == null ? "—" : number(p.job.usage.input_tokens)} · {zh ? "输出" : "output"} {p.job.usage.output_tokens == null ? "—" : number(p.job.usage.output_tokens)} tokens - {p.job.config ? ` · ${p.job.config.model}` : ""} - {p.job.analysis_wall_seconds != null ? ` · ${duration(p.job.analysis_wall_seconds)}` : ""} - : (zh ? "— 本次分析尝试未测得，不记零" : "— No measurement for this analysis attempt; not zero")}
-
{zh ? "合计边界" : "Aggregation boundary"}
{zh ? "分析仅显示当前回执的本次尝试；不含更早失败、恢复、评测或操作者开销。不与求解捕获盲目相加，金额及完整研究成本仍未知。" : "Analysis is this attempt's receipt only; earlier failures, recovery, evaluation and operator work are excluded. It is not blindly added to solver capture. Monetary and complete study costs remain unknown."}
-
-
-
{zh ? "捕获窗口跨度" : "Captured elapsed time"}
{usage ? duration(usage.elapsed_seconds) : "—"}
{usage ? `${number(usage.completed_turns)} ${zh ? "轮完整记录，不是全程总成本" : "complete Turns, not total study cost"}` : (zh ? "尚无完整用量捕获" : "No complete usage capture")}
-
{zh ? "未缓存输入" : "Uncached input"}
{usage ? number(usage.uncached_input_tokens) : "—"}
{usage ? "tokens" : (zh ? "未测，不记零" : "Unmeasured, not zero")}
-
{zh ? "输出" : "Output"}
{usage ? number(usage.output_tokens) : "—"}
{usage ? (zh ? "tokens，已含推理" : "tokens, includes reasoning") : (zh ? "未测，不记零" : "Unmeasured, not zero")}
-
- {usage && measured ? <>

— UTC

- {measured.applicability === "historical" ?

{zh ? "用量来源已变化或不可读；保留历史捕获值，当前用量未核验。" : "The usage source changed or is unavailable. Historical values are retained; current usage is unverified."}

: null} -
{zh ? "轮内时间合计" : "Time within Turns"}
{duration(usage.turn_elapsed_seconds)} · {zh ? "含工具和编排；不等于CPU、生成时间或可节省时间。" : "Includes tools and orchestration; not CPU, generation or avoidable time."}
-
{zh ? "输入总量" : "Total input"}
{number(usage.input_tokens)} tokens · {zh ? "其中缓存" : "including cached"} {number(usage.cached_input_tokens)}
{zh ? "用量来源" : "Usage source"}
{measured.artifact}
{measured.artifact_sha256}
{usage.method}
-
{zh ? "覆盖边界" : "Coverage limits"}
{zh ? "仅本次完整捕获；继承前缀、设置、原生评测、操作者及后续未完成轮不包含。金额与全研究总成本未知。" : "This complete capture only. Inherited prefixes, setup, native evaluation, operator work and later incomplete Turns are excluded. Dollar and total study costs are unknown."}
-
: null} -
+
{zh ? "来源与用量" : "Sources and usage"}{p.native_outcomes ? "" : (zh ? " · 原生测量未采集" : " · Native measurements uncaptured")}{zh ? " · 完整成本未知" : " · Total cost unknown"} +
+

{zh ? "求解用量" : "Solver usage"}

+

{zh ? "只读捕获，窗口可能与判读片段及 Explore 历史不同。金额与完整研究成本未测得，不能从 token 数推算。" : "Read-only capture; its window can differ from the judgments and Explore history. Monetary and complete study costs are unmeasured, not inferred from token counts."}

+ {usage ?

{usage.method === "codex_observed_session_counters_v1" ? (zh ? "仅已观测计数；未记录的请求、后续用量、继承前缀、设置、评测和操作者成本未知。" : "Observed counters only. Unrecorded requests, later usage, inherited prefixes, setup, evaluation and operator costs remain unknown.") : (zh ? "仅本次完整捕获；继承前缀、设置、原生评测、操作者及后续未完成轮不包含。" : "This complete capture only; inherited prefixes, setup, native evaluation, operator work and later incomplete Turns are excluded.")}

: null} +
+ {measured?.applicability === "historical" ?

{zh ? "历史捕获：用量来源已变化或不可读，当前用量未核验。" : "Historical capture: the usage source changed or is unavailable; current usage is unverified."}

: null} + {usage ?

{number(usage.completed_turns)} {zh ? "轮完整记录" : "complete Turns"}{incomplete ? (zh ? " · 末轮未完成" : " · final Turn incomplete") : ""}

:

{zh ? "尚无用量捕获 · 未测，不记零" : "No usage capture · unmeasured, not zero"}

} +
+
{zh ? "捕获窗口跨度" : "Captured elapsed time"}
{usage ? duration(usage.elapsed_seconds) : "—"}
+
{zh ? "未缓存输入" : "Uncached input"}
{usage ? number(usage.uncached_input_tokens) : "—"} tokens
+
{zh ? "输出（含推理）" : "Output (includes reasoning)"}
{usage ? number(usage.output_tokens) : "—"} tokens
+
+ {incomplete ?
+

{incomplete.usage ? <>{zh ? "其中末轮" : "Of which, unfinished Turn"} · {zh ? "输入" : "input"} {number(incomplete.usage.input_tokens)} · {zh ? "输出" : "output"} {number(incomplete.usage.output_tokens)} tokens : (zh ? "末轮尚无计数观测，不记零" : "No counter observation for the unfinished Turn; not zero")}

+

{zh ? "仅含已持久化用量，不能据此判定运行已结束。" : "Persisted usage only; this does not establish termination."}

+
: null} + {usage && measured ?
{zh ? "计数明细与来源" : "Counter details and source"} +
+
{zh ? "捕获时间（UTC）" : "Capture window (UTC)"}
—
+
{zh ? "轮内时间合计" : "Time within Turns"}
{duration(usage.turn_elapsed_seconds)}

{zh ? "含工具和编排；不等于 CPU、生成时间或可节省时间。" : "Includes tools and orchestration; not CPU, generation or avoidable time."}

+
{zh ? "输入总量" : "Total input"}
{number(usage.input_tokens)} tokens · {zh ? "其中缓存" : "including cached"} {number(usage.cached_input_tokens)}
+
{zh ? "用量来源" : "Usage source"}
{measured.artifact}{measured.artifact_sha256}
+
{zh ? "计数方法" : "Counter method"}
{usage.method}
+
+
: null} +
+

{zh ? "分析尝试" : "Analysis attempt"}

+

{zh ? "仅当前回执的本次尝试，不含更早失败、恢复、评测或操作者开销。不与求解捕获相加。" : "This attempt's receipt only; earlier failures, recovery, evaluation and operator work are excluded. Not added to solver capture."}

+
+

{p.job?.usage ? <>{zh ? "第" : "Attempt"} {p.job.attempt ?? "—"} · {zh ? "输入" : "input"} {p.job.usage.input_tokens == null ? "—" : number(p.job.usage.input_tokens)} · {zh ? "输出" : "output"} {p.job.usage.output_tokens == null ? "—" : number(p.job.usage.output_tokens)} tokens + {p.job.analysis_wall_seconds != null ? ` · ${duration(p.job.analysis_wall_seconds)}` : ""} + : (zh ? "未测得，不记零" : "Unmeasured, not zero")}

+
+
+ {p.native_outcomes ? : null} +
{zh ? "分析范围与来源" : "Analysis scope and sources"}
+
{zh ? "题目 / 运行" : "Case / run"}
{p.scope.case_id}
{p.scope.run_id}
+
{zh ? "证据窗口" : "Evidence window"}
{p.scope.window}
{zh ? "取样规则" : "Selection rule"}
{p.selection.rule}
+
{zh ? "总体研究片段数" : "Population of episodes"}
{p.selection.population_episodes == null ? (zh ? "未知" : "Unknown") : number(p.selection.population_episodes)}
+ {result ? <>
{zh ? "分析方法" : "Analysis method"}
{result.method_version} · {zh ? "Agent 判读未校准" : "Uncalibrated Agent judgment"}
{zh ? "结果标识" : "Result key"}
{result.analysis_key}
{zh ? "分析限制" : "Limitations"}
{result.limitations.map((line, i) =>

{line}

)}
: null} + {p.job?.config ? <>
{zh ? "分析模型" : "Analysis model"}
{p.job.config.model}
: null} +
{zh ? "来源版本" : "Source revision"}
{p.scope.revision}
+
+
: null} ; } diff --git a/docs/reference/trajectory-assessment.md b/docs/reference/trajectory-assessment.md index 4547a86106..98f7ac5308 100644 --- a/docs/reference/trajectory-assessment.md +++ b/docs/reference/trajectory-assessment.md @@ -529,3 +529,39 @@ checkpoint reads are qualified separately from this presentation example. ![Goal overview with registered trajectory analysis](../development/assets/research-analysis/first-screen.png) ![Scoped interpretation and usage reader](../development/assets/research-analysis/summary.png) + + +### Observe an unfinished final Turn + +For a captured run that stopped mid-Turn, the curator may explicitly add +`--include-incomplete-usage` to the same registration command, together with +`--usage-jsonl` and `--usage-session-id`. This reads persisted counters without +running a model or changing the solver. The default complete-session contract +above remains unchanged. + +The opt-in method is `codex_observed_session_counters_v1`. Its totals include +observed counters in the final unfinished Turn. `completed_turns` and `turns` +still contain only measured completions; `incomplete_turn` identifies the final +open interval, source lines, observation count and observed delta. With no +counter in that interval its `usage` is null, never an invented zero. With no +measured counters anywhere registration fails. Duplicate snapshots do not add +usage. Both methods reject wrong identity, forks, missing prefixes, gaps, resets, +unmatched completions and malformed JSONL. Cancellation without a completion +remains incomplete; recovery that starts another Turn without closing the prior +one remains unsupported. This opt-in is not a relaxed integrity qualification. + +The Research workbench's existing Sources and usage disclosure shows the +unfinished interval and its persisted input/output next to the capture totals, +with complete cost unknown. No extra owner confirmation or model run is needed +to read these facts. An unfinished interval ends at the last supported observed +event, not at an inferred process death. Observed time includes tools and waits; +unpersisted requests, later activity, model prices, evaluator costs and operator +attention remain unknown. Do not sum overlapping captures or repeated pack +registrations: the pack is a versioned diagnostic, not a billing ledger. + +The original source hash and historical/unavailable handling also apply to this +mode. Select a capture-free or default complete-session pack to stop using it; +old observations stay readable. The builtin benchmark-toolkit provider and its +TypeScript reducer retain ownership, with Python restricted to scoped file +reading and transport. There is no new telemetry, accounting store or scoring +policy. diff --git a/examples/personal-workspace-browser/evidence/observed-usage/after-default-desktop.png b/examples/personal-workspace-browser/evidence/observed-usage/after-default-desktop.png new file mode 100644 index 0000000000..9aba474834 Binary files /dev/null and b/examples/personal-workspace-browser/evidence/observed-usage/after-default-desktop.png differ diff --git a/examples/personal-workspace-browser/evidence/observed-usage/after-historical-desktop.png b/examples/personal-workspace-browser/evidence/observed-usage/after-historical-desktop.png new file mode 100644 index 0000000000..5bcdd99539 Binary files /dev/null and b/examples/personal-workspace-browser/evidence/observed-usage/after-historical-desktop.png differ diff --git a/examples/personal-workspace-browser/evidence/observed-usage/after-usage-desktop.png b/examples/personal-workspace-browser/evidence/observed-usage/after-usage-desktop.png new file mode 100644 index 0000000000..1fc19b2096 Binary files /dev/null and b/examples/personal-workspace-browser/evidence/observed-usage/after-usage-desktop.png differ diff --git a/examples/personal-workspace-browser/evidence/observed-usage/after-usage-narrow.png b/examples/personal-workspace-browser/evidence/observed-usage/after-usage-narrow.png new file mode 100644 index 0000000000..d77879b8df Binary files /dev/null and b/examples/personal-workspace-browser/evidence/observed-usage/after-usage-narrow.png differ diff --git a/examples/personal-workspace-browser/evidence/observed-usage/before-usage-desktop.png b/examples/personal-workspace-browser/evidence/observed-usage/before-usage-desktop.png new file mode 100644 index 0000000000..6c269e7769 Binary files /dev/null and b/examples/personal-workspace-browser/evidence/observed-usage/before-usage-desktop.png differ diff --git a/examples/personal-workspace-browser/evidence/usage-readability/after-default-desktop.png b/examples/personal-workspace-browser/evidence/usage-readability/after-default-desktop.png new file mode 100644 index 0000000000..79e13b6ed9 Binary files /dev/null and b/examples/personal-workspace-browser/evidence/usage-readability/after-default-desktop.png differ diff --git a/examples/personal-workspace-browser/evidence/usage-readability/after-default-narrow.png b/examples/personal-workspace-browser/evidence/usage-readability/after-default-narrow.png new file mode 100644 index 0000000000..c3f98fb262 Binary files /dev/null and b/examples/personal-workspace-browser/evidence/usage-readability/after-default-narrow.png differ diff --git a/examples/personal-workspace-browser/evidence/usage-readability/after-expanded-desktop.png b/examples/personal-workspace-browser/evidence/usage-readability/after-expanded-desktop.png new file mode 100644 index 0000000000..e638fa3910 Binary files /dev/null and b/examples/personal-workspace-browser/evidence/usage-readability/after-expanded-desktop.png differ diff --git a/examples/personal-workspace-browser/evidence/usage-readability/after-expanded-narrow.png b/examples/personal-workspace-browser/evidence/usage-readability/after-expanded-narrow.png new file mode 100644 index 0000000000..badefad23a Binary files /dev/null and b/examples/personal-workspace-browser/evidence/usage-readability/after-expanded-narrow.png differ diff --git a/examples/personal-workspace-browser/evidence/usage-readability/after-help-narrow.png b/examples/personal-workspace-browser/evidence/usage-readability/after-help-narrow.png new file mode 100644 index 0000000000..c2097570fb Binary files /dev/null and b/examples/personal-workspace-browser/evidence/usage-readability/after-help-narrow.png differ diff --git a/examples/personal-workspace-browser/evidence/usage-readability/after-historical-desktop.png b/examples/personal-workspace-browser/evidence/usage-readability/after-historical-desktop.png new file mode 100644 index 0000000000..c8d3d5d0f3 Binary files /dev/null and b/examples/personal-workspace-browser/evidence/usage-readability/after-historical-desktop.png differ diff --git a/examples/personal-workspace-browser/evidence/usage-readability/after-missing-narrow.png b/examples/personal-workspace-browser/evidence/usage-readability/after-missing-narrow.png new file mode 100644 index 0000000000..86a7c44a85 Binary files /dev/null and b/examples/personal-workspace-browser/evidence/usage-readability/after-missing-narrow.png differ diff --git a/examples/personal-workspace-browser/evidence/usage-readability/after-populated-default-desktop.png b/examples/personal-workspace-browser/evidence/usage-readability/after-populated-default-desktop.png new file mode 100644 index 0000000000..c9ae4f0257 Binary files /dev/null and b/examples/personal-workspace-browser/evidence/usage-readability/after-populated-default-desktop.png differ diff --git a/examples/personal-workspace-browser/evidence/usage-readability/after-populated-source-narrow.png b/examples/personal-workspace-browser/evidence/usage-readability/after-populated-source-narrow.png new file mode 100644 index 0000000000..00896647a4 Binary files /dev/null and b/examples/personal-workspace-browser/evidence/usage-readability/after-populated-source-narrow.png differ diff --git a/examples/personal-workspace-browser/evidence/usage-readability/after-populated-usage-desktop.png b/examples/personal-workspace-browser/evidence/usage-readability/after-populated-usage-desktop.png new file mode 100644 index 0000000000..0e5a9dcbe5 Binary files /dev/null and b/examples/personal-workspace-browser/evidence/usage-readability/after-populated-usage-desktop.png differ diff --git a/examples/personal-workspace-browser/evidence/usage-readability/before-expanded-desktop.png b/examples/personal-workspace-browser/evidence/usage-readability/before-expanded-desktop.png new file mode 100644 index 0000000000..75ea5a1e09 Binary files /dev/null and b/examples/personal-workspace-browser/evidence/usage-readability/before-expanded-desktop.png differ diff --git a/examples/personal-workspace-browser/evidence/usage-readability/before-expanded-narrow.png b/examples/personal-workspace-browser/evidence/usage-readability/before-expanded-narrow.png new file mode 100644 index 0000000000..45ab28a2a7 Binary files /dev/null and b/examples/personal-workspace-browser/evidence/usage-readability/before-expanded-narrow.png differ diff --git a/examples/personal-workspace-browser/research-history.mjs b/examples/personal-workspace-browser/research-history.mjs index 475b23a0f6..a3fa35089e 100644 --- a/examples/personal-workspace-browser/research-history.mjs +++ b/examples/personal-workspace-browser/research-history.mjs @@ -61,6 +61,15 @@ function analysisReadback(selected = "a") { final_candidate_delta: selected === "d" ? null : selected === "b" ? -2 : 4, }; } + if (selected === "d") { + const usage = snapshot.pack.solver_usage.projection; + usage.method = "codex_observed_session_counters_v1"; + usage.incomplete_turn = {turn_id: "unfinished", start: usage.end, observed_through: usage.end, + first_line: 4, last_line: 5, counter_observations: 1, + usage: {input_tokens: 400, cached_input_tokens: 300, uncached_input_tokens: 100, output_tokens: 8}}; + usage.input_tokens += 400; usage.cached_input_tokens += 300; + usage.uncached_input_tokens += 100; usage.output_tokens += 8; + } if (selected === "d") {snapshot.pack.result = null; snapshot.pack.job = null; snapshot.pack.native_outcomes.projection.initial = null;} return snapshot; } @@ -609,10 +618,19 @@ Path(sys.argv[1]).write_text(json.dumps({'goal_id':goal,'display_name':'Syntheti await history.getByRole('button', {name: '关闭分析', exact: true}).click(); await page.setViewportSize({width: 1280, height: 900}); await progress.locator('.research-progress-basis > summary').click(); - assert.equal(await progress.getByText("100", {exact: true}).count(), 1, "Uncached input is shown without charging cache twice"); + assert.equal(await progress.locator(".research-progress-values dd").filter({hasText: /^100 tokens$/}).count(), 1, "Uncached input is shown without charging cache twice"); + // Actual observations precede provenance; details and help remain operable. + assert.equal(await progress.locator('.research-usage-detail').evaluate(e => e.open), false); + assert.doesNotMatch(await progress.innerText(), /codex_complete_session_counters_v1/); + await progress.locator('.research-usage-detail > summary').focus(); + await page.keyboard.press('Enter'); assert.match(await progress.innerText(), /1,?100.*1,?000/s); + assert.match(await progress.innerText(), /codex_complete_session_counters_v1/); + await progress.locator('.research-usage-detail > summary').press('Enter'); assert.match(await progress.innerText(), /30.*4.*tokens/s, "Analysis attempt cost is distinct from solver usage"); - assert.match(await progress.innerText(), /更早失败.*恢复.*评测/s); + await progress.getByRole('button', {name: '分析开销范围', exact: true}).focus(); + assert.match(await progress.getByRole('tooltip').innerText(), /更早失败.*恢复.*评测/s); + await page.keyboard.press('Escape'); await progress.locator('.research-progress-basis > summary').click(); for (const limit of [1, 0]) { episodeMetadataLimit = limit; @@ -751,7 +769,10 @@ Path(sys.argv[1]).write_text(json.dumps({'goal_id':goal,'display_name':'Syntheti await progress.locator(`.research-progress-scope[data-pack-id="${"c".repeat(64)}"]`).waitFor(); assert.match(await progress.locator('.research-progress-basis > summary').innerText(), /完整成本未知/); await progress.locator('.research-progress-basis > summary').click(); - assert.match(await progress.innerText(), /尚无完整用量捕获|No complete usage/); + assert.match(await progress.innerText(), /尚无用量捕获|No usage capture/); + await progress.getByRole('button', {name: '用量范围与成本', exact: true}).focus(); + assert.doesNotMatch(await progress.getByRole('tooltip').innerText(), /本次完整捕获|This complete capture/); + await page.keyboard.press('Escape'); await progress.locator('.research-progress-basis > summary').click(); assert.equal(await analysis.count(), 0, "Changing a factual scope preserves the closed analysis panel"); assert.match(await progress.innerText(), /不适用|N\/A/); @@ -768,6 +789,10 @@ Path(sys.argv[1]).write_text(json.dumps({'goal_id':goal,'display_name':'Syntheti await analysis.press('Escape'); await progress.getByRole('combobox').selectOption('d'.repeat(64)); await progress.locator(`.research-progress-scope[data-pack-id="${"d".repeat(64)}"]`).waitFor(); + await progress.locator('.research-progress-basis > summary').click(); + assert.match(await progress.innerText(), /末轮未完成|final Turn incomplete/); + assert.match(await progress.innerText(), /400.*8.*tokens/s, "Persisted unfinished usage is visible without declaring total cost"); + await progress.locator('.research-progress-basis > summary').click(); assert.match(await progress.innerText(), /所选范围尚未分析/); assert.equal(await progress.locator('.research-effect-metrics').count(), 0, 'An unanalysed cost pack never borrows another pack’s judgments'); assert.equal(await analysis.count(), 0, 'Choosing an unanalysed source never requests analysis'); diff --git a/loopx/capabilities/benchmark_toolkit/trajectory_assessment_store.py b/loopx/capabilities/benchmark_toolkit/trajectory_assessment_store.py index 7fff8e758d..56a00a422b 100644 --- a/loopx/capabilities/benchmark_toolkit/trajectory_assessment_store.py +++ b/loopx/capabilities/benchmark_toolkit/trajectory_assessment_store.py @@ -91,7 +91,9 @@ def _source_path(source_root: Path, artifact: str) -> Path: return path -def _usage_capture(source_root: Path, artifact: str, session_id: str) -> dict: +def _usage_capture( + source_root: Path, artifact: str, session_id: str, *, include_incomplete: bool = False +) -> dict: data = _source_path(source_root, artifact).read_bytes() events = [] # Transport only the provider's counter/lifecycle fields, not model text, @@ -141,7 +143,7 @@ def _usage_capture(source_root: Path, artifact: str, session_id: str) -> dict: ) projection = effect_runtime_result( "benchmark.trajectory.codex-usage", - {"events": events, "session_id": session_id}, + {"events": events, "session_id": session_id, "include_incomplete": include_incomplete}, large_local_snapshot=True, ) return { @@ -243,6 +245,7 @@ def register_pack( config: dict | None = None, usage_jsonl: str | None = None, usage_session_id: str | None = None, + include_incomplete_usage: bool = False, native_outcomes_json: str | None = None, base_pack_id: str | None = None, ) -> dict: @@ -274,13 +277,16 @@ def register_pack( **prepared, "previous": _prepared(previous), }) record["base_pack_id"] = _opaque(base_pack_id) + if include_incomplete_usage and usage_jsonl is None: + raise ValueError("include-incomplete-usage requires a usage capture") if (usage_jsonl is None) != (usage_session_id is None): raise ValueError( "Usage capture and expected session identity must be supplied together" ) if usage_jsonl is not None: record["solver_usage"] = _usage_capture( - source_root, usage_jsonl, usage_session_id + source_root, usage_jsonl, usage_session_id, + include_incomplete=include_incomplete_usage, ) if native_outcomes_json is not None: data = _source_path(source_root, native_outcomes_json).read_bytes() diff --git a/loopx/cli_commands/benchmark_trajectory.py b/loopx/cli_commands/benchmark_trajectory.py index b57db32675..1306573bbd 100644 --- a/loopx/cli_commands/benchmark_trajectory.py +++ b/loopx/cli_commands/benchmark_trajectory.py @@ -42,6 +42,10 @@ def register_benchmark_trajectory_commands(subparsers, add_format) -> None: "--usage-session-id", help="Expected captured solver session identity; required with --usage-jsonl.", ) + register.add_argument( + "--include-incomplete-usage", action="store_true", + help="Observe persisted counters through an unfinished final Turn; requires usage capture. Does not establish complete cost or solver termination.", + ) register.add_argument( "--native-outcomes-json", help="Optional provider-recorded native outcomes, relative to source-root; same pack scope, never analyzer scores.", @@ -92,6 +96,7 @@ def _handle_benchmark_trajectory_command( title=args.title, usage_jsonl=args.usage_jsonl, usage_session_id=args.usage_session_id, + include_incomplete_usage=args.include_incomplete_usage, native_outcomes_json=args.native_outcomes_json, base_pack_id=args.base_pack_id, ) diff --git a/loopx/control_plane/capabilities/trajectory_assessment_types.ts b/loopx/control_plane/capabilities/trajectory_assessment_types.ts index 519c1207f6..df11cd739a 100644 --- a/loopx/control_plane/capabilities/trajectory_assessment_types.ts +++ b/loopx/control_plane/capabilities/trajectory_assessment_types.ts @@ -46,7 +46,10 @@ export interface TrajectoryContinuation { /** Provider-observed facts, separate from inferred episode judgments. */ export interface TrajectorySolverUsage { - session_id: string; method: 'codex_complete_session_counters_v1'; + session_id: string; method: 'codex_complete_session_counters_v1' | 'codex_observed_session_counters_v1'; + incomplete_turn?: {turn_id: string; start: string; observed_through: string; first_line: number; last_line: number; + counter_observations: number; usage: {input_tokens: number; cached_input_tokens: number; + uncached_input_tokens: number; output_tokens: number} | null} | null; start: string; end: string; completed_turns: number; counter_observations: number; input_tokens: number; cached_input_tokens: number; uncached_input_tokens: number; output_tokens: number; elapsed_seconds: number; turn_elapsed_seconds: number; diff --git a/loopx/control_plane/capabilities/trajectory_codex_usage.ts b/loopx/control_plane/capabilities/trajectory_codex_usage.ts index 229b534ee1..6d1a64393a 100644 --- a/loopx/control_plane/capabilities/trajectory_codex_usage.ts +++ b/loopx/control_plane/capabilities/trajectory_codex_usage.ts @@ -19,7 +19,10 @@ const time = (v: unknown): number => { }; export function summarizeTrajectoryCodexUsage(params: Record): TrajectorySolverUsage { if (!Array.isArray(params.events) || typeof params.session_id !== 'string' || !params.session_id) return fail('session events and expected identity required'); + if (params.include_incomplete !== undefined && typeof params.include_incomplete !== 'boolean') return fail('include_incomplete must be boolean'); + const observed = params.include_incomplete === true; const rows = params.events; + let observedThrough = '', observedLine = 0; let sessionSeen = false, previousTime = -Infinity, previous = zero(), samples = 0; let active: {turn_id: string; start: string; start_ms: number; first_line: number; baseline: Counters; samples: number} | null = null; const turns: Array<{turn_id: string; start: string; end: string; first_line: number; last_line: number; elapsed_seconds: number; usage: Counters & {uncached_input_tokens: number}}> = []; @@ -39,6 +42,7 @@ export function summarizeTrajectoryCodexUsage(params: Record): const stamp = time(event.timestamp); if (stamp < previousTime) return fail('out-of-order events'); previousTime = stamp; + observedThrough = event.timestamp; observedLine = event.line; if (p.type === 'task_started') { if (active || typeof p.turn_id !== 'string' || !p.turn_id || turns.some(t => t.turn_id === p.turn_id)) return fail('duplicate or overlapping Turn'); active = {turn_id: p.turn_id, start: event.timestamp, start_ms: stamp, first_line: event.line, baseline: {...previous}, samples: 0}; @@ -62,14 +66,25 @@ export function summarizeTrajectoryCodexUsage(params: Record): active = null; } } - if (active || !turns.length) return fail('capture must end after a completed Turn'); - return {session_id: params.session_id, method: 'codex_complete_session_counters_v1', - start: turns[0].start, end: turns.at(-1)!.end, completed_turns: turns.length, counter_observations: samples, + if (!observed && (active || !turns.length)) return fail('capture must end after a completed Turn'); + if (!samples) return fail('no measured usage in captured session'); + const tail = active ? subtract(previous, active.baseline) : null; + if (tail && tail.cached_input_tokens > tail.input_tokens) return fail('invalid delta cache accounting'); + const incomplete = active ? {turn_id: active.turn_id, start: active.start, observed_through: observedThrough, + first_line: active.first_line, last_line: observedLine, counter_observations: active.samples, + usage: active.samples && tail ? {...tail, uncached_input_tokens: tail.input_tokens - tail.cached_input_tokens} : null} : null; + const start = turns[0]?.start ?? active!.start, end = active ? observedThrough : turns.at(-1)!.end; + return {session_id: params.session_id, method: observed ? 'codex_observed_session_counters_v1' : 'codex_complete_session_counters_v1', + ...(observed ? {incomplete_turn: incomplete} : {}), + start, end, completed_turns: turns.length, counter_observations: samples, input_tokens: previous.input_tokens, cached_input_tokens: previous.cached_input_tokens, uncached_input_tokens: previous.input_tokens - previous.cached_input_tokens, output_tokens: previous.output_tokens, - elapsed_seconds: (time(turns.at(-1)!.end) - time(turns[0].start)) / 1000, - turn_elapsed_seconds: turns.reduce((sum, t) => sum + t.elapsed_seconds, 0), turns, - limitations: ['Complete captured session only; donor/prefix study costs and evaluator costs are not included.', + elapsed_seconds: (time(end) - time(start)) / 1000, + turn_elapsed_seconds: turns.reduce((sum, t) => sum + t.elapsed_seconds, 0) + (active ? (time(end) - active.start_ms) / 1000 : 0), turns, + limitations: [observed + ? 'Observed counters only, including any unfinished final Turn; unpersisted requests and later usage remain unknown. No termination or complete cost is inferred.' + : 'Complete captured session only; donor/prefix study costs and evaluator costs are not included.', + ...(observed ? ['Donor/prefix study costs and evaluator costs are not included.'] : []), 'Input includes cached input. Output includes any reasoning tokens. No dollar or generation-time estimate.', 'Elapsed time includes tools and orchestration; it does not measure CPU time or avoidable overhead.', 'Registered operator binding identifies the case; this does not independently prove run provenance.']}; diff --git a/tests/capabilities/test_trajectory_assessment_store.py b/tests/capabilities/test_trajectory_assessment_store.py index 704fe68384..1fddb2d0d2 100644 --- a/tests/capabilities/test_trajectory_assessment_store.py +++ b/tests/capabilities/test_trajectory_assessment_store.py @@ -787,3 +787,67 @@ def cli(*args): usage_session_id="solver-session", ) assert len(discover(**scope)["packs"]) == 2 + + +def test_observed_usage_preserves_incomplete_cost_and_default_gate(pack): + from loopx.control_plane.effect_runtime import EffectRuntimeRejected + + scope, _, old, input_path, sources, _ = pack + args = dict( + **scope, + input_path=input_path, + source_root=sources, + title=old["title"], + usage_jsonl="usage.jsonl", + usage_session_id="solver-session", + ) + path = sources / "usage.jsonl" + path.write_text("\n".join(_usage_fixture().splitlines()[:-1]) + "\n") + before = path.read_bytes() + with pytest.raises(EffectRuntimeRejected, match="completed Turn"): + register_pack(**args) + registered = register_pack(**args, include_incomplete_usage=True) + assert registered["analysis_key"] == old["analysis_key"] + view = inspect_pack(**scope)["solver_usage"] + assert view["projection"]["method"] == "codex_observed_session_counters_v1" + assert view["projection"]["input_tokens"] == 140 + assert view["projection"]["incomplete_turn"]["usage"]["input_tokens"] == 40 + assert view["projection"]["completed_turns"] == 1 + assert register_pack(**args, include_incomplete_usage=True) == registered + assert path.read_bytes() == before + assert inspect_pack(**scope)["job"] is None + with pytest.raises(ValueError, match="requires a usage capture"): + register_pack( + **{**args, "usage_jsonl": None, "usage_session_id": None}, + include_incomplete_usage=True, + ) + path.write_bytes(before + b'{"broken":') + with pytest.raises(ValueError): + register_pack(**args, include_incomplete_usage=True) + assert inspect_pack(**scope)["solver_usage"]["applicability"] == "historical" + path.unlink() + assert inspect_pack(**scope)["solver_usage"]["reason_code"] == "source_unavailable" + + +def test_observed_usage_real_cli_without_model_dispatch(pack): + scope, _, old, input_path, sources, _ = pack + (sources / 'usage.jsonl').write_text('\n'.join(_usage_fixture().splitlines()[:-1]) + '\n') + registry = sources.parent / 'registry-observed.json' + registry.write_text('{"projects":{}}') + prefix = [sys.executable, '-m', 'loopx.entrypoint', '--registry', str(registry), + '--runtime-root', str(scope['runtime_root']), '--format', 'json', 'benchmark'] + args = ['trajectory-pack-register', '--goal-id', scope['owner_goal_id'], + '--input-json', str(input_path), '--source-root', str(sources), '--title', old['title'], + '--usage-jsonl', 'usage.jsonl', '--usage-session-id', 'solver-session'] + default = subprocess.run([*prefix, *args], text=True, capture_output=True) + assert default.returncode != 0 and 'completed Turn' in default.stdout + default.stderr + observed = subprocess.run([*prefix, *args, '--include-incomplete-usage'], text=True, capture_output=True) + assert observed.returncode == 0, observed.stdout + observed.stderr + record = json.loads(observed.stdout) + discovery = subprocess.run([*prefix, 'trajectory-packs', '--goal-id', scope['owner_goal_id']], text=True, capture_output=True) + assert discovery.returncode == 0 + view = next(x for x in json.loads(discovery.stdout)['packs'] if x['pack_id'] == record['pack_id']) + assert view['solver_usage']['projection']['input_tokens'] == 140 + assert view['solver_usage']['projection']['incomplete_turn']['usage']['input_tokens'] == 40 + assert inspect_pack(**scope, pack_id=record['pack_id'])['job'] is None + assert not list(scope['runtime_root'].rglob('receipt.json')) diff --git a/tests/control_plane_ts/trajectory_codex_usage.test.ts b/tests/control_plane_ts/trajectory_codex_usage.test.ts index 6d7c8ec395..b856de1b64 100644 --- a/tests/control_plane_ts/trajectory_codex_usage.test.ts +++ b/tests/control_plane_ts/trajectory_codex_usage.test.ts @@ -38,3 +38,57 @@ test('unknown counters, resets, missing prefix, session mismatch and unfinished (e:any[])=>{e.splice(5,1);}, ]) {const es=fixture();edit(es);assert.throws(()=>run(es));} }); + +test('opt-in observation retains the unfinished tail without declaring completion', () => { + const events = fixture().slice(0, -1); + assert.throws(() => run(events), /completed Turn/); + const result = summarize({events, session_id: 'session', include_incomplete: true}); + assert.equal(result.method, 'codex_observed_session_counters_v1'); + assert.equal(result.input_tokens, 140); + assert.equal(result.cached_input_tokens, 80); + assert.equal(result.output_tokens, 15); + assert.equal(result.completed_turns, 1); + assert.equal(result.turns.length, 1); + assert.deepEqual(result.incomplete_turn?.usage, {...usage(40,20,5), uncached_input_tokens:20}); + assert.equal(result.incomplete_turn?.last_line, 6); + assert.equal(result.end, '2026-01-01T00:00:05Z'); + assert.equal(result.elapsed_seconds, 4); + assert.equal(result.turn_elapsed_seconds, 3); + // A cancellation is not a task_complete and supplies no extra counter. + events.push({line:7,type:'event_msg',timestamp:'2026-01-01T00:00:06Z',payload:{type:'turn_aborted'}}); + assert.deepEqual(summarize({events,session_id:'session',include_incomplete:true}),result); +}); + +test('first unfinished Turn works, unmeasured tails stay unknown, repeated snapshots do not add cost', () => { + const onlyFirst = fixture().slice(0,3); + const partial = (events:any[]) => summarize({events, session_id:'session',include_incomplete:true}); + assert.equal(partial(onlyFirst).completed_turns, 0); + assert.equal(partial(onlyFirst).input_tokens, 100); + const unmeasured = partial(fixture().slice(0,5)); + assert.equal(unmeasured.input_tokens, 100); + assert.equal(unmeasured.incomplete_turn?.usage, null); + const duplicate = fixture().slice(0,-1); duplicate.push({...structuredClone(duplicate[5]),line:7}); + assert.equal(partial(duplicate).input_tokens, 140); + assert.deepEqual(partial(duplicate).incomplete_turn?.usage, {...usage(40,20,5),uncached_input_tokens:20}); + assert.throws(()=>partial(fixture().slice(0,2)), /no measured/); + for (const mutate of [ + (e:any[])=>{e[0].payload.id='different-session';}, + (e:any[])=>{e[0].payload.forked_from_id='parent';}, + (e:any[])=>{e[2].payload.info.last_token_usage.input_tokens=99;}, + (e:any[])=>{e[5].payload.info.total_token_usage.input_tokens=50;}, + (e:any[])=>{e[5].payload.info.last_token_usage.output_tokens=7;}, + (e:any[])=>{delete e[5].payload.info.total_token_usage.cached_input_tokens;}, + (e:any[])=>{e[4].payload.turn_id='a';}, + (e:any[])=>{e[5].payload.info.total_token_usage.cached_input_tokens=120;e[5].payload.info.last_token_usage.cached_input_tokens=60;}, + ]) {const e=fixture().slice(0,-1);mutate(e);assert.throws(()=>partial(e));} +}); + +test('default complete shape remains unchanged; observed capture is separately identified', () => { + const complete=run(); assert.equal('incomplete_turn' in complete,false); + const observed=summarize({events:fixture(),session_id:'session',include_incomplete:true}); + assert.equal(observed.incomplete_turn,null); + const {method,incomplete_turn,limitations,...facts}=observed; + const {method:oldMethod,limitations:oldLimits,...oldFacts}=complete; + assert.deepEqual(facts,oldFacts); + assert.throws(()=>summarize({events:fixture(),session_id:'session',include_incomplete:'true'}),/boolean/); +});