From e00bcbc208441209a968d9bc9be74be7ae1fbe0e Mon Sep 17 00:00:00 2001 From: Saksham Uboweja <76595406+SakshamUboweja@users.noreply.github.com> Date: Sun, 8 Nov 2026 05:01:47 +0000 Subject: [PATCH] Remove code nothing uses Deletes about 500 lines that no part of the project calls anymore: leftover helpers, an unused script, and a feature that could never be turned on. Nothing changes for users, and all tests pass. --- experiments/scripts/summarize_stream.py | 255 ------------------------ experiments/scripts/sweep_plan.py | 6 - experiments/te_bench/codex_spiral.py | 140 +------------ experiments/te_bench/runner.py | 5 - experiments/te_bench/workloads.py | 6 - experiments/tests/test_spiral.py | 4 +- experiments/tests/test_sweep_arms.py | 1 - experiments/tests/test_workloads.py | 1 - tests/test_ledger.py | 14 +- tokeneconomizer/ledger.py | 33 --- tokeneconomizer/microskills/observe.py | 8 - tokeneconomizer/pricing.py | 3 - tokeneconomizer/toolprune/codex.py | 37 ---- 13 files changed, 5 insertions(+), 508 deletions(-) delete mode 100644 experiments/scripts/summarize_stream.py diff --git a/experiments/scripts/summarize_stream.py b/experiments/scripts/summarize_stream.py deleted file mode 100644 index 761b4f8..0000000 --- a/experiments/scripts/summarize_stream.py +++ /dev/null @@ -1,255 +0,0 @@ -#!/usr/bin/env python3 -""" -Amortized-cost aggregation for stream runs. - -A stream run (stream_run.py) writes one JSONL row per task to -results//_streams/____.jsonl. Each row records the task's -EXECUTION cost (`tokens_total`, what the agent spent solving the task — this -already includes the injected manifest/notes/carryover-log it read) and, kept in -a SEPARATE field, the per-task OPTIMIZER overhead (`optimizer_tokens`, the real -token cost of the gated/ungated note-writing call; 0 when the gate didn't fire). - -The HONEST amortized cost of a mode is the sum of BOTH: - - task_cost(t) = tokens_total(t) + optimizer_tokens(t) - amortized(T) = (1/T) * sum_{t<=T} task_cost(t) - -Reporting only `tokens_total` flatters `full`/`self-memory` (which pay optimizer -overhead) against `standalone`/`carryover` (which don't). This module folds the two -together so every mode is compared on total tokens actually spent. - -Usage: - python summarize_stream.py results/v1/_streams/*.jsonl # compare modes - python summarize_stream.py --curve run.jsonl # per-position curve - -Cost is reported CACHE-AWARE by default: per-task tokens are reweighted into -fresh-input-token equivalents (pricing.py) because `tokens_total` counts cached -re-reads (~90% of tokens) at full weight, which biases A/Bs. Pass --raw to report -the legacy full-weight `tokens_total` instead. -""" -import argparse, glob, json, os, sys - -# Import the vendored pricing table + the honest ledger from the tokeneconomizer package. -_EXPERIMENTS = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) -sys.path[:0] = [_EXPERIMENTS, os.path.dirname(_EXPERIMENTS)] # te_harness; the core at the root -from tokeneconomizer import ledger -from tokeneconomizer import pricing - - -def _num(x): - return x if isinstance(x, (int, float)) else 0 - - -_RUNDIR_CACHE = {} - - -def _row_breakdown(r, stream_path): - """Return the per-task token breakdown for cache-aware costing. Prefer the fields - on the stream row itself (new runs carry them); fall back to the per-task row.json - in the run dir (older streams predate the inline fields); else return the row as-is - (effective_tokens then falls back to tokens_total).""" - if pricing.has_breakdown(r): - return r - run_dir = r.get("run_dir") - if run_dir: - # stream file lives at /_streams/.jsonl -> run dirs are under - root = os.path.dirname(os.path.dirname(os.path.abspath(stream_path))) - rj = os.path.join(root, run_dir, "row.json") - if rj in _RUNDIR_CACHE: - return _RUNDIR_CACHE[rj] or r - loaded = None - if os.path.isfile(rj): - try: - loaded = json.load(open(rj)) - except (OSError, json.JSONDecodeError): - loaded = None - _RUNDIR_CACHE[rj] = loaded - if loaded is not None: - return loaded - return r - - -def summarize(path, raw=False, exclude_init=False): - """Read one stream JSONL and return a summary dict with per-position curves and - cumulative totals. `task_cost = exec_cost + optimizer_tokens`. By default exec_cost - is the CACHE-AWARE effective cost (pricing.effective_tokens); with raw=True it is the - legacy full-weight `tokens_total`. Optimizer tokens are counted at face value (they - are one-shot, largely uncached calls and small relative to exec). - - `exclude_init` drops the Web-Bench `init` scaffold row (instance == "init") before any - aggregation, so every project's amortized cost normalizes over the SAME 20 graded tasks - (calculator/table ship an `init` record; react/expressjs/zustand do not). Default False keeps - SWE-bench byte-identical -- SWE-bench streams never contain an `init` row.""" - rows = [] - with open(path) as f: - for line in f: - line = line.strip() - if line: - try: - rows.append(json.loads(line)) - except json.JSONDecodeError: - pass - if exclude_init: - rows = [r for r in rows if r.get("instance") != "init"] - rows.sort(key=lambda r: _num(r.get("position"))) - - curve, cum_exec, cum_opt, cum_resolved = [], 0, 0, 0 - cum_raw_exec = 0 - n_graded = 0 - n_with_breakdown = 0 - n_mech_effective = 0 # rows whose mechanism tokens are cache-aware effective - n_mech_raw = 0 # legacy rows with raw (face-value) mechanism tokens present - for r in rows: - rec = _row_breakdown(r, path) - raw_exec = _num(r.get("tokens_total")) - eff_exec = pricing.effective_tokens(rec) - if pricing.has_breakdown(rec): - n_with_breakdown += 1 - exec_t = raw_exec if raw else eff_exec - # OPTIMIZER + MECHANISM-INTERNAL overhead. optimizer_tokens is the note-writing - # call; untool_classify_tokens is the one-shot tool-classifier LLM call; - # untool_judge_tokens is the per-task instruction-aware tool judge; spiral_judge_tokens - # is the paid live trace-judge; spiral_nudge_writer_tokens is the v6.1 targeted-nudge - # writer. All are real mechanism-internal LLM cost that must be amortized into the - # honest task cost. The overhead-key set is the SINGLE definition in - # tokeneconomizer.ledger.OVERHEAD_KEYS, shared with the emit path so the two never - # drift; every key above, spiral_nudge_writer_tokens included, is summed. - # BASIS: rows tagged mech_tokens_basis=="effective" carry cache-aware effective - # tokens (same basis as exec) — sum directly. Legacy rows (no tag) carry RAW - # face-value tokens: an UNDER-count of effective (mechanism calls are uncached - # one-shots, so effective = input + 5*output >= raw). We sum them as a documented - # floor and surface the mixed basis in the summary rather than silently blending. - opt_t = ledger.overhead(r) - mech_basis = r.get("mech_tokens_basis") # "effective" for new rows, None legacy - if opt_t > 0: - if mech_basis == "effective": - n_mech_effective += 1 - else: - n_mech_raw += 1 - cum_exec += exec_t - cum_raw_exec += raw_exec - cum_opt += opt_t - resolved = r.get("resolved") - if resolved is not None: - n_graded += 1 - if resolved: - cum_resolved += 1 - k = len(curve) + 1 - curve.append({ - "position": r.get("position"), - "instance": r.get("instance"), - "exec_tokens": round(exec_t, 1), - "optimizer_tokens": opt_t, - "task_cost": round(exec_t + opt_t, 1), - "cum_exec_tokens": round(cum_exec, 1), - "cum_optimizer_tokens": cum_opt, - "cum_total_tokens": round(cum_exec + cum_opt, 1), - "amortized_cost_per_task": round((cum_exec + cum_opt) / k, 1), - "resolved": resolved, - }) - - n = len(rows) - mode = rows[0].get("mode") if rows else None - cum_total = cum_exec + cum_opt - return { - "path": path, - "mode": mode, - "cost_basis": "tokens_total (raw)" if raw else "cache-aware effective", - "n_tasks": n, - "n_graded": n_graded, - "n_with_breakdown": n_with_breakdown, - # mechanism-token basis audit: how many costed rows carry effective vs legacy-raw - # mechanism tokens. "mixed"/"raw" is a flag that the overhead is an under-count. - "mech_tokens_basis": ("effective" if n_mech_raw == 0 and n_mech_effective > 0 - else "raw" if n_mech_effective == 0 and n_mech_raw > 0 - else "mixed" if n_mech_effective and n_mech_raw - else "none"), - "n_mech_effective": n_mech_effective, - "n_mech_raw": n_mech_raw, - "resolved": cum_resolved, - "resolve_rate": round(cum_resolved / n_graded, 3) if n_graded else None, - "cum_exec_tokens": round(cum_exec, 1), - "cum_raw_exec_tokens": cum_raw_exec, - "cum_optimizer_tokens": cum_opt, - "cum_total_tokens": round(cum_total, 1), - "optimizer_overhead_frac": round(cum_opt / cum_total, 4) if cum_total else 0.0, - "amortized_cost_per_task": round(cum_total / n, 1) if n else 0.0, - "exec_cost_per_task": round(cum_exec / n, 1) if n else 0.0, - "raw_exec_cost_per_task": round(cum_raw_exec / n, 1) if n else 0.0, - "curve": curve, - } - - -def print_curve(s): - print(f"\n# {s['mode']} ({s['n_tasks']} tasks) {os.path.basename(s['path'])}") - print(f"{'pos':>4} {'instance':<24} {'exec':>9} {'opt':>8} {'task':>9} " - f"{'cum_total':>11} {'amort/task':>11} {'resolved':>8}") - for c in s["curve"]: - print(f"{str(c['position']):>4} {str(c['instance'])[:24]:<24} " - f"{c['exec_tokens']:>9} {c['optimizer_tokens']:>8} {c['task_cost']:>9} " - f"{c['cum_total_tokens']:>11} {c['amortized_cost_per_task']:>11} " - f"{str(c['resolved']):>8}") - - -def print_compare(summaries): - basis = summaries[0]["cost_basis"] if summaries else "?" - print(f"\ncost basis: {basis} (cache-aware = fresh-input-token equivalents; " - f"pass --raw for legacy tokens_total)") - print(f"\n{'mode':<14} {'tasks':>5} {'resolve':>8} {'exec/task':>11} " - f"{'opt/task':>10} {'amort/task':>11} {'opt%':>6}") - print("-" * 70) - for s in summaries: - opt_per = round(s["cum_optimizer_tokens"] / s["n_tasks"], 1) if s["n_tasks"] else 0 - rr = "n/a" if s["resolve_rate"] is None else f"{s['resolve_rate']:.3f}" - print(f"{str(s['mode']):<14} {s['n_tasks']:>5} {rr:>8} " - f"{s['exec_cost_per_task']:>11.0f} {opt_per:>10} " - f"{s['amortized_cost_per_task']:>11.0f} " - f"{s['optimizer_overhead_frac']*100:>5.1f}%") - # warn if any run lacked the breakdown (effective cost fell back to tokens_total) - missing = [s for s in summaries if s["n_with_breakdown"] < s["n_tasks"] - and "raw" not in s["cost_basis"]] - if missing: - print("\n NOTE: some tasks lacked a token breakdown; effective cost fell back " - "to tokens_total for those:") - for s in missing: - print(f" {os.path.basename(s['path'])}: " - f"{s['n_with_breakdown']}/{s['n_tasks']} tasks had a breakdown") - print("\nexec/task = mean execution cost (cache-aware unless --raw; incl. injected context)") - print("opt/task = mean optimizer overhead (face value; gating is free)") - print("amort/task = (exec + optimizer) / tasks <- the HONEST amortized cost") - - -def main(): - ap = argparse.ArgumentParser(description=__doc__, - formatter_class=argparse.RawDescriptionHelpFormatter) - ap.add_argument("paths", nargs="+", help="stream JSONL file(s) or globs") - ap.add_argument("--curve", action="store_true", - help="also print the per-position amortized-cost curve for each run") - ap.add_argument("--raw", action="store_true", - help="report legacy full-weight tokens_total instead of cache-aware cost") - ap.add_argument("--exclude-init", action="store_true", - help="drop the Web-Bench `init` scaffold row so cost normalizes over the " - "20 graded tasks (no-op on SWE-bench streams, which have no init row)") - ap.add_argument("--json", action="store_true", help="emit machine-readable JSON") - args = ap.parse_args() - - files = [] - for p in args.paths: - files.extend(sorted(glob.glob(p)) if any(c in p for c in "*?[") else [p]) - files = [f for f in files if os.path.isfile(f)] - if not files: - print("no stream JSONL files matched", file=sys.stderr) - sys.exit(1) - - summaries = [summarize(f, raw=args.raw, exclude_init=args.exclude_init) for f in files] - if args.json: - print(json.dumps(summaries, indent=2)) - return - if args.curve: - for s in summaries: - print_curve(s) - print_compare(summaries) - - -if __name__ == "__main__": - main() diff --git a/experiments/scripts/sweep_plan.py b/experiments/scripts/sweep_plan.py index 70280a2..2ab7d8d 100755 --- a/experiments/scripts/sweep_plan.py +++ b/experiments/scripts/sweep_plan.py @@ -98,12 +98,6 @@ # Generic arms only: this planner enumerates the Claude / Copilot sweep. The Codex arms # are launched by codex_ab.sh, and its Web-Bench list already leaves out the two repomap-bearing arms for the same reason as here. SWEEP_ARMS_WEBBENCH_UNAVAILABLE = ("serena", "repomap", "trimech-repomap") -# `trimech-rtk` is not "unavailable" on Web-Bench -- it is SUPERSEDED there by -# `trimech-rtk-web`, exactly as `trimech` is by `trimech-web`. Listing it as unavailable -# would claim the scaffold cannot run, which is false and would mislead the reader. -SWEEP_ARMS_WEBBENCH_SUPERSEDED = {"trimech": "trimech-web", - "spiral-only": "spiral-only-web", - "trimech-rtk": "trimech-rtk-web"} SWEEP_SEEDS = (0, 1, 2) # the paper's three seeds # Measured on the stored claude_swebench tree: 984 task-runs, mean $0.34/task on the honest diff --git a/experiments/te_bench/codex_spiral.py b/experiments/te_bench/codex_spiral.py index 501e437..d454ba0 100644 --- a/experiments/te_bench/codex_spiral.py +++ b/experiments/te_bench/codex_spiral.py @@ -24,8 +24,6 @@ The bare dataclass defaults (`DEFAULT_POLICY`) are a single trigger: after the envelope, a patch spanning at least 15 distinct files has failed verification after its latest edit. -An orthogonal, opt-in completion audit can use the same thread once when deterministic -end-state evidence shows an empty patch or a still-failing post-edit verification. """ from __future__ import annotations @@ -118,13 +116,6 @@ def verification_scopes(command: str) -> Optional[set[str]]: for match in _TEST_PATH_RE.finditer(command)} paths = {p for p in paths if p} return paths or {_ALL_SCOPE} -_NO_CHANGE_CLAIM_RE = re.compile( - r"(?i)(?:\balready (?:fixed|resolved) (?:at|in|on|by|the|this|current)\b|" - r"\bno (?:additional |source )?changes? (?:are|were|is|was) " - r"(?:necessary|required|needed)\b)" -) - -COMPLETION_AUDIT_MARKER = "[AGENTEO_CODEX_COMPLETION_AUDIT]" @dataclass(frozen=True) @@ -293,7 +284,6 @@ def profile_version(profile: str) -> str: # and what `codex-trimech` co-activates. The shipped policy stays reachable, and named, as # `codex-spiral-postgreen`. CODEX_DEFAULT_PROFILE = "defensive" -CODEX_SHIPPED_PROFILE = "post-green" def _normalize(text: str, limit: int = 180) -> str: @@ -431,16 +421,6 @@ def parse_events(line: str) -> list[CodexEvent]: return [] -def parse_event(line: str) -> Optional[CodexEvent]: - """The PRIMARY event of a line (kept for callers that expect one); the command event - for a command line even when it also carries an edit -- use ``parse_events`` to see - both.""" - events = parse_events(line) - if not events: - return None - return events[-1] - - def iter_events(lines: Iterable[str], *, first_thread_only: bool = False): """Yield parsed events, optionally excluding suffixes made by an old restart.""" threads = 0 @@ -453,24 +433,6 @@ def iter_events(lines: Iterable[str], *, first_thread_only: bool = False): yield event -def final_agent_message(lines: Iterable[str]) -> str: - """Return the last completed Codex agent message from JSONL text/lines.""" - if isinstance(lines, str): - lines = lines.splitlines() - final = "" - for line in lines: - try: - event = json.loads(line) - except Exception: - continue - item = event.get("item") or {} - if (event.get("type") == "item.completed" - and item.get("type") == "agent_message" - and isinstance(item.get("text"), str)): - final = item["text"] - return final - - class CodexSpiralState: """Incremental Codex-only process state used identically live and offline.""" @@ -886,32 +848,10 @@ def nudge_text(reason: str) -> str: ) -def completion_audit_text(reason: str) -> str: - evidence = { - "claimed-already-fixed": - "The previous response claimed the issue was already fixed but produced no patch.", - "empty-patch": - "The previous response completed without a source patch.", - "latest-verification-failed": - "The latest relevant verification still failed after the newest edit.", - }[reason] - return ( - COMPLETION_AUDIT_MARKER + "\n" - "Automated completion audit: " + evidence + " The current checkout is the " - "failing base for this benchmark; preserve any current work and do not assume the " - "issue is already fixed. Re-read the issue and `git diff`, produce the smallest " - "nonempty source patch, check every affected public entry point or call path, and " - "run the narrowest focused reproduction after the latest edit. Do not edit tests. " - "This continuation is capped, so prioritize the minimal fix and verification." - ) - - class CodexResumeGuard: """Live one-shot detector that continues only through native Codex resume.""" - def __init__(self, policy: CodexSpiralPolicy = DEFAULT_POLICY, *, verbose: bool = False, - completion_audit: bool = False, - completion_audit_tool_turns: int = 8): + def __init__(self, policy: CodexSpiralPolicy = DEFAULT_POLICY, *, verbose: bool = False): self.policy = policy self.verbose = verbose self.state = CodexSpiralState(policy.recent_window) @@ -919,13 +859,6 @@ def __init__(self, policy: CodexSpiralPolicy = DEFAULT_POLICY, *, verbose: bool self.pending_reason: Optional[str] = None self.fired_reasons: set[str] = set() # one nudge per reason per task self.events: list[dict] = [] - self.completion_audit_enabled = bool(completion_audit) - self.completion_audit_tool_turns = max(1, int(completion_audit_tool_turns)) - self.completion_audits_done = 0 - self.completion_audit_reason: Optional[str] = None - self.completion_audit_active = False - self.completion_audit_start_turn = 0 - self.completion_audit_capped = False def monitor(self, line: str) -> bool: events = parse_events(line) @@ -934,23 +867,6 @@ def monitor(self, line: str) -> bool: for event in events: self.state.update(event) event = events[-1] - if self.completion_audit_active: - used = self.state.turns - self.completion_audit_start_turn - if (event.kind in ("command", "edit") - and used >= self.completion_audit_tool_turns): - self.completion_audit_active = False - self.completion_audit_capped = True - self.events.append({ - "action": "completion-audit-cap", - "reason": self.completion_audit_reason, - "tool_turns": used, - "snapshot": self.state.snapshot(), - }) - if self.verbose: - print(f" [codex-completion-audit] CAP at {used} tool turns") - return True - # Do not stack another spiral intervention inside the final audit. - return False if event.kind == "thread" or self.nudges_done >= self.policy.nudge_budget: return False reason = self.state.decision(self.policy) @@ -1002,55 +918,6 @@ def continuation(self): request["effort"] = self.policy.resume_effort return request - def completion_continuation(self, patch: str, transcript: str): - """Return one deterministic same-thread end-state audit, if warranted.""" - if (not self.completion_audit_enabled or self.completion_audits_done - or not self.state.thread_id): - return None - patch_empty = not (patch or "").strip() - final_message = final_agent_message(transcript) - if patch_empty and _NO_CHANGE_CLAIM_RE.search(final_message): - reason = "claimed-already-fixed" - elif patch_empty: - reason = "empty-patch" - elif (self.state.last_failed_verification_turn > self.state.last_edit_turn - and not self.state.currently_green): - reason = "latest-verification-failed" - else: - return None - - self.completion_audits_done = 1 - self.completion_audit_reason = reason - self.completion_audit_active = True - self.completion_audit_start_turn = self.state.turns - record = { - "action": "completion-audit-resume", - "reason": reason, - "thread_id": self.state.thread_id, - "snapshot": self.state.snapshot(), - } - self.events.append(record) - if self.verbose: - print(f" [codex-completion-audit] RESUME ({reason})") - return { - "mode": "resume", - "session_id": self.state.thread_id, - "prompt": completion_audit_text(reason), - } - - def finish_completion_audit(self): - """Record a naturally completed audit (a capped one already records itself).""" - if not self.completion_audit_active: - return - used = self.state.turns - self.completion_audit_start_turn - self.completion_audit_active = False - self.events.append({ - "action": "completion-audit-complete", - "reason": self.completion_audit_reason, - "tool_turns": used, - "snapshot": self.state.snapshot(), - }) - def telemetry(self) -> dict: return { "policy": asdict(self.policy), @@ -1059,10 +926,5 @@ def telemetry(self) -> dict: "snapshot": self.state.snapshot(), "thread_id_captured": bool(self.state.thread_id), "paid_judge_effective_tokens": 0, - "completion_audit_enabled": self.completion_audit_enabled, - "completion_audits": self.completion_audits_done, - "completion_audit_reason": self.completion_audit_reason, - "completion_audit_capped": self.completion_audit_capped, - "completion_audit_tool_turn_cap": self.completion_audit_tool_turns, "resume_effort": self.policy.resume_effort or None, } diff --git a/experiments/te_bench/runner.py b/experiments/te_bench/runner.py index df426dc..76f621a 100644 --- a/experiments/te_bench/runner.py +++ b/experiments/te_bench/runner.py @@ -33,11 +33,6 @@ from typing import Optional, Protocol, Tuple, runtime_checkable -# The optimizer-call signature the mechanisms invoke: (prompt, model) -> (text, eff_tokens) -# where eff_tokens is the effective (cache-aware) token cost of the call, or None. -OptimizerRunner = "Callable[[str, str], Tuple[str, Optional[float]]]" - - @runtime_checkable class AgentRunner(Protocol): """Everything the economizer needs from the task-execution harness.""" diff --git a/experiments/te_bench/workloads.py b/experiments/te_bench/workloads.py index faa1d46..72acb4a 100644 --- a/experiments/te_bench/workloads.py +++ b/experiments/te_bench/workloads.py @@ -407,12 +407,6 @@ def instances(domain, group=None): ), } -# Every added id is outside SWE-bench_Verified (both repos' Verified rows were already -# fully consumed by the published streams), so provisioning must read the FULL test split -# for these -- exactly like SWEBENCH_FULL_SPLIT_ONLY above. -SWEBENCH_ADDED_FULL_SPLIT_ONLY: frozenset[str] = frozenset( - i for ids in SWEBENCH_ADDED.values() for i in ids) - def runnable_instances(domain, group=None): """Canonical instances PLUS SWEBENCH_ADDED MINUS SWEBENCH_EXCLUDED — what a NEW diff --git a/experiments/tests/test_spiral.py b/experiments/tests/test_spiral.py index 841f13e..414f138 100644 --- a/experiments/tests/test_spiral.py +++ b/experiments/tests/test_spiral.py @@ -83,9 +83,9 @@ def test_shell_applied_patch_is_an_edit_event_with_its_paths(): failed = _cs.parse_events(_line({"type": "command_execution", "command": _PATCH_CMD, "aggregated_output": "apply_patch: error: Failed to find context", "exit_code": 1})) assert [e.kind for e in failed] == ["command"] - # an ordinary command stays one event; parse_event still returns the primary one + # an ordinary command stays one event plain = _cs.parse_events(_line({"type": "command_execution", "command": "ls", "aggregated_output": "", "exit_code": 0})) - assert [e.kind for e in plain] == ["command"] and _cs.parse_event(_line({"type": "command_execution", "command": _PATCH_CMD, "aggregated_output": "Success. Updated", "exit_code": 0})).kind == "command" + assert [e.kind for e in plain] == ["command"] # string exit codes (defensive) are normalised assert _cs.parse_events(_line({"type": "command_execution", "command": "pytest -q", "aggregated_output": "1 passed", "exit_code": "0"}))[0].exit_code == 0 diff --git a/experiments/tests/test_sweep_arms.py b/experiments/tests/test_sweep_arms.py index d87c4b4..449beae 100644 --- a/experiments/tests/test_sweep_arms.py +++ b/experiments/tests/test_sweep_arms.py @@ -373,7 +373,6 @@ def test_the_codex_spiral_arms_have_the_shape_the_injection_arms_have(): assert S(n).codex_spiral and not S(n).guard and not S(n).nudge from te_bench import codex_spiral as cs assert cs.CODEX_DEFAULT_PROFILE == "defensive" - assert cs.CODEX_SHIPPED_PROFILE == "post-green" def test_the_codex_family_relates_to_codex_trimech_as_the_generic_one_does_to_trimech(): diff --git a/experiments/tests/test_workloads.py b/experiments/tests/test_workloads.py index 4de32a7..a026b9e 100644 --- a/experiments/tests/test_workloads.py +++ b/experiments/tests/test_workloads.py @@ -111,7 +111,6 @@ def test_added_does_not_move_the_published_record(): def test_added_ids_need_the_full_split(): """Both repos' Verified rows were already spent, so every addition is full-split-only.""" added = {i for ids in workloads.SWEBENCH_ADDED.values() for i in ids} - assert workloads.SWEBENCH_ADDED_FULL_SPLIT_ONLY == added assert not (added & workloads.SWEBENCH_FULL_SPLIT_ONLY) diff --git a/tests/test_ledger.py b/tests/test_ledger.py index 32a28a5..ffb0832 100644 --- a/tests/test_ledger.py +++ b/tests/test_ledger.py @@ -1,6 +1,6 @@ -"""Ledger: overhead(), effective_exec(), and the CostLedger accumulator.""" +"""Ledger: overhead() and effective_exec().""" from tokeneconomizer import ledger -from tokeneconomizer.ledger import CostLedger, OVERHEAD_KEYS +from tokeneconomizer.ledger import OVERHEAD_KEYS def test_overhead_sums_only_known_keys(): @@ -28,13 +28,3 @@ def test_effective_exec_uses_breakdown(): def test_effective_exec_falls_back_to_total(): assert ledger.effective_exec({"tokens_total": 777}) == 777 - -def test_cost_ledger_amortizes(): - cl = CostLedger() - cl.add({"tokens_total": 100, "optimizer_tokens": 20}) - cl.add({"tokens_total": 200}) - assert cl.cum_exec == 300 - assert cl.cum_overhead == 20 - assert cl.cum_total == 320 - assert cl.amortized_per_task == 160 - assert 0 < cl.overhead_frac < 1 diff --git a/tokeneconomizer/ledger.py b/tokeneconomizer/ledger.py index 282c08f..7e781e8 100644 --- a/tokeneconomizer/ledger.py +++ b/tokeneconomizer/ledger.py @@ -39,36 +39,3 @@ def effective_exec(row): """Cache-aware effective execution cost of a task (pricing.effective_tokens).""" return pricing.effective_tokens(row) - -class CostLedger: - """Accumulates the amortized cost over a stream. Thin wrapper over the two canonical - functions above -- used by tests and any caller that wants a running total without - re-deriving the math. The aggregation scripts share these definitions via - OVERHEAD_KEYS / overhead().""" - - def __init__(self): - self.cum_exec = 0.0 - self.cum_overhead = 0.0 - self.n = 0 - - def add(self, row, exec_cost=None): - """Fold one stream row. `exec_cost` overrides the effective exec (e.g. a - breakdown-resolved record); defaults to this row's effective exec.""" - ec = effective_exec(row) if exec_cost is None else exec_cost - oh = overhead(row) - self.cum_exec += ec - self.cum_overhead += oh - self.n += 1 - return ec + oh - - @property - def cum_total(self): - return self.cum_exec + self.cum_overhead - - @property - def amortized_per_task(self): - return (self.cum_total / self.n) if self.n else 0.0 - - @property - def overhead_frac(self): - return (self.cum_overhead / self.cum_total) if self.cum_total else 0.0 diff --git a/tokeneconomizer/microskills/observe.py b/tokeneconomizer/microskills/observe.py index efe4c45..a410183 100644 --- a/tokeneconomizer/microskills/observe.py +++ b/tokeneconomizer/microskills/observe.py @@ -95,17 +95,9 @@ class Vocabulary: _RUN_SCRIPT_HEADS = frozenset({"npm", "yarn", "pnpm", "bun"}) DEFAULT_VOCAB = PYTHON_GIT -# Module-level aliases of the default vocabulary, kept for the offline analyses. -INTERPRETERS = PYTHON_GIT.interpreters -RUNNER_HEADS = PYTHON_GIT.runner_heads -RUNNER_SCRIPTS = PYTHON_GIT.runner_scripts -RUNNER_MODULES = PYTHON_GIT.runner_modules -PACKAGE_TOOLS = PYTHON_GIT.package_tools -FETCH_HEADS = PYTHON_GIT.fetch_heads GIT_FETCH_SUBS = frozenset({"fetch", "clone", "pull", "ls-remote"}) GIT_PROBE_SUBS = frozenset({"remote", "branch", "tag", "fsck", "stash", "show-ref", "for-each-ref"}) -ENV_PROBE_HEADS = PYTHON_GIT.env_probe_heads PREFIX_HEADS = ("sudo", "env", "time", "exec", "nice", "nohup", "timeout") _PASSTHROUGH = frozenset({"cd", "echo", "head", "tail", "cat", "sort", "uniq", "tee", "true", "printf", "less", "more"}) diff --git a/tokeneconomizer/pricing.py b/tokeneconomizer/pricing.py index 5bb7f73..afedb04 100644 --- a/tokeneconomizer/pricing.py +++ b/tokeneconomizer/pricing.py @@ -109,9 +109,6 @@ # weights_for(); effective_tokens() applies it automatically. WEIGHTS = WEIGHTS_CLAUDE -_BREAKDOWN_KEYS = ("input_tokens", "cache_read_input_tokens", - "cache_creation_input_tokens", "output_tokens") - def resolve_model(rec): """The billed model id for a row: its explicit `model`, else the agent's default diff --git a/tokeneconomizer/toolprune/codex.py b/tokeneconomizer/toolprune/codex.py index 78b1e14..c547a3b 100644 --- a/tokeneconomizer/toolprune/codex.py +++ b/tokeneconomizer/toolprune/codex.py @@ -35,32 +35,6 @@ # =========================================================================== # Transcript capability detection (which Codex packages a trace item exercises) # =========================================================================== -CAPABILITY_DESCRIPTIONS = { - "collaboration": ( - "Spawn, message, wait for, or close sub-agents. The benchmark runs one sealed " - "agent process and provides no useful parallel-agent workflow."), - "goals": ( - "Create, inspect, and update long-lived product goals. These are orchestration " - "metadata rather than repository read/edit/test operations."), - "image_media": ( - "View or transport image/audio/generated-media blocks. SWE-bench repair tasks " - "operate on a checked-out source repository and text test output."), - "interactive_guidance": ( - "Persona, writing-style, commentary-channel and answer-formatting guidance for an " - "interactive chat session. A detached, single-turn run has no user reading " - "commentary and its final message is not graded."), - "planning": ( - "Maintain a structured todo/update_plan list. This can help some long repairs but " - "is optional because the agent can reason and report progress without the tool."), - "web": ( - "Search/open the public web and query online services."), - "request_user_input": ( - "Pause for interactive human input. Runs are detached, autonomous, and have no " - "human available inside a task."), -} - -CAPABILITIES = tuple(sorted(CAPABILITY_DESCRIPTIONS)) - _ITEM_TYPES_BY_CAPABILITY = { "collaboration": frozenset(( "spawn_agent", "send_input", "send_message", "followup_task", @@ -139,9 +113,6 @@ def _item_capabilities(item): ) _FAILED_STATUSES = frozenset(("failed", "error", "cancelled")) -_HINT_BASE_RELAY = ( - "deferred_wait", "image_media", "request_user_input", "skills", "web") - def task_hints(problem_statement): """Return the packages a task's issue text protects from demotion (image/skills).""" @@ -605,11 +576,3 @@ def observe(self, usage, *, resolved=None): self.save() self._pending = None return record - - def debug_state(self): - state = copy.deepcopy(self.state) - state["next_logical_mask"] = list(state.get("current_pruned") or ()) - state["next_relay_mask"] = relay_capabilities(state.get("current_pruned") or ()) - state["core_capabilities"] = list(CORE_CAPABILITIES) - state["state_path"] = self.path - return state