Polish README and fix benchmark reproducibility

.github/workflows/offline-checks.yml

@@ -0,0 +1,27 @@
+name: Offline reproducibility
+
+on:
+  push:
+  pull_request:
+
+permissions:
+  contents: read
+
+jobs:
+  offline:
+    runs-on: ubuntu-latest
+    env:
+      MPLBACKEND: Agg
+    steps:
+      - uses: actions/checkout@v7
+      - uses: actions/setup-python@v7
+        with:
+          python-version: '3.12'
+          cache: pip
+          cache-dependency-path: requirements.txt
+      - name: Install numerical environment
+        run: python -m pip install -r requirements.txt
+      - name: Test numerical core, parser, and CLI workflows
+        run: python -m unittest discover -s . -p 'test_*.py' -v
+      - name: Generate an offline example with its dashboard
+        run: python main_loop.py --mock --difficulty easy --seed 42 --output-dir runs/ci-smoke

.python-version

@@ -0,0 +1 @@
+3.12

generate_transcripts.py

@@ -14,6 +14,7 @@
 import time
 import json
 import os
+from uuid import uuid4
 import numpy as np
 
 from main_loop import (
@@ -41,9 +42,21 @@
 DEFAULT_DIFFICULTIES = ["easy", "medium", "hard"]
 DEFAULT_SEEDS = [42, 137]
 
+# Legacy summaries did not record every setting. Keep them distinguishable from
+# explicitly configured runs rather than guessing which experiment they belong to.
+_UNSPECIFIED = object()
+_RUN_KEY_FIELDS = (
+    "model", "difficulty", "seed", "prompt_condition", "temperature", "reasoning_effort",
+)
+
 
 # ── Helpers ──────────────────────────────────────────────────────────────────
 
+def run_key(record: dict) -> tuple:
+    """Identify an experiment without conflating prompt or sampling settings."""
+    return tuple(record.get(field, _UNSPECIFIED) for field in _RUN_KEY_FIELDS)
+
+
 def parse_seeds(seed_str: str) -> list[int]:
     """Parse '42-46' or '42 43 44' into a list of ints."""
     if "-" in seed_str and seed_str.count("-") == 1:
@@ -91,6 +104,7 @@ def run_single(model: str, provider: str, difficulty: str, seed: int,
             "budget_source": "auto",
             "prompt_condition": "baseline" if baseline else "standard",
             "reasoning_effort": reasoning_effort,
+            "temperature": temperature,
             "timestamp": time.strftime("%Y-%m-%dT%H:%M:%S"),
         },
         "turns": [],
@@ -209,7 +223,10 @@ def run_single(model: str, provider: str, difficulty: str, seed: int,
 
     # ── Save files ──
     model_slug = model.replace("/", "_").replace(".", "_")
-    stem = f"run_{difficulty}_s{seed}_{model_slug}_{time.strftime('%Y%m%d_%H%M%S')}"
+    # Prompt/settings can differ even when model, PDE seed and wall-clock second
+    # match. Never let one run replace another run's raw log or plots.
+    stem = (f"run_{difficulty}_s{seed}_{model_slug}_"
+            f"{time.strftime('%Y%m%d_%H%M%S')}_{uuid4().hex}")
 
     log_path = os.path.join(output_dir, stem + ".json")
     with open(log_path, "w") as f:
@@ -248,6 +265,8 @@ def run_single(model: str, provider: str, difficulty: str, seed: int,
         "difficulty": difficulty,
         "seed": seed,
         "reasoning_effort": reasoning_effort,
+        "prompt_condition": "baseline" if baseline else "standard",
+        "temperature": temperature,
         "queries_used": result["queries_used"],
         "max_queries": max_queries,
         "turns": result["turns"],
@@ -346,7 +365,7 @@ def main():
     parser.add_argument("--output-dir", type=str, default="runs",
                         help="Output directory (default: runs/)")
     parser.add_argument("--baseline", action="store_true",
-                        help="Use baseline prompt (no mathematical context)")
+                        help="Omit the initial identifiability hint (shared PDE/tool instructions remain)")
     parser.add_argument("--reasoning-effort", type=str, default=None,
                         choices=["none", "low", "medium", "high", "xhigh"],
                         help="Override reasoning effort for all OpenAI models")
@@ -355,7 +374,7 @@ def main():
     parser.add_argument("--retry-failed", action="store_true",
                         help="Read sweep_results.json from output-dir, re-run only failed/missing seeds")
     parser.add_argument("--temperature", type=float, default=0.0,
-                        help="LLM sampling temperature (0 = deterministic)")
+                        help="Requested LLM sampling temperature (default: 0; API outputs can vary)")
     args = parser.parse_args()
 
     seeds = parse_seeds(args.seeds)
@@ -371,6 +390,8 @@ def main():
             }
         elif "claude" in m or "anthropic" in m.lower():
             model_configs[m] = {"provider": "anthropic", "reasoning_effort": None}
+        elif "gemini" in m.lower():
+            model_configs[m] = {"provider": "google", "reasoning_effort": None}
         else:
             model_configs[m] = {"provider": "openai", "reasoning_effort": args.reasoning_effort}
 
@@ -381,6 +402,16 @@ def main():
         for seed in seeds
     ]
 
+    def requested_key(model, difficulty, seed):
+        return run_key({
+            "model": model,
+            "difficulty": difficulty,
+            "seed": seed,
+            "prompt_condition": "baseline" if args.baseline else "standard",
+            "temperature": args.temperature,
+            "reasoning_effort": model_configs[model]["reasoning_effort"],
+        })
+
     if args.retry_failed:
         sweep_path = os.path.join(args.output_dir, "sweep_results.json")
         if os.path.exists(sweep_path):
@@ -389,9 +420,12 @@ def main():
             done = set()
             for r in existing:
                 if "error" not in r and r.get("total_coeff_error") is not None:
-                    done.add((r["model"], r["difficulty"], r["seed"]))
-            all_combos = [(m, d, s) for m, d, s in all_combos if (m, d, s) not in done]
-            print(f"Retrying {len(all_combos)} failed/missing seeds ({len(done)} already succeeded)")
+                    done.add(run_key(r))
+            requested_count = len(all_combos)
+            all_combos = [(m, d, s) for m, d, s in all_combos
+                          if requested_key(m, d, s) not in done]
+            print(f"Retrying {len(all_combos)} failed/missing runs "
+                  f"({requested_count - len(all_combos)} matching runs already succeeded)")
         else:
             print(f"No sweep_results.json found in {args.output_dir}, running all")
 
@@ -445,6 +479,8 @@ def main():
                 "model": model, "provider": provider,
                 "difficulty": difficulty, "seed": seed,
                 "reasoning_effort": reasoning_effort,
+                "prompt_condition": "baseline" if args.baseline else "standard",
+                "temperature": args.temperature,
                 "error": str(e),
             })
 
@@ -453,9 +489,9 @@ def main():
     if os.path.exists(sweep_path):
         with open(sweep_path) as f:
             existing = json.load(f)
-        existing_map = {(r["model"], r["difficulty"], r["seed"]): r for r in existing}
+        existing_map = {run_key(r): r for r in existing}
         for r in new_results:
-            existing_map[(r["model"], r["difficulty"], r["seed"])] = r
+            existing_map[run_key(r)] = r
         all_results = sorted(existing_map.values(), key=lambda r: r["seed"])
     else:
         all_results = new_results
@@ -468,4 +504,4 @@ def main():
 
 
 if __name__ == "__main__":
-    main()
+    main()

main_loop.py

@@ -698,21 +698,22 @@ def chat(self, system: str, messages: list[dict]) -> str:
 
 
 class MockBackend:
-    """Simple mock that queries a few test functions then predicts zeros."""
-    def __init__(self):
+    """Scripted offline workflow check, not a model-performance baseline."""
+    def __init__(self, n_queries: int = 8):
         self.model = "mock"
         self.call_count = 0
         self.script = [
             "COMPUTE: eval_solution 0.1 0.2 0.3 0.4 0.5 0.6 0.7 0.8 0.9",
-            "QUERY: sin(pi*x)",
-            "QUERY: sin(2*pi*x)",
-            "QUERY: sin(3*pi*x)",
-            "QUERY: x*(1-x)",
-            "QUERY: x*(1-x)*(2*x-1)",
-            "QUERY: x**2*(1-x)**2",
-            "QUERY: x*(1-x)*sin(pi*x)",
-            "PREDICT:\na_coeffs = [1.0, 0.0, 0.0]\nb_coeffs = [0.0, 0.0, 0.0]\nc_coeffs = [0.0, 0.0, 0.0]\nf_coeffs = [0.0, 0.0, 0.0]",
         ]
+        for index in range(n_queries):
+            mode = index // 2 + 1
+            expression = (f"sin({mode}*pi*x)" if index % 2 == 0
+                          else f"x*(1-x)*exp(-{mode}*x)")
+            self.script.append(f"QUERY: {expression}")
+        self.script.extend([
+            "COMPUTE: solve",
+            "PREDICT:\nConfidence: a=0%, b=0%, c=0%, f=0%",
+        ])
 
     def chat(self, system, messages):
         idx = min(self.call_count, len(self.script) - 1)
@@ -1525,12 +1526,12 @@ def main():
     parser.add_argument("--model", default=None,
                         help="Model name (default: claude-haiku-4-5-20251001 for anthropic, gpt-4o for openai)")
     parser.add_argument("--max-turns", type=int, default=None,
-                        help="Override auto-scaled turn limit (default: 2x budget)")
+                        help="Override auto-scaled turn limit (default: 3x budget)")
     parser.add_argument("--save-transcript", type=str, default=None)
     parser.add_argument("--save-plot", type=str, default=None,
-                        help="Save results plot to file (default: probe_<difficulty>_<seed>.png)")
+                        help="Save dashboard to this path (default: alongside the timestamped JSON)")
     parser.add_argument("--baseline", action="store_true",
-                        help="Use minimal prompt without mathematical context (for ablation)")
+                        help="Omit the initial identifiability hint (shared PDE/tool instructions remain)")
     parser.add_argument("--reasoning-effort", type=str, default=None,
                         choices=["none", "low", "medium", "high", "xhigh"],
                         help="Reasoning effort for OpenAI GPT-5+ models "
@@ -1542,7 +1543,7 @@ def main():
     parser.add_argument("--no-plot", action="store_true",
                         help="Skip generating dashboard PNG (JSON only)")
     parser.add_argument("--temperature", type=float, default=0.0,
-                        help="LLM sampling temperature (0 = deterministic)")
+                        help="Requested LLM sampling temperature; API responses may vary")
     args = parser.parse_args()
 
     if args.model is None:
@@ -1553,12 +1554,19 @@ def main():
     config = DIFFICULTY_CONFIG[args.difficulty]
     max_queries = args.max_queries if args.max_queries is not None else config["budget"]
     max_turns = args.max_turns if args.max_turns is not None else 3 * max_queries
+    if max_queries <= 0 or max_turns <= 0:
+        parser.error("--max-queries and --max-turns must be positive")
+    mock_queries = max(config["unknowns"], args.min_queries or 0)
+    if args.mock and (max_queries < mock_queries or max_turns < mock_queries + 3):
+        parser.error(f"--mock needs at least {mock_queries} queries and {mock_queries + 3} turns")
 
     os.makedirs(args.output_dir, exist_ok=True)
 
     session = ProbeSession.from_difficulty(args.difficulty, args.seed, max_queries)
     if args.mock:
-        backend = MockBackend()
+        backend = MockBackend(n_queries=mock_queries)
+        args.model = "mock"
+        args.provider = "mock"
     elif args.provider == "openai":
         backend = OpenAIBackend(model=args.model, reasoning_effort=args.reasoning_effort,
                                 temperature=args.temperature)
@@ -1580,6 +1588,8 @@ def main():
             "budget_source": "override" if args.max_queries is not None else "auto",
             "prompt_condition": "baseline" if args.baseline else "standard",
             "reasoning_effort": args.reasoning_effort,
+            "temperature": args.temperature,
+            "min_queries": args.min_queries,
             "timestamp": datetime.now().isoformat(),
         },
         "turns": [],
@@ -1699,7 +1709,7 @@ def main():
     re_tag = f"_re{args.reasoning_effort}" if args.reasoning_effort else ""
     run_stem = (
         f"run_{args.difficulty}_s{args.seed}_{model_slug}{re_tag}"
-        f"_{datetime.now().strftime('%Y%m%d_%H%M%S')}"
+        f"_{datetime.now().strftime('%Y%m%d_%H%M%S_%f')}"
     )
     log_path = os.path.join(args.output_dir, run_stem + ".json")
     with open(log_path, "w") as f:
@@ -1744,4 +1754,4 @@ def main():
 
 
 if __name__ == "__main__":
-    main()
+    main()