Add certified full 545-case WebMainBench report

evaluation/webmainbench-full/content-binding.json

@@ -0,0 +1,9 @@
+{
+  "schema_version": 1,
+  "relative_path": "report-content.json",
+  "sha256": "1d0aa4cbd90b52e046062d00ceb97c4f3dd8a19f7c51c0b1547eb826ba9d61e4",
+  "infographic": "infographic.png",
+  "infographic_sha256": "402f90bb2d664b38e9efb697f910d99d142d4e872cc7d1e76b8ed733f8aa9e62",
+  "infographic_mobile": "infographic-mobile.png",
+  "infographic_mobile_sha256": "2c60966e07b88fe5d0c7e966af90f4027e61bd22e439f7a4d408b875c4e18eb7"
+}

evaluation/webmainbench-full/current-report.json

@@ -0,0 +1,32 @@
+{
+  "schema_version": 1,
+  "dataset_id": "webmainbench-545-full-five-methods",
+  "comparison_kind": "webmainbench_frozen_html_to_markdown",
+  "case_count": 545,
+  "case_ids_sha256": "5c0cf5810602b70e8f7c1e16723e24a49c3ea207d54865df5c03c38f3d88ffd5",
+  "source_dataset_sha256": "0efaa4b49a45e320a27fe6e5a0b6aad5b57259fc3321ac3448519cacc74c537e",
+  "providers": ["moli", "lightpanda", "obscura", "chromium", "kitesurf"],
+  "versions": {
+    "kitesurf": "Cloudflare hosted stable backend; version undisclosed",
+    "lightpanda": "0.4.1",
+    "obscura": "0.2.2",
+    "moli": "1.1.9",
+    "chromium": "Google Chrome for Testing 154.0.8037.57"
+  },
+  "timeout_seconds": 45,
+  "runs_per_case": 1,
+  "concurrency": 1,
+  "max_failure_retries": 1,
+  "reference_binding_mode": "complete_webmainbench_545_official_frozen_html_and_groundtruth_content",
+  "official_scorer_commit": "9d991bdc00c57b57521499494d96be85c31317ba",
+  "metric_config": {"use_llm": false},
+  "official_metrics": ["overall", "text_edit", "code_edit", "formula_edit", "table_edit", "table_TEDS"],
+  "input_contract": {
+    "body": "official html field encoded as UTF-8 without mutation",
+    "execution": "inline scripts may run; non-loopback subresources and data access are blocked; local engines receive the same body from an OS-isolated loopback HTTPS fixture with a per-run CA, Kitesurf receives the same body through Cloudflare Browser Rendering html input with external resource types rejected",
+    "original_url": "retained as evidence metadata only"
+  },
+  "publication_state": "pending",
+  "require_independent_result_review": false,
+  "conversion": "Native Markdown for Kitesurf, Lightpanda, Obscura and Moli; Chromium whole-DOM HTML via node-html-markdown 2.0.0"
+}

evaluation/webmainbench-full/failure-analysis-binding.json

@@ -0,0 +1,5 @@
+{
+  "schema_version": 1,
+  "relative_path": "failure-analysis.json",
+  "sha256": "4186a3726979db335ae0aeb4087c6fbac42830869b17c36a7114f7d2e6c02240"
+}

harness/web_fetch/report_build.py

@@ -5,7 +5,7 @@
 import tempfile
 from pathlib import Path
 from harness.reporting.assets import copy_bound_assets
-from .report import generate, generate_public_benchmark_research, generate_webmainbench, generate_wcxb, succeeded, measured, failure_category, TRACKS
+from .report import generate, generate_public_benchmark_research, generate_webmainbench, generate_webmainbench_full, webmainbench_full_machine_issue, generate_wcxb, succeeded, measured, failure_category, TRACKS
 from .report_changes import webmainbench_history
 from .failure_diagnosis import diagnose
 from .safety import contained_path, safe_identifier, validate_report_ids
@@ -501,6 +501,108 @@ def build_webmainbench_report(source: Path, output: Path, *, include_english: bo
     return output / 'index.html'
 
 
+def build_webmainbench_full_report(source: Path, output: Path) -> Path:
+    """Build the separately bound complete 545-case report from certified evidence."""
+    from scripts.run_certified_webmainbench_full import digest, read_jsonl
+    from .webmainbench_diagnostics import validate_teds_revision
+    if source.is_symlink() or any(path.is_symlink() for path in source.rglob('*')):
+        raise ValueError('Symlinked report evidence is forbidden')
+    profile_path = Path(__file__).resolve().parents[2] / 'evaluation/webmainbench-full/current-report.json'
+    profile = json.loads(profile_path.read_text(encoding='utf-8'))
+    certificate = json.loads((source / 'cohort-certificate.json').read_text(encoding='utf-8'))
+    scoring_certificate = json.loads((source / 'scoring-certificate.json').read_text(encoding='utf-8'))
+    if (certificate.get('kind') != 'certified_webmainbench_545_full_v1'
+            or certificate.get('profile_sha256') != digest(profile_path)
+            or certificate.get('preflight') != certificate.get('postflight')
+            or json.loads((source / 'current-report.json').read_text(encoding='utf-8')) != profile):
+        raise ValueError('Full-cohort certificate or profile mismatch')
+    if (scoring_certificate.get('kind') != 'certified_webmainbench_scoring_v1'
+            or scoring_certificate.get('cohort_certificate_sha256') != digest(source / 'cohort-certificate.json')
+            or scoring_certificate.get('source_binding') != certificate['postflight']):
+        raise ValueError('Complete scoring certificate mismatch')
+    dataset = json.loads((source / 'dataset.json').read_text(encoding='utf-8'))
+    records = read_jsonl(source / 'results.jsonl')
+    providers = profile['providers']
+    cases = {case['id']: case for case in dataset['cases']}
+    if (len(cases) != 545 or len(records) != 545 * len(providers)
+            or {(row['provider'], row['case_id']) for row in records} !=
+            {(provider, case_id) for provider in providers for case_id in cases}):
+        raise ValueError('Full WebMainBench result matrix mismatch')
+    validate_teds_revision(source)
+    content_binding_path = Path(__file__).resolve().parents[2] / 'evaluation/webmainbench-full/content-binding.json'
+    content_binding = json.loads(content_binding_path.read_text(encoding='utf-8'))
+    content_path = data_home() / 'report-assets/webmainbench-full' / content_binding['relative_path']
+    if digest(content_path) != content_binding['sha256']:
+        raise ValueError('Report prose differs from the compact content binding')
+    illustration_path = data_home() / 'report-assets/webmainbench-full' / content_binding['infographic']
+    if digest(illustration_path) != content_binding['infographic_sha256']:
+        raise ValueError('Report illustration differs from the compact asset binding')
+    mobile_illustration_path = data_home() / 'report-assets/webmainbench-full' / content_binding['infographic_mobile']
+    if digest(mobile_illustration_path) != content_binding['infographic_mobile_sha256']:
+        raise ValueError('Mobile report illustration differs from the compact asset binding')
+    failure_binding_path = Path(__file__).resolve().parents[2] / 'evaluation/webmainbench-full/failure-analysis-binding.json'
+    failure_binding = json.loads(failure_binding_path.read_text(encoding='utf-8'))
+    failure_path = data_home() / 'report-assets/webmainbench-full' / failure_binding['relative_path']
+    if digest(failure_path) != failure_binding['sha256']:
+        raise ValueError('Failure analysis differs from the compact review binding')
+    failure_analysis = json.loads(failure_path.read_text(encoding='utf-8'))
+    failures = {(row['provider'], row['case_id']) for row in records
+                if webmainbench_full_machine_issue(source, row)}
+    reviewed = {(row['provider'], row['case_id']) for row in failure_analysis['reviewed_failures']}
+    if (failure_analysis.get('kind') != 'webmainbench_full_failure_analysis_v1'
+            or failure_analysis.get('cohort_certificate_sha256') != digest(source / 'cohort-certificate.json')
+            or reviewed != failures or len(reviewed) != len(failure_analysis['reviewed_failures'])
+            or any(not all(isinstance(row.get(key), str) and row[key].strip()
+                           for key in ('category_label', 'cause', 'treatment', 'status'))
+                   for row in failure_analysis['reviewed_failures'])):
+        raise ValueError('Failure analysis does not cover every measured request failure')
+    for case_id, case in cases.items():
+        if digest(contained_path(source, f'references/{case_id}.md')) != case['reference_sha256']:
+            raise ValueError('Reference changed: ' + case_id)
+        if digest(contained_path(source, f'sources/{case_id}.html.txt')) != case['input_html_sha256']:
+            raise ValueError('Frozen page changed: ' + case_id)
+    for row in records:
+        if digest(contained_path(source, row['artifact_path'])) != row['output_sha256']:
+            raise ValueError('Output changed: ' + row['attempt_id'])
+    output.parent.mkdir(parents=True, exist_ok=True)
+    staging = Path(tempfile.mkdtemp(prefix='.webmainbench-full-build-', dir=output.parent))
+    try:
+        for name in ('artifacts', 'references', 'sources', 'fixture-certificates', 'correction-source'):
+            shutil.copytree(source / name, staging / name)
+        for name in ('dataset.json', 'results.jsonl', 'original-results.jsonl',
+                     'physical-attempts.jsonl', 'execution.json', 'score-summaries.json',
+                     'cohort-certificate.json', 'current-report.json',
+                     'scoring-certificate.json',
+                     'teds-correction.json', 'teds-correction-review.json'):
+            shutil.copy2(source / name, staging / name)
+        shutil.copy2(content_path, staging / 'report-content.json')
+        shutil.copy2(failure_path, staging / 'failure-analysis.json')
+        shutil.copy2(Path(__file__).resolve().parents[2] / 'research/provider-adoption.json', staging / 'provider-adoption.json')
+        (staging / 'assets').mkdir(exist_ok=True)
+        shutil.copy2(illustration_path, staging / 'assets/infographic.png')
+        shutil.copy2(mobile_illustration_path, staging / 'assets/infographic-mobile.png')
+        generate_webmainbench_full(staging, dataset, records,
+                                   json.loads((source / 'score-summaries.json').read_text(encoding='utf-8')),
+                                   profile)
+        from scripts.check_webmainbench_full_report import check as check_webmainbench_full
+        check_webmainbench_full(staging)
+        if output.exists():
+            backup = Path(tempfile.mkdtemp(prefix='.report-swap-', dir=output.parent)) / 'report'
+            output.rename(backup)
+            try:
+                staging.rename(output)
+            except BaseException:
+                backup.rename(output)
+                raise
+            shutil.rmtree(backup.parent)
+        else:
+            staging.rename(output)
+    finally:
+        if staging.exists():
+            shutil.rmtree(staging)
+    return output / 'index.html'
+
+
 def build_wcxb_report(source: Path, corpus: Path, sample_path: Path, profile: dict, output: Path) -> Path:
     """Build the one current report from a fully certified WCXB matrix."""
     from scripts.run_webmainbench_benchmark import WCXB_PROVIDERS, load_wcxb_source

research/report-changes.json

@@ -1275,6 +1275,17 @@
     }
   ],
   "releases": [
+    {
+      "id": "2026-09-23-webmainbench-complete-545-five-methods",
+      "comparison_kinds": ["webmainbench_frozen_html_to_markdown"],
+      "date": "2026-09-23",
+      "title": "发布 WebMainBench 完整 545 题五方法对照",
+      "title_i18n": {"en": "Publish the complete 545-case WebMainBench comparison"},
+      "summary": "以同一批 545 个官方冻结 HTML 和人工参考，实测五种稳定版浏览器方法。新报告提供逐题输入、输出、评分、失败证据和已复核的机器问题分析。",
+      "summary_i18n": {"en": "Five stable browser methods were measured against the same 545 official frozen HTML inputs and human references. The new report exposes case inputs, outputs, scores, failure evidence, and reviewed machine-issue analysis."},
+      "changes": [["完整评测与评分", "先前公开报告只覆盖 200 题，无法代表完整数据集。官方 TEDS 实现还可能高估部分表格相似度。", "认证五种方法各 545 题，并用保留原始输出和官方分数的结构化树比较校正 TEDS；报告同时展示请求重试和失败分类。", "共 2725 个逻辑结果、2747 次实际调用；292 个 TEDS 分数校正。28 个机器问题已逐条复核，其余 2697 个结果未逐条人工复核。"]],
+      "changes_i18n": {"en": [["Full evaluation and scoring", "The earlier public report covered only 200 cases, and the official TEDS implementation could overstate some table similarities.", "Certify 545 cases for each of five methods and correct TEDS with structured tree comparison while preserving raw outputs and official scores; show retries and failure classes.", "There are 2,725 logical results and 2,747 physical calls; 292 TEDS scores were corrected. All 28 machine issues were reviewed individually, while the other 2,697 results were not individually human reviewed."]]}
+    },
     {
       "id": "2026-09-20-wcxb-measured-mode-labels",
       "comparison_kinds": ["wcxb_frozen_html_to_markdown"],