' if unique else ''
+ cells.append(' ' + marker + f'{number:,.0f} ms' + ('' if unique else '') + ' ')
+ rows.append('' + esc(labels[provider]) + ' 545/545 ' + ''.join(cells) + '' + prose('home_title') + ' ' + prose('home_lead') + '
' + prose('infographic_caption') + ' 核心结果 ' + metric_table() + '' + prose('score_note') + '
全部页面的调用耗时 ' + all_call_latency_table() + '' + prose('all_latency_note') + '
成功页面的调用耗时与重试成本 ' + latency_table() + '' + prose('latency_note') + '
问题分布 ' + table(["自动诊断", "结果数"], ['' + anchor("analysis/index.html?kind=" + kind, kind) + ' ' + str(count) + ' ' + prose('diagnostic_note') + '
" + anchor("cases/index.html?id=" + case_id, case_id) + " " +
+ esc(meta.get("language") or "—") + " " + esc(meta.get("level") or "—") +
+ ' ' + anchor(case["url"], case["url"]) + " " +
+ anchor("viewer/index.html?path=sources/" + case_id + ".html.txt", "冻结网页") + " · " +
+ anchor("viewer/index.html?path=references/" + case_id + ".md", "人工答案") + " 冻结网页与人工答案 ' + prose('dataset_lead') + '
搜索任务或网址 方法全部方法 ' +
+ ''.join('' + esc(labels[provider]) + " " for provider in providers) +
+ ' 诊断' +
+ ''.join('' + esc("全部问题" if kind == "issue" else "全部结果" if kind == "all" else kind) + " " for kind in options) +
+ "
")
+ result_rows = []
+ for row in records:
+ case_id, provider = row["case_id"], row["provider"]
+ kind = categories[row["attempt_id"]]
+ result_rows.append('' +
+ anchor("results/index.html?provider=" + provider + "&id=" + case_id, case_id) + " " +
+ esc(labels[provider]) + " " + esc(kind) + " " +
+ percent(score(row, "text_edit")) + " " + f'{row["elapsed_ms"]:,.0f} ms' +
+ " " + anchor("cases/index.html?id=" + case_id, "同页") + " 逐项结果与问题定位 ' + prose('analysis_lead') + '
当前显示 当前条件无结果。
' +
+ table(["任务", "方法", "诊断", "全文分", "最终调用", "同页"], result_rows))
+ page("cases/index.html", "同页比较",
+ ''
+ '冻结网页 人工答案 ' + anchor("dataset/index.html", "返回数据集") + '
'
+ '')
+ page("results/index.html", "逐项结果",
+ '')
+ shards = defaultdict(lambda: {"results": {}, "artifacts": {}, "blobs": {}, "physical": {}})
+ result_index = {}
+ artifact_index = {}
+ def shard_name(provider, case_id):
+ bucket = int(hashlib.sha256(case_id.encode()).hexdigest()[:2], 16) % 8
+ return f"shards/{provider}-{bucket}.json"
+ def bound_artifact(path, expected, shard_path):
+ source = contained_path(out, path)
+ raw = source.read_bytes()
+ if hashlib.sha256(raw).hexdigest() != expected:
+ raise ValueError("Public artifact changed: " + path)
+ shards[shard_path]["artifacts"][path] = expected
+ shards[shard_path]["blobs"][expected] = raw.decode("utf-8")
+ artifact_index[path] = shard_path
+ for row in records:
+ key = row["provider"] + "|" + row["case_id"]
+ target = shard_name(row["provider"], row["case_id"])
+ shards[target]["results"][key] = {**row, "diagnosis": categories[row["attempt_id"]],
+ "review": reviewed_failures.get((row["provider"], row["case_id"]))}
+ result_index[key] = target
+ bound_artifact(row["artifact_path"], row["output_sha256"], target)
+ physical_path = out / "physical-attempts.jsonl"
+ physical = [json.loads(line) for line in physical_path.read_text(encoding="utf-8").splitlines() if line.strip()] if physical_path.is_file() else []
+ for attempt in physical:
+ case_id = attempt["track_id"]
+ target = shard_name(attempt["provider"], case_id)
+ shards[target]["physical"].setdefault(case_id, []).append(attempt)
+ bound_artifact(attempt["physical_artifact_path"], attempt["output_sha256"], target)
+ for case in dataset["cases"]:
+ case_id = case["id"]
+ target = shard_name("source", case_id)
+ bound_artifact(f"sources/{case_id}.html.txt", case["input_html_sha256"], target)
+ bound_artifact(f"references/{case_id}.md", case["reference_sha256"], target)
+ for relative, data in shards.items():
+ target = out / relative
+ target.parent.mkdir(parents=True, exist_ok=True)
+ target.write_text(json.dumps(data, ensure_ascii=False, separators=(",", ":")), encoding="utf-8")
+ (out / "result-shard-index.json").write_text(json.dumps(result_index, separators=(",", ":")), encoding="utf-8")
+ (out / "artifact-shard-index.json").write_text(json.dumps(artifact_index, separators=(",", ":")), encoding="utf-8")
+ shutil.rmtree(out / "artifacts")
+ shutil.rmtree(out / "sources")
+ shutil.rmtree(out / "references")
+ interface_rows = ['' + esc(labels[provider]) + ' ' + esc(content['interfaces'][provider]) + '测量与评分方法 ' + prose('method_lead') + '
输入 ' + prose('method_input') + '
数据 SHA-256：' +
+ esc(profile["source_dataset_sha256"]) + '。评分器提交：' + esc(profile["official_scorer_commit"]) +
+ '。
失败重试 本轮共 ' + str(len(records) + retry_count) +
+ ' 次物理调用，包含 ' + str(retry_count) + ' 次失败后的重试；其中 ' + str(recovered_count) +
+ ' 条重试后返回成功，' + str(final_request_failures) +
+ ' 条最终请求失败均调用两次。返回成功但正文为空或为错误页的结果不触发请求失败重试。
评分 ' + prose('method_scoring') + '
' + prose('method_correction') + '
' +
+ anchor("teds-correction.json", "修正收据") + " · " + anchor("teds-correction-review.json", "修正复核") + "调用接口 ' + table(['方法', '本轮调用方式'], interface_rows) + '' + prose('kitesurf_version_note') + '
' +
+ '公开采用证据 ' + prose('adoption_lead') + ' 截至 ' + esc(adoption['cutoff'][:10]) +
+ '，Cloudflare Markdown 接口在筛选范围内见于 ' + str(cloudflare_repositories) + ' 个仓库、' + str(len(cloudflare_matches)) +
+ ' 个文件。
' + prose('adoption_limit') + '
' + anchor('provider-adoption.json', '查看检索样本与纳入规则') + '" + anchor("analysis/index.html?provider=" + provider, labels[provider]) +
+ " " + str(summaries[provider]["request_successes"]) +
+ "/545 " + str(issues) + " " +
+ prose("guide_method_" + provider) + " " + esc(labels[provider]) + " " + "".join(
+ '' +
+ anchor("analysis/index.html?provider=" + provider + "&kind=" + kind,
+ str(sum(categories[row["attempt_id"]] == kind for row in selected))) + " "
+ for kind in diagnostic_kinds) + "结果解读 ' + esc(outcome) + " " + prose('guide_lead') + '
质量与样本 ' +
+ metric_table() + '' + prose('guide_scope') + '
核对完整输出 ' + prose('guide_outputs') + '
' +
+ anchor("analysis/index.html?kind=低全文相似度", "查看低全文分案例") +
+ '可检查的一页 ' + prose('guide_example') + '
' + example_links + '
' +
+ '各方法的主要问题 ' +
+ table(["方法", "返回成功", "自动诊断数", "本轮观察"], method_rows) +
+ '' + prose('guide_method_scope') + '
' +
+ table(["方法", *diagnostic_kinds], diagnostic_rows) + '错误类别 ' +
+ table(["类别", "结果数"], ["" + esc(kind) + " " + str(count) + " ' + prose('guide_diagnosis') + '
已复核的机器问题原因 ' +
+ table(["原因", "结果数"], ["" + esc(kind) + " " + str(count) + " ' + str(len(reviewed_failures)) + '/' + str(machine_issue_count) + ' ' + prose('review_scope_failure') +
+ '；' + str(len(records) - len(reviewed_failures)) + ' ' + prose('review_scope_pass') + '
' +
+ anchor('failure-analysis.json', '查看逐条失败原因与处理') +
+ '结论 ' + prose('guide_conclusion') + '
复现依据 ' + prose('reproduce_lead') + '
' +
+ " · ".join(anchor(name, name) for name in evidence_links) + "
")
+ page("viewer/index.html", "证据查看器",
+ '')
+ page("en/index.html", "英文版未生成",
+ '' +
+ anchor("index.html", "返回中文报告"))
+ (out / "report-manifest.json").write_text(json.dumps({
+ "schema_version": 1, "report": "webmainbench-545-full", "languages": ["zh"],
+ "case_count": 545, "provider_count": len(providers), "result_count": len(records),
+ }, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
diff --git a/harness/web_fetch/report_build.py b/harness/web_fetch/report_build.py
index 6809d422e..da2e75f70 100644
--- a/harness/web_fetch/report_build.py
+++ b/harness/web_fetch/report_build.py
@@ -5,7 +5,7 @@
import tempfile
from pathlib import Path
from harness.reporting.assets import copy_bound_assets
-from .report import generate, generate_public_benchmark_research, generate_webmainbench, generate_wcxb, succeeded, measured, failure_category, TRACKS
+from .report import generate, generate_public_benchmark_research, generate_webmainbench, generate_webmainbench_full, webmainbench_full_machine_issue, generate_wcxb, succeeded, measured, failure_category, TRACKS
from .report_changes import webmainbench_history
from .failure_diagnosis import diagnose
from .safety import contained_path, safe_identifier, validate_report_ids
@@ -501,6 +501,108 @@ def build_webmainbench_report(source: Path, output: Path, *, include_english: bo
return output / 'index.html'
+def build_webmainbench_full_report(source: Path, output: Path) -> Path:
+ """Build the separately bound complete 545-case report from certified evidence."""
+ from scripts.run_certified_webmainbench_full import digest, read_jsonl
+ from .webmainbench_diagnostics import validate_teds_revision
+ if source.is_symlink() or any(path.is_symlink() for path in source.rglob('*')):
+ raise ValueError('Symlinked report evidence is forbidden')
+ profile_path = Path(__file__).resolve().parents[2] / 'evaluation/webmainbench-full/current-report.json'
+ profile = json.loads(profile_path.read_text(encoding='utf-8'))
+ certificate = json.loads((source / 'cohort-certificate.json').read_text(encoding='utf-8'))
+ scoring_certificate = json.loads((source / 'scoring-certificate.json').read_text(encoding='utf-8'))
+ if (certificate.get('kind') != 'certified_webmainbench_545_full_v1'
+ or certificate.get('profile_sha256') != digest(profile_path)
+ or certificate.get('preflight') != certificate.get('postflight')
+ or json.loads((source / 'current-report.json').read_text(encoding='utf-8')) != profile):
+ raise ValueError('Full-cohort certificate or profile mismatch')
+ if (scoring_certificate.get('kind') != 'certified_webmainbench_scoring_v1'
+ or scoring_certificate.get('cohort_certificate_sha256') != digest(source / 'cohort-certificate.json')
+ or scoring_certificate.get('source_binding') != certificate['postflight']):
+ raise ValueError('Complete scoring certificate mismatch')
+ dataset = json.loads((source / 'dataset.json').read_text(encoding='utf-8'))
+ records = read_jsonl(source / 'results.jsonl')
+ providers = profile['providers']
+ cases = {case['id']: case for case in dataset['cases']}
+ if (len(cases) != 545 or len(records) != 545 * len(providers)
+ or {(row['provider'], row['case_id']) for row in records} !=
+ {(provider, case_id) for provider in providers for case_id in cases}):
+ raise ValueError('Full WebMainBench result matrix mismatch')
+ validate_teds_revision(source)
+ content_binding_path = Path(__file__).resolve().parents[2] / 'evaluation/webmainbench-full/content-binding.json'
+ content_binding = json.loads(content_binding_path.read_text(encoding='utf-8'))
+ content_path = data_home() / 'report-assets/webmainbench-full' / content_binding['relative_path']
+ if digest(content_path) != content_binding['sha256']:
+ raise ValueError('Report prose differs from the compact content binding')
+ illustration_path = data_home() / 'report-assets/webmainbench-full' / content_binding['infographic']
+ if digest(illustration_path) != content_binding['infographic_sha256']:
+ raise ValueError('Report illustration differs from the compact asset binding')
+ mobile_illustration_path = data_home() / 'report-assets/webmainbench-full' / content_binding['infographic_mobile']
+ if digest(mobile_illustration_path) != content_binding['infographic_mobile_sha256']:
+ raise ValueError('Mobile report illustration differs from the compact asset binding')
+ failure_binding_path = Path(__file__).resolve().parents[2] / 'evaluation/webmainbench-full/failure-analysis-binding.json'
+ failure_binding = json.loads(failure_binding_path.read_text(encoding='utf-8'))
+ failure_path = data_home() / 'report-assets/webmainbench-full' / failure_binding['relative_path']
+ if digest(failure_path) != failure_binding['sha256']:
+ raise ValueError('Failure analysis differs from the compact review binding')
+ failure_analysis = json.loads(failure_path.read_text(encoding='utf-8'))
+ failures = {(row['provider'], row['case_id']) for row in records
+ if webmainbench_full_machine_issue(source, row)}
+ reviewed = {(row['provider'], row['case_id']) for row in failure_analysis['reviewed_failures']}
+ if (failure_analysis.get('kind') != 'webmainbench_full_failure_analysis_v1'
+ or failure_analysis.get('cohort_certificate_sha256') != digest(source / 'cohort-certificate.json')
+ or reviewed != failures or len(reviewed) != len(failure_analysis['reviewed_failures'])
+ or any(not all(isinstance(row.get(key), str) and row[key].strip()
+ for key in ('category_label', 'cause', 'treatment', 'status'))
+ for row in failure_analysis['reviewed_failures'])):
+ raise ValueError('Failure analysis does not cover every measured request failure')
+ for case_id, case in cases.items():
+ if digest(contained_path(source, f'references/{case_id}.md')) != case['reference_sha256']:
+ raise ValueError('Reference changed: ' + case_id)
+ if digest(contained_path(source, f'sources/{case_id}.html.txt')) != case['input_html_sha256']:
+ raise ValueError('Frozen page changed: ' + case_id)
+ for row in records:
+ if digest(contained_path(source, row['artifact_path'])) != row['output_sha256']:
+ raise ValueError('Output changed: ' + row['attempt_id'])
+ output.parent.mkdir(parents=True, exist_ok=True)
+ staging = Path(tempfile.mkdtemp(prefix='.webmainbench-full-build-', dir=output.parent))
+ try:
+ for name in ('artifacts', 'references', 'sources', 'fixture-certificates', 'correction-source'):
+ shutil.copytree(source / name, staging / name)
+ for name in ('dataset.json', 'results.jsonl', 'original-results.jsonl',
+ 'physical-attempts.jsonl', 'execution.json', 'score-summaries.json',
+ 'cohort-certificate.json', 'current-report.json',
+ 'scoring-certificate.json',
+ 'teds-correction.json', 'teds-correction-review.json'):
+ shutil.copy2(source / name, staging / name)
+ shutil.copy2(content_path, staging / 'report-content.json')
+ shutil.copy2(failure_path, staging / 'failure-analysis.json')
+ shutil.copy2(Path(__file__).resolve().parents[2] / 'research/provider-adoption.json', staging / 'provider-adoption.json')
+ (staging / 'assets').mkdir(exist_ok=True)
+ shutil.copy2(illustration_path, staging / 'assets/infographic.png')
+ shutil.copy2(mobile_illustration_path, staging / 'assets/infographic-mobile.png')
+ generate_webmainbench_full(staging, dataset, records,
+ json.loads((source / 'score-summaries.json').read_text(encoding='utf-8')),
+ profile)
+ from scripts.check_webmainbench_full_report import check as check_webmainbench_full
+ check_webmainbench_full(staging)
+ if output.exists():
+ backup = Path(tempfile.mkdtemp(prefix='.report-swap-', dir=output.parent)) / 'report'
+ output.rename(backup)
+ try:
+ staging.rename(output)
+ except BaseException:
+ backup.rename(output)
+ raise
+ shutil.rmtree(backup.parent)
+ else:
+ staging.rename(output)
+ finally:
+ if staging.exists():
+ shutil.rmtree(staging)
+ return output / 'index.html'
+
+
def build_wcxb_report(source: Path, corpus: Path, sample_path: Path, profile: dict, output: Path) -> Path:
"""Build the one current report from a fully certified WCXB matrix."""
from scripts.run_webmainbench_benchmark import WCXB_PROVIDERS, load_wcxb_source
diff --git a/research/report-changes.json b/research/report-changes.json
index 6b3f6f07c..5aefe4b58 100644
--- a/research/report-changes.json
+++ b/research/report-changes.json
@@ -1275,6 +1275,17 @@
}
],
"releases": [
+ {
+ "id": "2026-09-23-webmainbench-complete-545-five-methods",
+ "comparison_kinds": ["webmainbench_frozen_html_to_markdown"],
+ "date": "2026-09-23",
+ "title": "发布 WebMainBench 完整 545 题五方法对照",
+ "title_i18n": {"en": "Publish the complete 545-case WebMainBench comparison"},
+ "summary": "以同一批 545 个官方冻结 HTML 和人工参考，实测五种稳定版浏览器方法。新报告提供逐题输入、输出、评分、失败证据和已复核的机器问题分析。",
+ "summary_i18n": {"en": "Five stable browser methods were measured against the same 545 official frozen HTML inputs and human references. The new report exposes case inputs, outputs, scores, failure evidence, and reviewed machine-issue analysis."},
+ "changes": [["完整评测与评分", "先前公开报告只覆盖 200 题，无法代表完整数据集。官方 TEDS 实现还可能高估部分表格相似度。", "认证五种方法各 545 题，并用保留原始输出和官方分数的结构化树比较校正 TEDS；报告同时展示请求重试和失败分类。", "共 2725 个逻辑结果、2747 次实际调用；292 个 TEDS 分数校正。28 个机器问题已逐条复核，其余 2697 个结果未逐条人工复核。"]],
+ "changes_i18n": {"en": [["Full evaluation and scoring", "The earlier public report covered only 200 cases, and the official TEDS implementation could overstate some table similarities.", "Certify 545 cases for each of five methods and correct TEDS with structured tree comparison while preserving raw outputs and official scores; show retries and failure classes.", "There are 2,725 logical results and 2,747 physical calls; 292 TEDS scores were corrected. All 28 machine issues were reviewed individually, while the other 2,697 results were not individually human reviewed."]]}
+ },
{
"id": "2026-09-20-wcxb-measured-mode-labels",
"comparison_kinds": ["wcxb_frozen_html_to_markdown"],
diff --git a/scripts/assemble_webmainbench_full.py b/scripts/assemble_webmainbench_full.py
new file mode 100644
index 000000000..177923d31
--- /dev/null
+++ b/scripts/assemble_webmainbench_full.py
@@ -0,0 +1,161 @@
+"""Package only certified, scored WebMainBench-545 evidence for the full report."""
+from __future__ import annotations
+
+import argparse
+import hashlib
+import json
+import re
+import shutil
+import sys
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parents[1]
+sys.path.insert(0, str(ROOT))
+
+from harness.web_fetch.safety import contained_path
+from scripts.run_certified_webmainbench_full import digest, read_jsonl, verify_shard
+from scripts.run_webmainbench_benchmark import load_source
+
+
+def public_value(value):
+ """Remove machine-specific absolute paths from the public evidence copy."""
+ if isinstance(value, dict):
+ return {key: public_value(item) for key, item in value.items()}
+ if isinstance(value, list):
+ return [public_value(item) for item in value]
+ if isinstance(value, str):
+ value = re.sub(r"/Users/[^/\s:]+", "/redacted-home", value)
+ value = re.sub(r"/private/tmp/[^\s:]+", "/redacted-temp", value)
+ return value
+
+
+def copy_bound(root: Path, output: Path, relative: str, expected: str) -> None:
+ source = contained_path(root, relative)
+ if (root / relative).is_symlink() or digest(source) != expected:
+ raise ValueError("Evidence artifact changed: " + relative)
+ target = contained_path(output, relative)
+ target.parent.mkdir(parents=True, exist_ok=True)
+ shutil.copy2(source, target)
+
+
+def assemble(root: Path, output: Path, source: Path, profile_path: Path) -> None:
+ if output.exists():
+ raise ValueError("Output already exists; report evidence is immutable")
+ if root.is_symlink() or source.is_symlink() or profile_path.is_symlink():
+ raise ValueError("Symlinked aggregate input is forbidden")
+ profile = json.loads(profile_path.read_text(encoding="utf-8"))
+ cert_path = root / "cohort-certificate.json"
+ certificate = json.loads(cert_path.read_text(encoding="utf-8"))
+ scoring_certificate_path = root / "scoring-certificate.json"
+ scoring_certificate = json.loads(scoring_certificate_path.read_text(encoding="utf-8"))
+ if (certificate.get("kind") != "certified_webmainbench_545_full_v1"
+ or certificate.get("profile_sha256") != digest(profile_path)
+ or certificate.get("preflight") != certificate.get("postflight")
+ or certificate["preflight"]["source_sha256"] != profile["source_dataset_sha256"]
+ or [row["provider"] for row in certificate["providers"]] != profile["providers"]):
+ raise ValueError("Complete-cohort certificate or profile mismatch")
+ if (scoring_certificate.get("kind") != "certified_webmainbench_scoring_v1"
+ or scoring_certificate.get("cohort_certificate_sha256") != digest(cert_path)
+ or scoring_certificate.get("source_binding") != certificate["postflight"]
+ or [row["provider"] for row in scoring_certificate["providers"]] != profile["providers"]):
+ raise ValueError("Complete scoring certificate differs from the certified cohort")
+ source_records = load_source(source)
+ ids = {row["track_id"] for row in source_records}
+ if len(ids) != 545 or len(source_records) != profile["case_count"]:
+ raise ValueError("Source case inventory mismatch")
+ results: list[dict] = []
+ physical: list[dict] = []
+ summaries: dict[str, dict] = {}
+ executions: list[dict] = []
+ output.mkdir(parents=True)
+ (output / "references").mkdir()
+ (output / "sources").mkdir()
+ (output / "fixture-certificates").mkdir()
+ for record in source_records:
+ case_id = record["track_id"]
+ (output / "references" / (case_id + ".md")).write_text(
+ record["groundtruth_content"], encoding="utf-8")
+ (output / "sources" / (case_id + ".html.txt")).write_text(
+ record["html"], encoding="utf-8")
+ for expected, scored_receipt in zip(certificate["providers"], scoring_certificate["providers"]):
+ provider = expected["provider"]
+ shard = root / provider
+ if shard.is_symlink() or any(path.is_symlink() for path in shard.rglob("*")):
+ raise ValueError(provider + ": symlinked evidence is forbidden")
+ if verify_shard(shard, provider, ids, profile) != expected:
+ raise ValueError(provider + ": certificate differs from measured shard")
+ rows = read_jsonl(shard / "scored-results.jsonl")
+ if (scored_receipt["scored_results_sha256"] != digest(shard / "scored-results.jsonl")
+ or scored_receipt["score_summary_sha256"] != digest(shard / "score-summary.json")
+ or scored_receipt["logical_results"] != profile["case_count"]):
+ raise ValueError(provider + ": scoring certificate differs from scored shard")
+ if (len(rows) != 545 or {row["track_id"] for row in rows} != ids
+ or len({row["track_id"] for row in rows}) != 545
+ or any(row["provider"] != provider for row in rows)):
+ raise ValueError(provider + ": incomplete scored matrix")
+ summary = json.loads((shard / "score-summary.json").read_text(encoding="utf-8"))
+ if (summary.get("case_count") != 545 or summary.get("provider") != provider
+ or summary.get("scorer", {}).get("metric_source_sha256") !=
+ scoring_certificate["scorer_source_sha256"]):
+ raise ValueError(provider + ": score summary scope mismatch")
+ logical = {row["track_id"]: row for row in read_jsonl(shard / "results.jsonl")}
+ for row in rows:
+ base = logical[row["track_id"]]
+ if {k: v for k, v in row.items() if k != "metrics"} != base:
+ raise ValueError(provider + ": scoring changed measured result")
+ copy_bound(shard, output, row["artifact_path"], row["output_sha256"])
+ results.append(public_value({**row, "case_id": row["track_id"]}))
+ attempts = read_jsonl(shard / "physical-attempts.jsonl")
+ for attempt in attempts:
+ copy_bound(shard, output, attempt["physical_artifact_path"], attempt["output_sha256"])
+ physical.extend(public_value(attempts))
+ execution = json.loads((shard / "execution.json").read_text(encoding="utf-8"))
+ ca = execution.get("fixture_ca_artifact")
+ if ca:
+ ca_source = contained_path(shard, ca)
+ if digest(ca_source) != execution["network_isolation_probe"]["fixture_ca_sha256"]:
+ raise ValueError(provider + ": fixture CA changed")
+ ca_target = output / "fixture-certificates" / (provider + "-ca.pem")
+ shutil.copy2(ca_source, ca_target)
+ execution["assembled_fixture_ca_path"] = str(ca_target.relative_to(output))
+ executions.append(execution)
+ summaries[provider] = summary
+ expected_pairs = {(provider, case_id) for provider in profile["providers"] for case_id in ids}
+ if (len(results) != len(expected_pairs)
+ or {(row["provider"], row["case_id"]) for row in results} != expected_pairs
+ or len({row["attempt_id"] for row in results}) != len(results)
+ or len({row["attempt_id"] for row in physical}) != len(physical)):
+ raise ValueError("Full result or physical attempt matrix mismatch")
+ cases = [{"id": row["track_id"], "name": row["track_id"], "url": row["url"],
+ "category": "webmainbench", "track": "static_html", "meta": row["meta"],
+ "input_html_sha256": hashlib.sha256(row["html"].encode()).hexdigest(),
+ "reference_sha256": hashlib.sha256(row["groundtruth_content"].encode()).hexdigest(),
+ "reference_chars": len(row["groundtruth_content"])} for row in source_records]
+ dataset = {"schema_version": 1, "tool": "web_fetch", "id": profile["dataset_id"],
+ "name": "WebMainBench-545 full · five Web Fetch methods",
+ "source_dataset_sha256": profile["source_dataset_sha256"],
+ "cases": cases, "coverage_plan": {"static_html": profile["providers"]},
+ "comparison": profile}
+ (output / "dataset.json").write_text(json.dumps(dataset, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
+ (output / "score-summaries.json").write_text(json.dumps(summaries, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
+ (output / "execution.json").write_text(json.dumps({"batches": executions}, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
+ (output / "results.jsonl").write_text("".join(json.dumps(row, ensure_ascii=False) + "\n" for row in results), encoding="utf-8")
+ (output / "physical-attempts.jsonl").write_text("".join(json.dumps(row, ensure_ascii=False) + "\n" for row in physical), encoding="utf-8")
+ shutil.copy2(cert_path, output / "cohort-certificate.json")
+ shutil.copy2(scoring_certificate_path, output / "scoring-certificate.json")
+ shutil.copy2(profile_path, output / "current-report.json")
+ print(f"Assembled {len(results)} results and {len(physical)} physical calls", flush=True)
+
+
+def main() -> None:
+ parser = argparse.ArgumentParser(description=__doc__)
+ parser.add_argument("--source-root", type=Path, required=True)
+ parser.add_argument("--output", type=Path, required=True)
+ parser.add_argument("--dataset", type=Path, required=True)
+ parser.add_argument("--profile", type=Path, default=ROOT / "evaluation/webmainbench-full/current-report.json")
+ args = parser.parse_args()
+ assemble(args.source_root, args.output, args.dataset, args.profile)
+
+
+if __name__ == "__main__":
+ main()
diff --git a/scripts/build_report.py b/scripts/build_report.py
index 2836af831..d69b02f55 100644
--- a/scripts/build_report.py
+++ b/scripts/build_report.py
@@ -3,7 +3,7 @@
from pathlib import Path
ROOT=Path(__file__).resolve().parents[1]
sys.path.insert(0,str(ROOT))
-from harness.web_fetch.report_build import build_report, build_public_benchmark_research, build_wcxb_report
+from harness.web_fetch.report_build import build_report, build_public_benchmark_research, build_wcxb_report, build_webmainbench_full_report
from harness.web_fetch.storage import (
current_evidence_dir,
require_canonical_inputs,
@@ -21,7 +21,13 @@
parser.add_argument('--wcxb-source',type=Path)
parser.add_argument('--wcxb-sample',type=Path)
parser.add_argument('--wcxb',action='store_true',help='Build the independently bound WCXB report')
+ parser.add_argument('--webmainbench-full',action='store_true',help='Build the separately certified complete 545-case report')
args=parser.parse_args()
+ if args.webmainbench_full:
+ if not args.evidence or args.include_english:
+ parser.error('--webmainbench-full requires --evidence and generates Chinese only')
+ print(build_webmainbench_full_report(args.evidence,args.output or ROOT/'reports/webmainbench-full'))
+ raise SystemExit(0)
if args.public_benchmark_research:
if args.evidence:
parser.error('--evidence and --public-benchmark-research are mutually exclusive')
diff --git a/scripts/check_webmainbench_full_report.py b/scripts/check_webmainbench_full_report.py
new file mode 100644
index 000000000..630ef7226
--- /dev/null
+++ b/scripts/check_webmainbench_full_report.py
@@ -0,0 +1,166 @@
+"""Release gate for the separately certified complete WebMainBench report."""
+from __future__ import annotations
+
+import argparse
+from collections import Counter, defaultdict
+import hashlib
+import json
+from pathlib import Path
+from bs4 import BeautifulSoup
+
+from scripts.report_presentation_contract import check_presentation
+from harness.web_fetch.safety import contained_path
+from harness.web_fetch.report import ERROR_PAGE
+
+
+def check_result_payload(row: dict, index: dict, artifact_index: dict, shards: dict) -> None:
+ key = row["provider"] + "|" + row["case_id"]
+ shard_path = index.get(key)
+ if shard_path not in shards:
+ raise ValueError("Result shard missing: " + key)
+ bundled = shards[shard_path]["results"].get(key)
+ if not bundled or {name: value for name, value in bundled.items() if name not in ("diagnosis", "review")} != row:
+ raise ValueError("Result shard differs from scored evidence: " + key)
+ artifact = row["artifact_path"]
+ if artifact_index.get(artifact) != shard_path:
+ raise ValueError("Evidence target points to wrong shard: " + artifact)
+ digest = shards[shard_path]["artifacts"].get(artifact)
+ value = shards[shard_path]["blobs"].get(digest)
+ if not isinstance(value, str) or hashlib.sha256(value.encode()).hexdigest() != row["output_sha256"]:
+ raise ValueError("Raw output absent or changed: " + artifact)
+
+
+def check(root: Path) -> None:
+ binding_path = Path(__file__).resolve().parents[1] / "evaluation/webmainbench-full/content-binding.json"
+ binding = json.loads(binding_path.read_text(encoding="utf-8"))
+ for path, expected in ((root / "report-content.json", binding["sha256"]),
+ (root / "assets/infographic.png", binding["infographic_sha256"]),
+ (root / "assets/infographic-mobile.png", binding["infographic_mobile_sha256"])):
+ if hashlib.sha256(path.read_bytes()).hexdigest() != expected:
+ raise ValueError("Bound report asset changed: " + path.name)
+ failure_binding = json.loads((Path(__file__).resolve().parents[1] / "evaluation/webmainbench-full/failure-analysis-binding.json").read_text(encoding="utf-8"))
+ failure_bytes = (root / "failure-analysis.json").read_bytes()
+ if hashlib.sha256(failure_bytes).hexdigest() != failure_binding["sha256"]:
+ raise ValueError("Bound failure analysis changed")
+ failure_analysis = json.loads(failure_bytes)
+ manifest = json.loads((root / "report-manifest.json").read_text(encoding="utf-8"))
+ if manifest != {"schema_version": 1, "report": "webmainbench-545-full",
+ "languages": ["zh"], "case_count": 545, "provider_count": 5,
+ "result_count": 2725}:
+ raise ValueError("Incomplete WebMainBench report manifest")
+ dataset = json.loads((root / "dataset.json").read_text(encoding="utf-8"))
+ records = [json.loads(line) for line in (root / "results.jsonl").read_text(encoding="utf-8").splitlines()]
+ providers = dataset["coverage_plan"]["static_html"]
+ case_ids = {row["id"] for row in dataset["cases"]}
+ if len(case_ids) != 545 or len(records) != 2725 or len(providers) != 5:
+ raise ValueError("Full matrix missing")
+ expected = {(provider, case_id) for provider in providers for case_id in case_ids}
+ if {(row["provider"], row["case_id"]) for row in records} != expected:
+ raise ValueError("Report matrix differs from evidence")
+ reviewed = {(row["provider"], row["case_id"]) for row in failure_analysis["reviewed_failures"]}
+ if (len(reviewed) != len(failure_analysis["reviewed_failures"])
+ or any(not all(isinstance(row.get(key), str) and row[key].strip()
+ for key in ("category_label", "cause", "treatment", "status"))
+ for row in failure_analysis["reviewed_failures"])):
+ raise ValueError("Published failure analysis contains duplicate or empty reviews")
+ home = check_presentation((root / "index.html").read_text(encoding="utf-8"))
+ guide = check_presentation((root / "guide/index.html").read_text(encoding="utf-8"))
+ for page in (home, guide):
+ if len(page.select('td[data-metric-direction="max"]')) != 30:
+ raise ValueError("Core metrics or winner controls missing")
+ if "545" not in page.get_text(" ", strip=True):
+ raise ValueError("Complete population missing")
+ if len(home.select('td[data-metric-direction="min"]')) != 35:
+ raise ValueError("All-call or conditional latency comparison missing")
+ first_case = dataset["cases"][0]["id"]
+ example_targets = {"viewer/index.html?path=sources/" + first_case + ".html.txt",
+ "viewer/index.html?path=references/" + first_case + ".md",
+ "cases/index.html?id=" + first_case}
+ if not example_targets.issubset({link.get("href") for link in guide.select("a[href]")}):
+ raise ValueError("Interpretation example lacks source, reference or same-case result")
+ if not home.select_one('img[src="assets/infographic.png"]') or not (root / "assets/infographic.png").is_file():
+ raise ValueError("Bound infographic missing")
+ if not home.select_one('source[media="(max-width:650px)"][srcset="assets/infographic-mobile.png"]'):
+ raise ValueError("Mobile infographic route missing")
+ analysis = BeautifulSoup((root / "analysis/index.html").read_text(encoding="utf-8"), "html.parser")
+ if len(analysis.select("tr[data-result]")) != 2725 or not analysis.select_one("#kind"):
+ raise ValueError("Result inventory incomplete")
+ if not any(row.get("data-kind") != "未触发诊断" for row in analysis.select("tr[data-result]")):
+ raise ValueError("Default diagnostic population is empty")
+ if not (root / "en/index.html").is_file() or "返回中文报告" not in (root / "en/index.html").read_text(encoding="utf-8"):
+ raise ValueError("English not-generated route missing")
+ case_template = BeautifulSoup((root / "cases/index.html").read_text(encoding="utf-8"), "html.parser")
+ if not case_template.select_one("#case-results") or not case_template.select_one("#case-source"):
+ raise ValueError("Same-case comparison template is incomplete")
+ result_template = check_presentation((root / "results/index.html").read_text(encoding="utf-8"))
+ if not result_template.select_one('[data-result-summary]') or not result_template.select_one('#result-logs pre[data-kind="error"]'):
+ raise ValueError("Shared result template lacks summary or exact error log")
+ index = json.loads((root / "result-shard-index.json").read_text(encoding="utf-8"))
+ artifact_index = json.loads((root / "artifact-shard-index.json").read_text(encoding="utf-8"))
+ shard_paths = set(index.values()) | set(artifact_index.values())
+ shards = {path: json.loads(contained_path(root, path).read_text(encoding="utf-8")) for path in shard_paths}
+ if len(index) != len(records) or len(shards) > 48:
+ raise ValueError("Complete result shard index missing or oversized")
+ failures = set()
+ diagnostic_counts = defaultdict(Counter)
+ for row in records:
+ shard = shards[index[row["provider"] + "|" + row["case_id"]]]
+ diagnosis = shard["results"][row["provider"] + "|" + row["case_id"]]["diagnosis"]
+ diagnostic_counts[row["provider"]][diagnosis] += 1
+ path = row["artifact_path"]
+ output = shard["blobs"].get(shard["artifacts"].get(path), "")
+ if not row["ok"] or not row["output_chars"] or ERROR_PAGE.search(output):
+ failures.add((row["provider"], row["case_id"]))
+ if reviewed != failures:
+ raise ValueError("Published failure analysis scope is incomplete")
+ cells = guide.select("td[data-diagnostic-provider][data-kind]")
+ expected_cells = {(provider, kind) for provider in providers
+ for kind in ("请求失败", "空输出", "错误页", "低全文相似度")}
+ if ({(cell["data-diagnostic-provider"], cell["data-kind"]) for cell in cells} != expected_cells
+ or len(cells) != len(expected_cells) or any(
+ int(cell.get_text(strip=True)) != diagnostic_counts[cell["data-diagnostic-provider"]][cell["data-kind"]]
+ for cell in cells
+ )):
+ raise ValueError("Interpretation diagnosis matrix differs from measured results")
+ reviewed_by_key = {(row["provider"], row["case_id"]): row for row in failure_analysis["reviewed_failures"]}
+ for row in records:
+ check_result_payload(row, index, artifact_index, shards)
+ bundled = shards[index[row["provider"] + "|" + row["case_id"]]]["results"][row["provider"] + "|" + row["case_id"]]
+ if bundled.get("review") != reviewed_by_key.get((row["provider"], row["case_id"])):
+ raise ValueError("Result detail differs from bound failure analysis: " + row["attempt_id"])
+ for case in dataset["cases"]:
+ case_id = case["id"]
+ for path, expected_hash in (("sources/" + case_id + ".html.txt", case["input_html_sha256"]),
+ ("references/" + case_id + ".md", case["reference_sha256"])):
+ shard_path = artifact_index.get(path)
+ if shard_path not in shards:
+ raise ValueError("Frozen task evidence shard missing: " + path)
+ blob_hash = shards[shard_path]["artifacts"].get(path)
+ content = shards[shard_path]["blobs"].get(blob_hash)
+ if not isinstance(content, str) or hashlib.sha256(content.encode()).hexdigest() != expected_hash:
+ raise ValueError("Frozen task evidence changed: " + path)
+ physical = [json.loads(line) for line in (root / "physical-attempts.jsonl").read_text(encoding="utf-8").splitlines() if line.strip()]
+ for attempt in physical:
+ path = attempt["physical_artifact_path"]
+ shard_path = artifact_index.get(path)
+ if shard_path not in shards:
+ raise ValueError("Physical output shard missing: " + path)
+ digest = shards[shard_path]["artifacts"].get(path)
+ content = shards[shard_path]["blobs"].get(digest)
+ if not isinstance(content, str) or hashlib.sha256(content.encode()).hexdigest() != attempt["output_sha256"]:
+ raise ValueError("Physical output absent or changed: " + path)
+ if attempt not in shards[shard_path]["physical"].get(attempt["track_id"], []):
+ raise ValueError("Physical call missing from result detail: " + attempt["attempt_id"])
+ public_files = [path for path in root.rglob("*") if path.is_file()]
+ if len(public_files) > 2000 or sum(path.stat().st_size for path in public_files) > 314572800:
+ raise ValueError("Report package exceeds Artifact Site deployment limits")
+ viewer = (root / "viewer/index.html").read_text(encoding="utf-8")
+ if "id=\"evidence-content\"" not in viewer or not (root / "assets/report.js").is_file():
+ raise ValueError("On-demand evidence viewer missing")
+
+
+if __name__ == "__main__":
+ parser = argparse.ArgumentParser(description=__doc__)
+ parser.add_argument("report", type=Path)
+ check(parser.parse_args().report)
+ print("Complete WebMainBench report release gate passed")
diff --git a/scripts/correct_webmainbench_teds.py b/scripts/correct_webmainbench_teds.py
index 0047ff157..226a4a44a 100644
--- a/scripts/correct_webmainbench_teds.py
+++ b/scripts/correct_webmainbench_teds.py
@@ -80,9 +80,9 @@ def main():
raise ValueError('Refusing symlinked correction evidence')
expected = {(provider, case['id']) for provider in dataset['coverage_plan']['static_html']
for case in dataset['cases']}
- if len(records) != len(expected) or len(expected) != 200 * len(summaries) or {
+ if len(records) != len(expected) or len(expected) != len(dataset['cases']) * len(summaries) or {
(row['provider'], row['case_id']) for row in records} != expected:
- raise ValueError(f'Correction requires the complete declared 200-by-{len(summaries)} matrix')
+ raise ValueError(f'Correction requires the complete declared {len(dataset["cases"])}-by-{len(summaries)} matrix')
# Verify the exact official implementation, not only its Git HEAD.
for summary in summaries.values():
for relative, expected in summary['scorer']['metric_source_sha256'].items():
@@ -175,7 +175,7 @@ def category(row):
'input_results_sha256': digest(source / 'results.jsonl'),
'output_results_sha256': digest(results_path),
'dataset_sha256': digest(source / 'dataset.json'),
- 'sample_manifest_sha256': dataset['sample_manifest_sha256'],
+ 'sample_manifest_sha256': dataset.get('sample_manifest_sha256'),
'inherited_correction_receipt_sha256': inherited_receipt_sha256,
'inherited_logical_results': len(inherited),
'source_code_sha256': {str(path.relative_to(ROOT)): digest(path) for path in source_files},
diff --git a/scripts/review_webmainbench_teds_correction.py b/scripts/review_webmainbench_teds_correction.py
index 55fba3e02..7c66a45d6 100644
--- a/scripts/review_webmainbench_teds_correction.py
+++ b/scripts/review_webmainbench_teds_correction.py
@@ -40,6 +40,8 @@ def review(source, corrected):
if len({row['attempt_id'] for row in after}) != len(after):
raise ValueError('Duplicate corrected attempt')
for name in ('dataset.json', 'sample-manifest.json', 'physical-attempts.jsonl', 'execution.json', 'result-review.json'):
+ if not (source / name).exists() and not (corrected / name).exists():
+ continue
if digest(source / name) != digest(corrected / name):
raise ValueError('Unexpected change: ' + name)
changed = []
@@ -76,7 +78,7 @@ def review(source, corrected):
impacts = {}
for provider, summary in summaries.items():
records = [row for row in after if row['provider'] == provider]
- if len(records) != 200:
+ if len(records) != len(json.loads((source / 'dataset.json').read_text())['cases']):
raise ValueError('Provider scope mismatch')
current = summary['official_aggregate']
for name in ('text_edit', 'code_edit', 'formula_edit', 'table_edit', 'table_TEDS'):
diff --git a/scripts/run_certified_webmainbench_full.py b/scripts/run_certified_webmainbench_full.py
new file mode 100644
index 000000000..0d539499f
--- /dev/null
+++ b/scripts/run_certified_webmainbench_full.py
@@ -0,0 +1,117 @@
+"""Run the predeclared complete WebMainBench cohort with source and result checks."""
+from __future__ import annotations
+
+import argparse
+import hashlib
+import json
+import subprocess
+import sys
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parents[1]
+sys.path.insert(0, str(ROOT))
+
+from scripts.run_webmainbench_benchmark import load_source
+
+
+def digest(path: Path) -> str:
+ return hashlib.sha256(path.read_bytes()).hexdigest()
+
+
+def read_jsonl(path: Path) -> list[dict]:
+ return [json.loads(line) for line in path.read_text(encoding="utf-8").splitlines() if line.strip()]
+
+
+def source_binding(source: Path, scorer: Path, profile: dict) -> dict:
+ if digest(source) != profile["source_dataset_sha256"]:
+ raise ValueError("WebMainBench source bytes changed")
+ records = load_source(source)
+ case_ids = [record["track_id"] for record in records]
+ case_digest = hashlib.sha256(("\n".join(case_ids) + "\n").encode()).hexdigest()
+ if len(records) != profile["case_count"] or case_digest != profile["case_ids_sha256"]:
+ raise ValueError("WebMainBench case inventory changed")
+ commit = subprocess.check_output(["git", "-C", str(scorer), "rev-parse", "HEAD"], text=True).strip()
+ if commit != profile["official_scorer_commit"]:
+ raise ValueError("Official scorer commit changed")
+ return {"source_sha256": digest(source), "case_ids_sha256": case_digest,
+ "scorer_commit": commit, "case_count": len(records)}
+
+
+def verify_shard(shard: Path, provider: str, case_ids: set[str], profile: dict) -> dict:
+ rows = read_jsonl(shard / "results.jsonl")
+ physical = read_jsonl(shard / "physical-attempts.jsonl")
+ execution = json.loads((shard / "execution.json").read_text(encoding="utf-8"))
+ dataset = json.loads((shard / "dataset.json").read_text(encoding="utf-8"))
+ if (len(rows) != profile["case_count"] or {row["track_id"] for row in rows} != case_ids
+ or len({row["track_id"] for row in rows}) != len(rows)
+ or dataset["case_count"] != profile["case_count"]
+ or set(dataset["case_ids"]) != case_ids
+ or execution["case_count"] != profile["case_count"]
+ or execution["provider"] != provider
+ or execution["source_dataset_sha256"] != profile["source_dataset_sha256"]):
+ raise ValueError(provider + ": incomplete or mismatched cohort")
+ attempt_ids = {row["attempt_id"] for row in physical}
+ if len(attempt_ids) != len(physical) or sum(row["physical_attempt_count"] for row in rows) != len(physical):
+ raise ValueError(provider + ": physical call inventory mismatch")
+ for row in rows:
+ path = shard / row["artifact_path"]
+ if path.is_symlink() or digest(path) != row["output_sha256"]:
+ raise ValueError(provider + ": result artifact mismatch: " + row["track_id"])
+ for row in physical:
+ path = shard / row["physical_artifact_path"]
+ if path.is_symlink() or digest(path) != row["output_sha256"]:
+ raise ValueError(provider + ": physical artifact mismatch: " + row["attempt_id"])
+ return {"provider": provider, "logical_results": len(rows), "physical_calls": len(physical),
+ "request_successes": sum(bool(row["ok"]) for row in rows),
+ "results_sha256": digest(shard / "results.jsonl"),
+ "physical_attempts_sha256": digest(shard / "physical-attempts.jsonl"),
+ "execution_sha256": digest(shard / "execution.json")}
+
+
+def main() -> None:
+ parser = argparse.ArgumentParser(description=__doc__)
+ parser.add_argument("--output-root", type=Path, required=True)
+ parser.add_argument("--source", type=Path, required=True)
+ parser.add_argument("--scorer", type=Path, required=True)
+ parser.add_argument("--profile", type=Path, default=ROOT / "evaluation/webmainbench-full/current-report.json")
+ args = parser.parse_args()
+ if args.output_root.exists():
+ parser.error("Output root already exists; cohort evidence is immutable")
+ profile = json.loads(args.profile.read_text(encoding="utf-8"))
+ if (profile.get("dataset_id") != "webmainbench-545-full-five-methods"
+ or profile.get("case_count") != 545
+ or profile.get("providers") != ["moli", "lightpanda", "obscura", "chromium", "kitesurf"]
+ or profile.get("runs_per_case") != 1 or profile.get("concurrency") != 1):
+ parser.error("Not the complete declared WebMainBench method matrix")
+ profile_sha = digest(args.profile)
+ before = source_binding(args.source, args.scorer, profile)
+ case_ids = {record["track_id"] for record in load_source(args.source)}
+ args.output_root.mkdir(parents=True)
+ (args.output_root / "preflight.json").write_text(
+ json.dumps({"profile_sha256": profile_sha, **before}, indent=2) + "\n", encoding="utf-8")
+ summaries = []
+ for provider in profile["providers"]:
+ print("Starting certified full-cohort method: " + provider, flush=True)
+ command = [sys.executable, str(ROOT / "scripts/run_webmainbench_benchmark.py"),
+ "--provider", provider, "--dataset", str(args.source),
+ "--output", str(args.output_root / provider),
+ "--timeout", str(profile["timeout_seconds"]),
+ "--max-failure-retries", str(profile["max_failure_retries"])]
+ if provider != "kitesurf":
+ version = profile["versions"][provider].split()[-1]
+ command += ["--expected-version", version]
+ subprocess.run(command, cwd=ROOT, check=True)
+ summaries.append(verify_shard(args.output_root / provider, provider, case_ids, profile))
+ if source_binding(args.source, args.scorer, profile) != before or digest(args.profile) != profile_sha:
+ raise ValueError("Source, scorer or profile changed during provider calls")
+ certificate = {"schema_version": 1, "kind": "certified_webmainbench_545_full_v1",
+ "profile_sha256": profile_sha, "preflight": before,
+ "postflight": source_binding(args.source, args.scorer, profile),
+ "providers": summaries}
+ (args.output_root / "cohort-certificate.json").write_text(
+ json.dumps(certificate, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
+ print("Certified complete WebMainBench cohort: " + str(args.output_root), flush=True)
+
+
+if __name__ == "__main__":
+ main()
diff --git a/scripts/run_webmainbench_benchmark.py b/scripts/run_webmainbench_benchmark.py
index ba0728cb5..e0c79ff9e 100644
--- a/scripts/run_webmainbench_benchmark.py
+++ b/scripts/run_webmainbench_benchmark.py
@@ -305,7 +305,9 @@ def run(args: argparse.Namespace) -> None:
else:
records = load_source(args.dataset, selected)
source_digest = DATASET_SHA256
- expected_versions = EXPECTED_VERSIONS
+ expected_versions = {**EXPECTED_VERSIONS}
+ if args.expected_version:
+ expected_versions[args.provider] = args.expected_version
dataset_id = "webmainbench-545"
if args.limit:
records = records[: args.limit]
@@ -461,6 +463,7 @@ def main() -> None:
parser.add_argument("--limit", type=int)
parser.add_argument("--timeout", type=float, default=45.0)
parser.add_argument("--max-failure-retries", type=int, choices=(0, 1), default=1)
+ parser.add_argument("--expected-version", help="Exact release version required for this provider")
args = parser.parse_args()
if args.benchmark == 'wcxb' and args.provider not in WCXB_PROVIDERS:
parser.error(f'--provider {args.provider} is not in the WCXB method matrix')
diff --git a/scripts/score_certified_webmainbench_full.py b/scripts/score_certified_webmainbench_full.py
new file mode 100644
index 000000000..940d4b452
--- /dev/null
+++ b/scripts/score_certified_webmainbench_full.py
@@ -0,0 +1,91 @@
+"""Score every provider in the certified complete WebMainBench cohort."""
+from __future__ import annotations
+
+import argparse
+import json
+import subprocess
+import sys
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parents[1]
+sys.path.insert(0, str(ROOT))
+
+from scripts.run_certified_webmainbench_full import digest, read_jsonl, source_binding
+from scripts.score_webmainbench_results import aggregate
+
+
+def verify_score(shard: Path, provider: str, case_ids: set[str], scorer: Path, profile: dict) -> dict:
+ scored = read_jsonl(shard / "scored-results.jsonl")
+ measured = read_jsonl(shard / "results.jsonl")
+ summary = json.loads((shard / "score-summary.json").read_text(encoding="utf-8"))
+ if (len(scored) != profile["case_count"] or len(measured) != len(scored)
+ or {row["track_id"] for row in scored} != case_ids
+ or len({row["track_id"] for row in scored}) != len(scored)
+ or any(row["provider"] != provider for row in scored)
+ or [{key: value for key, value in row.items() if key != "metrics"} for row in scored] != measured
+ or summary.get("provider") != provider or summary.get("case_count") != len(scored)
+ or summary.get("scorer", {}).get("commit") != profile["official_scorer_commit"]
+ or summary.get("scorer", {}).get("metric_config") != profile["metric_config"]
+ or summary["official_aggregate"] != aggregate(scored)):
+ raise ValueError(provider + ": scored evidence is incomplete or differs from measured calls")
+ for relative, expected in summary["scorer"]["metric_source_sha256"].items():
+ if digest(scorer / relative) != expected:
+ raise ValueError(provider + ": scorer source changed: " + relative)
+ return {"provider": provider, "scored_results_sha256": digest(shard / "scored-results.jsonl"),
+ "score_summary_sha256": digest(shard / "score-summary.json"),
+ "logical_results": len(scored)}
+
+
+def scorer_source_binding(scorer: Path) -> dict[str, str]:
+ files = sorted((scorer / "webmainbench/metrics").glob("*.py"))
+ if not files:
+ raise ValueError("Pinned scorer metrics source is absent")
+ return {str(path.relative_to(scorer)): digest(path) for path in files}
+
+
+def main() -> None:
+ parser = argparse.ArgumentParser(description=__doc__)
+ parser.add_argument("--cohort", type=Path, required=True)
+ parser.add_argument("--source", type=Path, required=True)
+ parser.add_argument("--scorer", type=Path, required=True)
+ parser.add_argument("--profile", type=Path, default=ROOT / "evaluation/webmainbench-full/current-report.json")
+ args = parser.parse_args()
+ if (args.cohort / "scoring-certificate.json").exists():
+ parser.error("Scoring certificate already exists; refusing overwrite")
+ profile = json.loads(args.profile.read_text(encoding="utf-8"))
+ cohort = json.loads((args.cohort / "cohort-certificate.json").read_text(encoding="utf-8"))
+ if (cohort.get("kind") != "certified_webmainbench_545_full_v1"
+ or cohort.get("profile_sha256") != digest(args.profile)):
+ parser.error("Complete certified cohort is required")
+ before = source_binding(args.source, args.scorer, profile)
+ scorer_files = scorer_source_binding(args.scorer)
+ if before != cohort["postflight"]:
+ parser.error("Scoring input differs from the certified cohort")
+ source_rows = read_jsonl(args.source)
+ case_ids = {row["track_id"] for row in source_rows}
+ if len(source_rows) != profile["case_count"] or len(case_ids) != len(source_rows):
+ parser.error("Source case inventory differs from the declared cohort")
+ receipts = []
+ for provider in profile["providers"]:
+ shard = args.cohort / provider
+ print("Scoring " + provider, flush=True)
+ with (shard / "scoring.log").open("w", encoding="utf-8") as stream:
+ subprocess.run([sys.executable, str(ROOT / "scripts/score_webmainbench_results.py"),
+ "--evidence", str(shard), "--dataset", str(args.source),
+ "--scorer-repo", str(args.scorer)], cwd=ROOT, stdout=stream,
+ stderr=subprocess.STDOUT, check=True)
+ receipts.append(verify_score(shard, provider, case_ids, args.scorer, profile))
+ if source_binding(args.source, args.scorer, profile) != before or scorer_source_binding(args.scorer) != scorer_files:
+ raise ValueError("Source or scorer changed during scoring")
+ print(provider + " scored and verified", flush=True)
+ certificate = {"schema_version": 1, "kind": "certified_webmainbench_scoring_v1",
+ "cohort_certificate_sha256": digest(args.cohort / "cohort-certificate.json"),
+ "source_binding": before, "scorer_source_sha256": scorer_files,
+ "providers": receipts}
+ (args.cohort / "scoring-certificate.json").write_text(
+ json.dumps(certificate, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
+ print("Certified complete scoring", flush=True)
+
+
+if __name__ == "__main__":
+ main()
diff --git a/tests/test_certified_webmainbench_full.py b/tests/test_certified_webmainbench_full.py
new file mode 100644
index 000000000..9d32c8dd5
--- /dev/null
+++ b/tests/test_certified_webmainbench_full.py
@@ -0,0 +1,106 @@
+"""Scenario checks for the full-cohort evidence boundary."""
+import hashlib
+import json
+from pathlib import Path
+
+import pytest
+
+from scripts.run_certified_webmainbench_full import verify_shard
+from scripts.assemble_webmainbench_full import public_value
+from scripts.score_certified_webmainbench_full import verify_score
+from scripts.score_webmainbench_results import aggregate
+
+
+def _digest(value: bytes) -> str:
+ return hashlib.sha256(value).hexdigest()
+
+
+def _write_json(path: Path, value: object) -> None:
+ path.write_text(json.dumps(value), encoding="utf-8")
+
+
+def _write_jsonl(path: Path, rows: list[dict]) -> None:
+ path.write_text("".join(json.dumps(row) + "\n" for row in rows), encoding="utf-8")
+
+
+def _cohort(tmp_path: Path) -> Path:
+ shard = tmp_path / "moli"
+ artifact_dir = shard / "artifacts"
+ artifact_dir.mkdir(parents=True)
+ rows = []
+ calls = []
+ for case_id in ("one", "two"):
+ body = f"answer for {case_id}".encode()
+ logical_path = f"artifacts/{case_id}.md"
+ physical_path = f"artifacts/{case_id}-p01.txt"
+ (shard / logical_path).write_bytes(body)
+ (shard / physical_path).write_bytes(body)
+ rows.append({"provider": "moli", "track_id": case_id, "ok": True,
+ "artifact_path": logical_path, "output_sha256": _digest(body),
+ "physical_attempt_count": 1})
+ calls.append({"attempt_id": f"moli-{case_id}-1", "track_id": case_id,
+ "physical_artifact_path": physical_path, "output_sha256": _digest(body)})
+ _write_jsonl(shard / "results.jsonl", rows)
+ _write_jsonl(shard / "physical-attempts.jsonl", calls)
+ _write_json(shard / "dataset.json", {"case_count": 2, "case_ids": ["one", "two"]})
+ _write_json(shard / "execution.json", {"case_count": 2, "provider": "moli",
+ "source_dataset_sha256": "source"})
+ return shard
+
+
+def test_complete_cohort_has_bound_logical_and_physical_evidence(tmp_path):
+ shard = _cohort(tmp_path)
+ result = verify_shard(shard, "moli", {"one", "two"},
+ {"case_count": 2, "source_dataset_sha256": "source"})
+ assert result["logical_results"] == result["physical_calls"] == 2
+
+
+def test_missing_case_cannot_be_certified(tmp_path):
+ shard = _cohort(tmp_path)
+ rows = [json.loads(line) for line in (shard / "results.jsonl").read_text().splitlines()]
+ _write_jsonl(shard / "results.jsonl", rows[:1])
+ with pytest.raises(ValueError, match="incomplete or mismatched"):
+ verify_shard(shard, "moli", {"one", "two"},
+ {"case_count": 2, "source_dataset_sha256": "source"})
+
+
+def test_changed_physical_output_cannot_be_certified(tmp_path):
+ shard = _cohort(tmp_path)
+ (shard / "artifacts/one-p01.txt").write_text("changed")
+ with pytest.raises(ValueError, match="physical artifact mismatch"):
+ verify_shard(shard, "moli", {"one", "two"},
+ {"case_count": 2, "source_dataset_sha256": "source"})
+
+
+def test_public_evidence_hides_local_build_path_without_losing_error():
+ row = {"error_message": "panic at /Users/runner/.cargo/html5ever/mod.rs:254:9",
+ "meta": ["/private/tmp/run-123/output.txt"]}
+ public = public_value(row)
+ assert "/Users/" not in json.dumps(public)
+ assert "/private/tmp/" not in json.dumps(public)
+ assert "html5ever/mod.rs:254:9" in public["error_message"]
+
+
+def test_scored_cohort_keeps_exact_measured_results_and_scorer_source(tmp_path):
+ shard = _cohort(tmp_path)
+ raw = [json.loads(line) for line in (shard / "results.jsonl").read_text().splitlines()]
+ names = ("text_edit", "code_edit", "formula_edit", "table_edit", "table_TEDS")
+ scored = [{**row, "metrics": {name: {"score": .5, "success": True} for name in names}} for row in raw]
+ _write_jsonl(shard / "scored-results.jsonl", scored)
+ scorer = tmp_path / "scorer"
+ scorer.mkdir()
+ (scorer / "metric.py").write_text("frozen scorer")
+ _write_json(shard / "score-summary.json", {
+ "provider": "moli", "case_count": 2, "official_aggregate": aggregate(scored),
+ "scorer": {"commit": "pinned", "metric_config": {"use_llm": False},
+ "metric_source_sha256": {"metric.py": _digest(b"frozen scorer")}},
+ })
+ assert verify_score(shard, "moli", {"one", "two"}, scorer, {
+ "case_count": 2, "official_scorer_commit": "pinned", "metric_config": {"use_llm": False},
+ })["logical_results"] == 2
+ scored[0]["ok"] = False
+ _write_jsonl(shard / "scored-results.jsonl", scored)
+ with pytest.raises(ValueError, match="differs from measured"):
+ verify_score(shard, "moli", {"one", "two"}, scorer, {
+ "case_count": 2, "official_scorer_commit": "pinned", "metric_config": {"use_llm": False},
+ })
diff --git a/tests/test_webmainbench_full_report.py b/tests/test_webmainbench_full_report.py
new file mode 100644
index 000000000..954cc5097
--- /dev/null
+++ b/tests/test_webmainbench_full_report.py
@@ -0,0 +1,109 @@
+"""Reader-facing controls for the complete-cohort report renderer."""
+import json
+import hashlib
+import subprocess
+from pathlib import Path
+from bs4 import BeautifulSoup
+import pytest
+
+from harness.web_fetch.report import generate_webmainbench_full
+from scripts.report_presentation_contract import check_presentation
+from scripts.check_webmainbench_full_report import check_result_payload
+
+
+def test_full_report_renders_five_way_details_and_failure_evidence(tmp_path: Path) -> None:
+ providers = ["moli", "lightpanda", "obscura", "chromium", "kitesurf"]
+ case_id = "case-1"
+ metrics = ("text_edit", "code_edit", "formula_edit", "table_edit", "table_TEDS")
+ (tmp_path / "artifacts").mkdir()
+ rows = []
+ physical = []
+ summaries = {}
+ for index, provider in enumerate(providers):
+ text = "" if provider == "moli" else f"response {provider}"
+ artifact = f"artifacts/{provider}.md"
+ (tmp_path / artifact).write_text(text, encoding="utf-8")
+ physical_path = f"artifacts/{provider}-p01.txt"
+ (tmp_path / physical_path).write_text(text, encoding="utf-8")
+ output_sha = hashlib.sha256(text.encode()).hexdigest()
+ physical.append({"provider": provider, "track_id": case_id,
+ "physical_artifact_path": physical_path, "output_sha256": output_sha,
+ "elapsed_ms": 100 + index})
+ value = (index + 1) / 10
+ scores = {metric: value for metric in metrics}
+ scores["overall"] = value
+ summaries[provider] = {
+ "official_aggregate": {"scores": scores, "denominators": {metric: 1 for metric in metrics}},
+ "request_successes": 0 if provider == "moli" else 1,
+ }
+ rows.append({
+ "attempt_id": provider + "__case-1__r001", "case_id": case_id, "provider": provider,
+ "original_url": "https://example.test/page", "ok": provider != "moli",
+ "output_chars": len(text), "artifact_path": artifact, "output_sha256": output_sha,
+ "elapsed_ms": 100 + index, "total_elapsed_ms": 100 + index,
+ "physical_attempt_count": 1, "provider_metadata": {"stderr": "fixture error"},
+ "error_type": "fixture_error" if provider == "moli" else None,
+ "error_message": "fixture error" if provider == "moli" else None,
+ "metrics": {metric: {"score": value, "success": True} for metric in metrics},
+ })
+ dataset = {"cases": [{"id": case_id, "url": "https://example.test/page",
+ "meta": {"language": "en", "level": "easy"},
+ "input_html_sha256": hashlib.sha256(b"source
").hexdigest(),
+ "reference_sha256": hashlib.sha256(b"answer").hexdigest()}]}
+ (tmp_path / "sources").mkdir()
+ (tmp_path / "sources/case-1.html.txt").write_text("source
")
+ (tmp_path / "references").mkdir()
+ (tmp_path / "references/case-1.md").write_text("answer")
+ profile = {"providers": providers, "versions": {provider: "1.0" for provider in providers},
+ "source_dataset_sha256": "a" * 64, "official_scorer_commit": "b" * 40}
+ content_keys = ("home_title", "home_lead", "score_note", "latency_note", "all_latency_note", "diagnostic_note", "infographic_caption",
+ "dataset_lead", "analysis_lead", "method_lead", "method_input",
+ "method_scoring", "method_correction", "guide_lead", "guide_scope",
+ "guide_outputs", "guide_diagnosis", "guide_example", "reproduce_lead",
+ "guide_method_scope", "guide_conclusion")
+ (tmp_path / "report-content.json").write_text(json.dumps({key: "说明" for key in content_keys}), encoding="utf-8")
+ content = json.loads((tmp_path / "report-content.json").read_text())
+ content["interfaces"] = {provider: "调用接口" for provider in providers}
+ content["adoption_lead"] = "采用证据"
+ content["adoption_limit"] = "采用限制"
+ content["review_scope_failure"] = "失败已复核"
+ content["review_scope_pass"] = "成功未复核"
+ content["kitesurf_version_note"] = "后端版本未披露"
+ for provider in providers:
+ content["guide_method_" + provider] = "逐项观察"
+ (tmp_path / "report-content.json").write_text(json.dumps(content), encoding="utf-8")
+ (tmp_path / "provider-adoption.json").write_text(json.dumps({"included": [], "cutoff": "2026-08-26"}), encoding="utf-8")
+ (tmp_path / "failure-analysis.json").write_text(json.dumps({"reviewed_failures": [
+ {"provider": "moli", "case_id": case_id, "category": "fixture_error", "category_label": "测试环境错误",
+ "cause": "Fixture failed", "treatment": "Investigate", "status": "open"}]}), encoding="utf-8")
+ (tmp_path / "physical-attempts.jsonl").write_text("".join(json.dumps(row) + "\n" for row in physical), encoding="utf-8")
+ generate_webmainbench_full(tmp_path, dataset, rows, summaries, profile)
+ subprocess.run(["node", "--check", str(tmp_path / "assets/report.js")], check=True)
+ home = check_presentation((tmp_path / "index.html").read_text(encoding="utf-8"))
+ assert len(home.select('td[data-metric-direction="max"]')) == 30
+ assert len(home.select('td[data-metric-direction="max"] .comparison-winner')) == 6
+ detail = check_presentation((tmp_path / "results/index.html").read_text(encoding="utf-8"))
+ assert detail.select_one('[data-failure-diagnosis]') is not None
+ comparison = BeautifulSoup((tmp_path / "cases/index.html").read_text(encoding="utf-8"), "html.parser")
+ assert comparison.select_one("#case-results") is not None
+ analysis = BeautifulSoup((tmp_path / "analysis/index.html").read_text(encoding="utf-8"), "html.parser")
+ assert len(analysis.select("tr[data-result]")) == 5
+ assert json.loads((tmp_path / "report-manifest.json").read_text())["languages"] == ["zh"]
+ home_html = (tmp_path / "index.html").read_text(encoding="utf-8")
+ with pytest.raises(ValueError, match="extremum"):
+ check_presentation(home_html.replace("winner comparison-winner", "winner", 1))
+ detail_html = (tmp_path / "results/index.html").read_text(encoding="utf-8")
+ index = json.loads((tmp_path / "result-shard-index.json").read_text())
+ artifact_index = json.loads((tmp_path / "artifact-shard-index.json").read_text())
+ shards = {path: json.loads((tmp_path / path).read_text()) for path in set(index.values())}
+ check_result_payload(rows[0], index, artifact_index, shards)
+ with pytest.raises(ValueError, match="Evidence target"):
+ check_result_payload(rows[0], index, {**artifact_index, rows[0]["artifact_path"]: "shards/missing.json"}, shards)
+ assert not (tmp_path / "artifacts").exists()
+ assert not (tmp_path / "sources").exists()
+ assert not (tmp_path / "references").exists()
+ assert len(list((tmp_path / "shards").glob("*.json"))) == 6
+ with pytest.raises(ValueError, match="exact error"):
+ check_presentation(detail_html.replace('data-report-detail="pending"', 'data-report-detail="failed"').replace('data-kind="error"', 'data-kind="other"'))
+ with pytest.raises(ValueError, match="exactly one summary"):
+ check_presentation(detail_html.replace('data-result-summary', 'data-no-summary'))
diff --git a/tests/test_webmainbench_runner.py b/tests/test_webmainbench_runner.py
index 39333fa90..4d183043c 100644
--- a/tests/test_webmainbench_runner.py
+++ b/tests/test_webmainbench_runner.py
@@ -4,7 +4,7 @@
import tempfile
from pathlib import Path
-from scripts.run_webmainbench_benchmark import DATASET_SHA256, fetch_record, load_selected_ids
+from scripts.run_webmainbench_benchmark import DATASET_SHA256, check_identity, fetch_record, load_selected_ids
RECORD = {
@@ -27,6 +27,11 @@ def fetch(self, case, timeout):
class WebMainBenchRunnerTests(unittest.TestCase):
+ def test_release_identity_must_match_declared_current_version(self):
+ check_identity("moli", {"reported_version": "moli 1.1.9"}, {"moli": "1.1.9"})
+ with self.assertRaisesRegex(ValueError, "expected version 1.1.9"):
+ check_identity("moli", {"reported_version": "moli 1.1.8"}, {"moli": "1.1.9"})
+
def test_loads_and_binds_frozen_sample_manifest(self):
with tempfile.TemporaryDirectory() as folder:
path = Path(folder) / "sample.json"
