' if unique else ''
+ cells.append(' ' + marker + f'{number:,.0f} ms' + ('' if unique else '') + ' ')
+ rows.append('' + esc(labels[provider]) + ' 545/545 ' + ''.join(cells) + '' + prose('home_title') + ' ' + prose('home_lead') + '
' + prose('infographic_caption') + ' 核心结果 ' + metric_table() + '' + prose('score_note') + '
全部页面的调用耗时 ' + all_call_latency_table() + '' + prose('all_latency_note') + '
成功页面的调用耗时与重试成本 ' + latency_table() + '' + prose('latency_note') + '
问题分布 ' + table(["自动诊断", "结果数"], ['' + anchor("analysis/index.html?kind=" + kind, kind) + ' ' + str(count) + ' ' + prose('diagnostic_note') + '
" + anchor("cases/index.html?id=" + case_id, case_id) + " " +
+ esc(meta.get("language") or "—") + " " + esc(meta.get("level") or "—") +
+ ' ' + anchor(case["url"], case["url"]) + " " +
+ anchor("viewer/index.html?path=sources/" + case_id + ".html.txt", "冻结网页") + " · " +
+ anchor("viewer/index.html?path=references/" + case_id + ".md", "人工答案") + " 冻结网页与人工答案 ' + prose('dataset_lead') + '
搜索任务或网址 方法全部方法 ' +
+ ''.join('' + esc(labels[provider]) + " " for provider in providers) +
+ ' 诊断' +
+ ''.join('' + esc("全部问题" if kind == "issue" else "全部结果" if kind == "all" else kind) + " " for kind in options) +
+ "
")
+ result_rows = []
+ for row in records:
+ case_id, provider = row["case_id"], row["provider"]
+ kind = categories[row["attempt_id"]]
+ result_rows.append('' +
+ anchor("results/index.html?provider=" + provider + "&id=" + case_id, case_id) + " " +
+ esc(labels[provider]) + " " + esc(kind) + " " +
+ percent(score(row, "text_edit")) + " " + f'{row["elapsed_ms"]:,.0f} ms' +
+ " " + anchor("cases/index.html?id=" + case_id, "同页") + " 逐项结果与问题定位 ' + prose('analysis_lead') + '
当前显示 当前条件无结果。
' +
+ table(["任务", "方法", "诊断", "全文分", "最终调用", "同页"], result_rows))
+ page("cases/index.html", "同页比较",
+ ''
+ '冻结网页 人工答案 ' + anchor("dataset/index.html", "返回数据集") + '
'
+ '')
+ page("results/index.html", "逐项结果",
+ '')
+ shards = defaultdict(lambda: {"results": {}, "artifacts": {}, "blobs": {}, "physical": {}})
+ result_index = {}
+ artifact_index = {}
+ def shard_name(provider, case_id):
+ bucket = int(hashlib.sha256(case_id.encode()).hexdigest()[:2], 16) % 8
+ return f"shards/{provider}-{bucket}.json"
+ def bound_artifact(path, expected, shard_path):
+ source = contained_path(out, path)
+ raw = source.read_bytes()
+ if hashlib.sha256(raw).hexdigest() != expected:
+ raise ValueError("Public artifact changed: " + path)
+ shards[shard_path]["artifacts"][path] = expected
+ shards[shard_path]["blobs"][expected] = raw.decode("utf-8")
+ artifact_index[path] = shard_path
+ for row in records:
+ key = row["provider"] + "|" + row["case_id"]
+ target = shard_name(row["provider"], row["case_id"])
+ shards[target]["results"][key] = {**row, "diagnosis": categories[row["attempt_id"]],
+ "review": reviewed_failures.get((row["provider"], row["case_id"]))}
+ result_index[key] = target
+ bound_artifact(row["artifact_path"], row["output_sha256"], target)
+ physical_path = out / "physical-attempts.jsonl"
+ physical = [json.loads(line) for line in physical_path.read_text(encoding="utf-8").splitlines() if line.strip()] if physical_path.is_file() else []
+ for attempt in physical:
+ case_id = attempt["track_id"]
+ target = shard_name(attempt["provider"], case_id)
+ shards[target]["physical"].setdefault(case_id, []).append(attempt)
+ bound_artifact(attempt["physical_artifact_path"], attempt["output_sha256"], target)
+ for case in dataset["cases"]:
+ case_id = case["id"]
+ target = shard_name("source", case_id)
+ bound_artifact(f"sources/{case_id}.html.txt", case["input_html_sha256"], target)
+ bound_artifact(f"references/{case_id}.md", case["reference_sha256"], target)
+ for relative, data in shards.items():
+ target = out / relative
+ target.parent.mkdir(parents=True, exist_ok=True)
+ target.write_text(json.dumps(data, ensure_ascii=False, separators=(",", ":")), encoding="utf-8")
+ (out / "result-shard-index.json").write_text(json.dumps(result_index, separators=(",", ":")), encoding="utf-8")
+ (out / "artifact-shard-index.json").write_text(json.dumps(artifact_index, separators=(",", ":")), encoding="utf-8")
+ shutil.rmtree(out / "artifacts")
+ shutil.rmtree(out / "sources")
+ shutil.rmtree(out / "references")
+ interface_rows = ['' + esc(labels[provider]) + ' ' + esc(content['interfaces'][provider]) + '测量与评分方法 ' + prose('method_lead') + '
输入 ' + prose('method_input') + '
数据 SHA-256：' +
+ esc(profile["source_dataset_sha256"]) + '。评分器提交：' + esc(profile["official_scorer_commit"]) +
+ '。
失败重试 本轮共 ' + str(len(records) + retry_count) +
+ ' 次物理调用，包含 ' + str(retry_count) + ' 次失败后的重试；其中 ' + str(recovered_count) +
+ ' 条重试后返回成功，' + str(final_request_failures) +
+ ' 条最终请求失败均调用两次。返回成功但正文为空或为错误页的结果不触发请求失败重试。
评分 ' + prose('method_scoring') + '
' + prose('method_correction') + '
' +
+ anchor("teds-correction.json", "修正收据") + " · " + anchor("teds-correction-review.json", "修正复核") + "调用接口 ' + table(['方法', '本轮调用方式'], interface_rows) + '' + prose('kitesurf_version_note') + '
' +
+ '公开采用证据 ' + prose('adoption_lead') + ' 截至 ' + esc(adoption['cutoff'][:10]) +
+ '，Cloudflare Markdown 接口在筛选范围内见于 ' + str(cloudflare_repositories) + ' 个仓库、' + str(len(cloudflare_matches)) +
+ ' 个文件。
' + prose('adoption_limit') + '
' + anchor('provider-adoption.json', '查看检索样本与纳入规则') + '" + anchor("analysis/index.html?provider=" + provider, labels[provider]) +
+ " " + str(summaries[provider]["request_successes"]) +
+ "/545 " + str(issues) + " " +
+ prose("guide_method_" + provider) + " " + esc(labels[provider]) + " " + "".join(
+ '' +
+ anchor("analysis/index.html?provider=" + provider + "&kind=" + kind,
+ str(sum(categories[row["attempt_id"]] == kind for row in selected))) + " "
+ for kind in diagnostic_kinds) + "结果解读 ' + esc(outcome) + " " + prose('guide_lead') + '
质量与样本 ' +
+ metric_table() + '' + prose('guide_scope') + '
核对完整输出 ' + prose('guide_outputs') + '
' +
+ anchor("analysis/index.html?kind=低全文相似度", "查看低全文分案例") +
+ '可检查的一页 ' + prose('guide_example') + '
' + example_links + '
' +
+ '各方法的主要问题 ' +
+ table(["方法", "返回成功", "自动诊断数", "本轮观察"], method_rows) +
+ '' + prose('guide_method_scope') + '
' +
+ table(["方法", *diagnostic_kinds], diagnostic_rows) + '错误类别 ' +
+ table(["类别", "结果数"], ["" + esc(kind) + " " + str(count) + " ' + prose('guide_diagnosis') + '
已复核的机器问题原因 ' +
+ table(["原因", "结果数"], ["" + esc(kind) + " " + str(count) + " ' + str(len(reviewed_failures)) + '/' + str(machine_issue_count) + ' ' + prose('review_scope_failure') +
+ '；' + str(len(records) - len(reviewed_failures)) + ' ' + prose('review_scope_pass') + '
' +
+ anchor('failure-analysis.json', '查看逐条失败原因与处理') +
+ '结论 ' + prose('guide_conclusion') + '
复现依据 ' + prose('reproduce_lead') + '
' +
+ " · ".join(anchor(name, name) for name in evidence_links) + "
")
+ page("viewer/index.html", "证据查看器",
+ '')
+ page("en/index.html", "英文版未生成",
+ '' +
+ anchor("index.html", "返回中文报告"))
+ (out / "report-manifest.json").write_text(json.dumps({
+ "schema_version": 1, "report": "webmainbench-545-full", "languages": ["zh"],
+ "case_count": 545, "provider_count": len(providers), "result_count": len(records),
+ }, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
diff --git a/harness/web_fetch/report_build.py b/harness/web_fetch/report_build.py
index 251fc65c3..fd68c90f5 100644
--- a/harness/web_fetch/report_build.py
+++ b/harness/web_fetch/report_build.py
@@ -5,7 +5,7 @@
import tempfile
from pathlib import Path
from harness.reporting.assets import copy_bound_assets
-from .report import generate, generate_public_benchmark_research, generate_webmainbench, generate_wcxb, succeeded, measured, failure_category, TRACKS
+from .report import generate, generate_public_benchmark_research, generate_webmainbench, generate_webmainbench_full, webmainbench_full_machine_issue, generate_wcxb, succeeded, measured, failure_category, TRACKS
from .report_changes import webmainbench_history
from .failure_diagnosis import diagnose
from .safety import contained_path, safe_identifier, validate_report_ids
@@ -559,6 +559,108 @@ def build_webmainbench_report(source: Path, output: Path, *, include_english: bo
return output / 'index.html'
+def build_webmainbench_full_report(source: Path, output: Path) -> Path:
+ """Build the separately bound complete 545-case report from certified evidence."""
+ from scripts.run_certified_webmainbench_full import digest, read_jsonl
+ from .webmainbench_diagnostics import validate_teds_revision
+ if source.is_symlink() or any(path.is_symlink() for path in source.rglob('*')):
+ raise ValueError('Symlinked report evidence is forbidden')
+ profile_path = Path(__file__).resolve().parents[2] / 'evaluation/webmainbench-full/current-report.json'
+ profile = json.loads(profile_path.read_text(encoding='utf-8'))
+ certificate = json.loads((source / 'cohort-certificate.json').read_text(encoding='utf-8'))
+ scoring_certificate = json.loads((source / 'scoring-certificate.json').read_text(encoding='utf-8'))
+ if (certificate.get('kind') != 'certified_webmainbench_545_full_v1'
+ or certificate.get('profile_sha256') != digest(profile_path)
+ or certificate.get('preflight') != certificate.get('postflight')
+ or json.loads((source / 'current-report.json').read_text(encoding='utf-8')) != profile):
+ raise ValueError('Full-cohort certificate or profile mismatch')
+ if (scoring_certificate.get('kind') != 'certified_webmainbench_scoring_v1'
+ or scoring_certificate.get('cohort_certificate_sha256') != digest(source / 'cohort-certificate.json')
+ or scoring_certificate.get('source_binding') != certificate['postflight']):
+ raise ValueError('Complete scoring certificate mismatch')
+ dataset = json.loads((source / 'dataset.json').read_text(encoding='utf-8'))
+ records = read_jsonl(source / 'results.jsonl')
+ providers = profile['providers']
+ cases = {case['id']: case for case in dataset['cases']}
+ if (len(cases) != 545 or len(records) != 545 * len(providers)
+ or {(row['provider'], row['case_id']) for row in records} !=
+ {(provider, case_id) for provider in providers for case_id in cases}):
+ raise ValueError('Full WebMainBench result matrix mismatch')
+ validate_teds_revision(source)
+ content_binding_path = Path(__file__).resolve().parents[2] / 'evaluation/webmainbench-full/content-binding.json'
+ content_binding = json.loads(content_binding_path.read_text(encoding='utf-8'))
+ content_path = data_home() / 'report-assets/webmainbench-full' / content_binding['relative_path']
+ if digest(content_path) != content_binding['sha256']:
+ raise ValueError('Report prose differs from the compact content binding')
+ illustration_path = data_home() / 'report-assets/webmainbench-full' / content_binding['infographic']
+ if digest(illustration_path) != content_binding['infographic_sha256']:
+ raise ValueError('Report illustration differs from the compact asset binding')
+ mobile_illustration_path = data_home() / 'report-assets/webmainbench-full' / content_binding['infographic_mobile']
+ if digest(mobile_illustration_path) != content_binding['infographic_mobile_sha256']:
+ raise ValueError('Mobile report illustration differs from the compact asset binding')
+ failure_binding_path = Path(__file__).resolve().parents[2] / 'evaluation/webmainbench-full/failure-analysis-binding.json'
+ failure_binding = json.loads(failure_binding_path.read_text(encoding='utf-8'))
+ failure_path = data_home() / 'report-assets/webmainbench-full' / failure_binding['relative_path']
+ if digest(failure_path) != failure_binding['sha256']:
+ raise ValueError('Failure analysis differs from the compact review binding')
+ failure_analysis = json.loads(failure_path.read_text(encoding='utf-8'))
+ failures = {(row['provider'], row['case_id']) for row in records
+ if webmainbench_full_machine_issue(source, row)}
+ reviewed = {(row['provider'], row['case_id']) for row in failure_analysis['reviewed_failures']}
+ if (failure_analysis.get('kind') != 'webmainbench_full_failure_analysis_v1'
+ or failure_analysis.get('cohort_certificate_sha256') != digest(source / 'cohort-certificate.json')
+ or reviewed != failures or len(reviewed) != len(failure_analysis['reviewed_failures'])
+ or any(not all(isinstance(row.get(key), str) and row[key].strip()
+ for key in ('category_label', 'cause', 'treatment', 'status'))
+ for row in failure_analysis['reviewed_failures'])):
+ raise ValueError('Failure analysis does not cover every measured request failure')
+ for case_id, case in cases.items():
+ if digest(contained_path(source, f'references/{case_id}.md')) != case['reference_sha256']:
+ raise ValueError('Reference changed: ' + case_id)
+ if digest(contained_path(source, f'sources/{case_id}.html.txt')) != case['input_html_sha256']:
+ raise ValueError('Frozen page changed: ' + case_id)
+ for row in records:
+ if digest(contained_path(source, row['artifact_path'])) != row['output_sha256']:
+ raise ValueError('Output changed: ' + row['attempt_id'])
+ output.parent.mkdir(parents=True, exist_ok=True)
+ staging = Path(tempfile.mkdtemp(prefix='.webmainbench-full-build-', dir=output.parent))
+ try:
+ for name in ('artifacts', 'references', 'sources', 'fixture-certificates', 'correction-source'):
+ shutil.copytree(source / name, staging / name)
+ for name in ('dataset.json', 'results.jsonl', 'original-results.jsonl',
+ 'physical-attempts.jsonl', 'execution.json', 'score-summaries.json',
+ 'cohort-certificate.json', 'current-report.json',
+ 'scoring-certificate.json',
+ 'teds-correction.json', 'teds-correction-review.json'):
+ shutil.copy2(source / name, staging / name)
+ shutil.copy2(content_path, staging / 'report-content.json')
+ shutil.copy2(failure_path, staging / 'failure-analysis.json')
+ shutil.copy2(Path(__file__).resolve().parents[2] / 'research/provider-adoption.json', staging / 'provider-adoption.json')
+ (staging / 'assets').mkdir(exist_ok=True)
+ shutil.copy2(illustration_path, staging / 'assets/infographic.png')
+ shutil.copy2(mobile_illustration_path, staging / 'assets/infographic-mobile.png')
+ generate_webmainbench_full(staging, dataset, records,
+ json.loads((source / 'score-summaries.json').read_text(encoding='utf-8')),
+ profile)
+ from scripts.check_webmainbench_full_report import check as check_webmainbench_full
+ check_webmainbench_full(staging)
+ if output.exists():
+ backup = Path(tempfile.mkdtemp(prefix='.report-swap-', dir=output.parent)) / 'report'
+ output.rename(backup)
+ try:
+ staging.rename(output)
+ except BaseException:
+ backup.rename(output)
+ raise
+ shutil.rmtree(backup.parent)
+ else:
+ staging.rename(output)
+ finally:
+ if staging.exists():
+ shutil.rmtree(staging)
+ return output / 'index.html'
+
+
def build_wcxb_report(source: Path, corpus: Path, sample_path: Path, profile: dict, output: Path) -> Path:
"""Build the one current report from a fully certified WCXB matrix."""
from scripts.run_webmainbench_benchmark import WCXB_PROVIDERS, load_wcxb_source
diff --git a/research/report-changes.json b/research/report-changes.json
index 6b3f6f07c..5aefe4b58 100644
--- a/research/report-changes.json
+++ b/research/report-changes.json
@@ -1275,6 +1275,17 @@
}
],
"releases": [
+ {
+ "id": "2026-09-23-webmainbench-complete-545-five-methods",
+ "comparison_kinds": ["webmainbench_frozen_html_to_markdown"],
+ "date": "2026-09-23",
+ "title": "发布 WebMainBench 完整 545 题五方法对照",
+ "title_i18n": {"en": "Publish the complete 545-case WebMainBench comparison"},
+ "summary": "以同一批 545 个官方冻结 HTML 和人工参考，实测五种稳定版浏览器方法。新报告提供逐题输入、输出、评分、失败证据和已复核的机器问题分析。",
+ "summary_i18n": {"en": "Five stable browser methods were measured against the same 545 official frozen HTML inputs and human references. The new report exposes case inputs, outputs, scores, failure evidence, and reviewed machine-issue analysis."},
+ "changes": [["完整评测与评分", "先前公开报告只覆盖 200 题，无法代表完整数据集。官方 TEDS 实现还可能高估部分表格相似度。", "认证五种方法各 545 题，并用保留原始输出和官方分数的结构化树比较校正 TEDS；报告同时展示请求重试和失败分类。", "共 2725 个逻辑结果、2747 次实际调用；292 个 TEDS 分数校正。28 个机器问题已逐条复核，其余 2697 个结果未逐条人工复核。"]],
+ "changes_i18n": {"en": [["Full evaluation and scoring", "The earlier public report covered only 200 cases, and the official TEDS implementation could overstate some table similarities.", "Certify 545 cases for each of five methods and correct TEDS with structured tree comparison while preserving raw outputs and official scores; show retries and failure classes.", "There are 2,725 logical results and 2,747 physical calls; 292 TEDS scores were corrected. All 28 machine issues were reviewed individually, while the other 2,697 results were not individually human reviewed."]]}
+ },
{
"id": "2026-09-20-wcxb-measured-mode-labels",
"comparison_kinds": ["wcxb_frozen_html_to_markdown"],
diff --git a/scripts/assemble_webmainbench_full.py b/scripts/assemble_webmainbench_full.py
new file mode 100644
index 000000000..177923d31
--- /dev/null
+++ b/scripts/assemble_webmainbench_full.py
@@ -0,0 +1,161 @@
+"""Package only certified, scored WebMainBench-545 evidence for the full report."""
+from __future__ import annotations
+
+import argparse
+import hashlib
+import json
+import re
+import shutil
+import sys
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parents[1]
+sys.path.insert(0, str(ROOT))
+
+from harness.web_fetch.safety import contained_path
+from scripts.run_certified_webmainbench_full import digest, read_jsonl, verify_shard
+from scripts.run_webmainbench_benchmark import load_source
+
+
+def public_value(value):
+ """Remove machine-specific absolute paths from the public evidence copy."""
+ if isinstance(value, dict):
+ return {key: public_value(item) for key, item in value.items()}
+ if isinstance(value, list):
+ return [public_value(item) for item in value]
+ if isinstance(value, str):
+ value = re.sub(r"/Users/[^/\s:]+", "/redacted-home", value)
+ value = re.sub(r"/private/tmp/[^\s:]+", "/redacted-temp", value)
+ return value
+
+
+def copy_bound(root: Path, output: Path, relative: str, expected: str) -> None:
+ source = contained_path(root, relative)
+ if (root / relative).is_symlink() or digest(source) != expected:
+ raise ValueError("Evidence artifact changed: " + relative)
+ target = contained_path(output, relative)
+ target.parent.mkdir(parents=True, exist_ok=True)
+ shutil.copy2(source, target)
+
+
+def assemble(root: Path, output: Path, source: Path, profile_path: Path) -> None:
+ if output.exists():
+ raise ValueError("Output already exists; report evidence is immutable")
+ if root.is_symlink() or source.is_symlink() or profile_path.is_symlink():
+ raise ValueError("Symlinked aggregate input is forbidden")
+ profile = json.loads(profile_path.read_text(encoding="utf-8"))
+ cert_path = root / "cohort-certificate.json"
+ certificate = json.loads(cert_path.read_text(encoding="utf-8"))
+ scoring_certificate_path = root / "scoring-certificate.json"
+ scoring_certificate = json.loads(scoring_certificate_path.read_text(encoding="utf-8"))
+ if (certificate.get("kind") != "certified_webmainbench_545_full_v1"
+ or certificate.get("profile_sha256") != digest(profile_path)
+ or certificate.get("preflight") != certificate.get("postflight")
+ or certificate["preflight"]["source_sha256"] != profile["source_dataset_sha256"]
+ or [row["provider"] for row in certificate["providers"]] != profile["providers"]):
+ raise ValueError("Complete-cohort certificate or profile mismatch")
+ if (scoring_certificate.get("kind") != "certified_webmainbench_scoring_v1"
+ or scoring_certificate.get("cohort_certificate_sha256") != digest(cert_path)
+ or scoring_certificate.get("source_binding") != certificate["postflight"]
+ or [row["provider"] for row in scoring_certificate["providers"]] != profile["providers"]):
+ raise ValueError("Complete scoring certificate differs from the certified cohort")
+ source_records = load_source(source)
+ ids = {row["track_id"] for row in source_records}
+ if len(ids) != 545 or len(source_records) != profile["case_count"]:
+ raise ValueError("Source case inventory mismatch")
+ results: list[dict] = []
+ physical: list[dict] = []
+ summaries: dict[str, dict] = {}
+ executions: list[dict] = []
+ output.mkdir(parents=True)
+ (output / "references").mkdir()
+ (output / "sources").mkdir()
+ (output / "fixture-certificates").mkdir()
+ for record in source_records:
+ case_id = record["track_id"]
+ (output / "references" / (case_id + ".md")).write_text(
+ record["groundtruth_content"], encoding="utf-8")
+ (output / "sources" / (case_id + ".html.txt")).write_text(
+ record["html"], encoding="utf-8")
+ for expected, scored_receipt in zip(certificate["providers"], scoring_certificate["providers"]):
+ provider = expected["provider"]
+ shard = root / provider
+ if shard.is_symlink() or any(path.is_symlink() for path in shard.rglob("*")):
+ raise ValueError(provider + ": symlinked evidence is forbidden")
+ if verify_shard(shard, provider, ids, profile) != expected:
+ raise ValueError(provider + ": certificate differs from measured shard")
+ rows = read_jsonl(shard / "scored-results.jsonl")
+ if (scored_receipt["scored_results_sha256"] != digest(shard / "scored-results.jsonl")
+ or scored_receipt["score_summary_sha256"] != digest(shard / "score-summary.json")
+ or scored_receipt["logical_results"] != profile["case_count"]):
+ raise ValueError(provider + ": scoring certificate differs from scored shard")
+ if (len(rows) != 545 or {row["track_id"] for row in rows} != ids
+ or len({row["track_id"] for row in rows}) != 545
+ or any(row["provider"] != provider for row in rows)):
+ raise ValueError(provider + ": incomplete scored matrix")
+ summary = json.loads((shard / "score-summary.json").read_text(encoding="utf-8"))
+ if (summary.get("case_count") != 545 or summary.get("provider") != provider
+ or summary.get("scorer", {}).get("metric_source_sha256") !=
+ scoring_certificate["scorer_source_sha256"]):
+ raise ValueError(provider + ": score summary scope mismatch")
+ logical = {row["track_id"]: row for row in read_jsonl(shard / "results.jsonl")}
+ for row in rows:
+ base = logical[row["track_id"]]
+ if {k: v for k, v in row.items() if k != "metrics"} != base:
+ raise ValueError(provider + ": scoring changed measured result")
+ copy_bound(shard, output, row["artifact_path"], row["output_sha256"])
+ results.append(public_value({**row, "case_id": row["track_id"]}))
+ attempts = read_jsonl(shard / "physical-attempts.jsonl")
+ for attempt in attempts:
+ copy_bound(shard, output, attempt["physical_artifact_path"], attempt["output_sha256"])
+ physical.extend(public_value(attempts))
+ execution = json.loads((shard / "execution.json").read_text(encoding="utf-8"))
+ ca = execution.get("fixture_ca_artifact")
+ if ca:
+ ca_source = contained_path(shard, ca)
+ if digest(ca_source) != execution["network_isolation_probe"]["fixture_ca_sha256"]:
+ raise ValueError(provider + ": fixture CA changed")
+ ca_target = output / "fixture-certificates" / (provider + "-ca.pem")
+ shutil.copy2(ca_source, ca_target)
+ execution["assembled_fixture_ca_path"] = str(ca_target.relative_to(output))
+ executions.append(execution)
+ summaries[provider] = summary
+ expected_pairs = {(provider, case_id) for provider in profile["providers"] for case_id in ids}
+ if (len(results) != len(expected_pairs)
+ or {(row["provider"], row["case_id"]) for row in results} != expected_pairs
+ or len({row["attempt_id"] for row in results}) != len(results)
+ or len({row["attempt_id"] for row in physical}) != len(physical)):
+ raise ValueError("Full result or physical attempt matrix mismatch")
+ cases = [{"id": row["track_id"], "name": row["track_id"], "url": row["url"],
+ "category": "webmainbench", "track": "static_html", "meta": row["meta"],
+ "input_html_sha256": hashlib.sha256(row["html"].encode()).hexdigest(),
+ "reference_sha256": hashlib.sha256(row["groundtruth_content"].encode()).hexdigest(),
+ "reference_chars": len(row["groundtruth_content"])} for row in source_records]
+ dataset = {"schema_version": 1, "tool": "web_fetch", "id": profile["dataset_id"],
+ "name": "WebMainBench-545 full · five Web Fetch methods",
+ "source_dataset_sha256": profile["source_dataset_sha256"],
+ "cases": cases, "coverage_plan": {"static_html": profile["providers"]},
+ "comparison": profile}
+ (output / "dataset.json").write_text(json.dumps(dataset, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
+ (output / "score-summaries.json").write_text(json.dumps(summaries, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
+ (output / "execution.json").write_text(json.dumps({"batches": executions}, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
+ (output / "results.jsonl").write_text("".join(json.dumps(row, ensure_ascii=False) + "\n" for row in results), encoding="utf-8")
+ (output / "physical-attempts.jsonl").write_text("".join(json.dumps(row, ensure_ascii=False) + "\n" for row in physical), encoding="utf-8")
+ shutil.copy2(cert_path, output / "cohort-certificate.json")
+ shutil.copy2(scoring_certificate_path, output / "scoring-certificate.json")
+ shutil.copy2(profile_path, output / "current-report.json")
+ print(f"Assembled {len(results)} results and {len(physical)} physical calls", flush=True)
+
+
+def main() -> None:
+ parser = argparse.ArgumentParser(description=__doc__)
+ parser.add_argument("--source-root", type=Path, required=True)
+ parser.add_argument("--output", type=Path, required=True)
+ parser.add_argument("--dataset", type=Path, required=True)
+ parser.add_argument("--profile", type=Path, default=ROOT / "evaluation/webmainbench-full/current-report.json")
+ args = parser.parse_args()
+ assemble(args.source_root, args.output, args.dataset, args.profile)
+
+
+if __name__ == "__main__":
+ main()
diff --git a/scripts/build_report.py b/scripts/build_report.py
index 0e603dcfa..a3028c9c8 100644
--- a/scripts/build_report.py
+++ b/scripts/build_report.py
@@ -3,7 +3,7 @@
from pathlib import Path
ROOT=Path(__file__).resolve().parents[1]
sys.path.insert(0,str(ROOT))
-from harness.web_fetch.report_build import build_report, build_public_benchmark_research, build_wcxb_report
+from harness.web_fetch.report_build import build_report, build_public_benchmark_research, build_wcxb_report, build_webmainbench_full_report
from harness.web_fetch.storage import (
current_evidence_dir,
require_canonical_inputs,
@@ -23,9 +23,15 @@
parser.add_argument('--wcxb-source',type=Path)
parser.add_argument('--wcxb-sample',type=Path)
parser.add_argument('--wcxb',action='store_true',help='Build the independently bound WCXB report')
+ parser.add_argument('--webmainbench-full',action='store_true',help='Build the separately certified complete 545-case report')
args=parser.parse_args()
- if args.profile and args.wcxb:
- parser.error('--profile and --wcxb are mutually exclusive')
+ if sum(bool(value) for value in (args.profile, args.wcxb, args.webmainbench_full)) > 1:
+ parser.error('--profile, --wcxb and --webmainbench-full are mutually exclusive')
+ if args.webmainbench_full:
+ if not args.evidence or args.include_english:
+ parser.error('--webmainbench-full requires --evidence and generates Chinese only')
+ print(build_webmainbench_full_report(args.evidence,args.output or ROOT/'reports/webmainbench-full'))
+ raise SystemExit(0)
if args.public_benchmark_research:
if args.evidence:
parser.error('--evidence and --public-benchmark-research are mutually exclusive')
diff --git a/scripts/check_webmainbench_full_report.py b/scripts/check_webmainbench_full_report.py
new file mode 100644
index 000000000..630ef7226
--- /dev/null
+++ b/scripts/check_webmainbench_full_report.py
@@ -0,0 +1,166 @@
+"""Release gate for the separately certified complete WebMainBench report."""
+from __future__ import annotations
+
+import argparse
+from collections import Counter, defaultdict
+import hashlib
+import json
+from pathlib import Path
+from bs4 import BeautifulSoup
+
+from scripts.report_presentation_contract import check_presentation
+from harness.web_fetch.safety import contained_path
+from harness.web_fetch.report import ERROR_PAGE
+
+
+def check_result_payload(row: dict, index: dict, artifact_index: dict, shards: dict) -> None:
+ key = row["provider"] + "|" + row["case_id"]
+ shard_path = index.get(key)
+ if shard_path not in shards:
+ raise ValueError("Result shard missing: " + key)
+ bundled = shards[shard_path]["results"].get(key)
+ if not bundled or {name: value for name, value in bundled.items() if name not in ("diagnosis", "review")} != row:
+ raise ValueError("Result shard differs from scored evidence: " + key)
+ artifact = row["artifact_path"]
+ if artifact_index.get(artifact) != shard_path:
+ raise ValueError("Evidence target points to wrong shard: " + artifact)
+ digest = shards[shard_path]["artifacts"].get(artifact)
+ value = shards[shard_path]["blobs"].get(digest)
+ if not isinstance(value, str) or hashlib.sha256(value.encode()).hexdigest() != row["output_sha256"]:
+ raise ValueError("Raw output absent or changed: " + artifact)
+
+
+def check(root: Path) -> None:
+ binding_path = Path(__file__).resolve().parents[1] / "evaluation/webmainbench-full/content-binding.json"
+ binding = json.loads(binding_path.read_text(encoding="utf-8"))
+ for path, expected in ((root / "report-content.json", binding["sha256"]),
+ (root / "assets/infographic.png", binding["infographic_sha256"]),
+ (root / "assets/infographic-mobile.png", binding["infographic_mobile_sha256"])):
+ if hashlib.sha256(path.read_bytes()).hexdigest() != expected:
+ raise ValueError("Bound report asset changed: " + path.name)
+ failure_binding = json.loads((Path(__file__).resolve().parents[1] / "evaluation/webmainbench-full/failure-analysis-binding.json").read_text(encoding="utf-8"))
+ failure_bytes = (root / "failure-analysis.json").read_bytes()
+ if hashlib.sha256(failure_bytes).hexdigest() != failure_binding["sha256"]:
+ raise ValueError("Bound failure analysis changed")
+ failure_analysis = json.loads(failure_bytes)
+ manifest = json.loads((root / "report-manifest.json").read_text(encoding="utf-8"))
+ if manifest != {"schema_version": 1, "report": "webmainbench-545-full",
+ "languages": ["zh"], "case_count": 545, "provider_count": 5,
+ "result_count": 2725}:
+ raise ValueError("Incomplete WebMainBench report manifest")
+ dataset = json.loads((root / "dataset.json").read_text(encoding="utf-8"))
+ records = [json.loads(line) for line in (root / "results.jsonl").read_text(encoding="utf-8").splitlines()]
+ providers = dataset["coverage_plan"]["static_html"]
+ case_ids = {row["id"] for row in dataset["cases"]}
+ if len(case_ids) != 545 or len(records) != 2725 or len(providers) != 5:
+ raise ValueError("Full matrix missing")
+ expected = {(provider, case_id) for provider in providers for case_id in case_ids}
+ if {(row["provider"], row["case_id"]) for row in records} != expected:
+ raise ValueError("Report matrix differs from evidence")
+ reviewed = {(row["provider"], row["case_id"]) for row in failure_analysis["reviewed_failures"]}
+ if (len(reviewed) != len(failure_analysis["reviewed_failures"])
+ or any(not all(isinstance(row.get(key), str) and row[key].strip()
+ for key in ("category_label", "cause", "treatment", "status"))
+ for row in failure_analysis["reviewed_failures"])):
+ raise ValueError("Published failure analysis contains duplicate or empty reviews")
+ home = check_presentation((root / "index.html").read_text(encoding="utf-8"))
+ guide = check_presentation((root / "guide/index.html").read_text(encoding="utf-8"))
+ for page in (home, guide):
+ if len(page.select('td[data-metric-direction="max"]')) != 30:
+ raise ValueError("Core metrics or winner controls missing")
+ if "545" not in page.get_text(" ", strip=True):
+ raise ValueError("Complete population missing")
+ if len(home.select('td[data-metric-direction="min"]')) != 35:
+ raise ValueError("All-call or conditional latency comparison missing")
+ first_case = dataset["cases"][0]["id"]
+ example_targets = {"viewer/index.html?path=sources/" + first_case + ".html.txt",
+ "viewer/index.html?path=references/" + first_case + ".md",
+ "cases/index.html?id=" + first_case}
+ if not example_targets.issubset({link.get("href") for link in guide.select("a[href]")}):
+ raise ValueError("Interpretation example lacks source, reference or same-case result")
+ if not home.select_one('img[src="assets/infographic.png"]') or not (root / "assets/infographic.png").is_file():
+ raise ValueError("Bound infographic missing")
+ if not home.select_one('source[media="(max-width:650px)"][srcset="assets/infographic-mobile.png"]'):
+ raise ValueError("Mobile infographic route missing")
+ analysis = BeautifulSoup((root / "analysis/index.html").read_text(encoding="utf-8"), "html.parser")
+ if len(analysis.select("tr[data-result]")) != 2725 or not analysis.select_one("#kind"):
+ raise ValueError("Result inventory incomplete")
+ if not any(row.get("data-kind") != "未触发诊断" for row in analysis.select("tr[data-result]")):
+ raise ValueError("Default diagnostic population is empty")
+ if not (root / "en/index.html").is_file() or "返回中文报告" not in (root / "en/index.html").read_text(encoding="utf-8"):
+ raise ValueError("English not-generated route missing")
+ case_template = BeautifulSoup((root / "cases/index.html").read_text(encoding="utf-8"), "html.parser")
+ if not case_template.select_one("#case-results") or not case_template.select_one("#case-source"):
+ raise ValueError("Same-case comparison template is incomplete")
+ result_template = check_presentation((root / "results/index.html").read_text(encoding="utf-8"))
+ if not result_template.select_one('[data-result-summary]') or not result_template.select_one('#result-logs pre[data-kind="error"]'):
+ raise ValueError("Shared result template lacks summary or exact error log")
+ index = json.loads((root / "result-shard-index.json").read_text(encoding="utf-8"))
+ artifact_index = json.loads((root / "artifact-shard-index.json").read_text(encoding="utf-8"))
+ shard_paths = set(index.values()) | set(artifact_index.values())
+ shards = {path: json.loads(contained_path(root, path).read_text(encoding="utf-8")) for path in shard_paths}
+ if len(index) != len(records) or len(shards) > 48:
+ raise ValueError("Complete result shard index missing or oversized")
+ failures = set()
+ diagnostic_counts = defaultdict(Counter)
+ for row in records:
+ shard = shards[index[row["provider"] + "|" + row["case_id"]]]
+ diagnosis = shard["results"][row["provider"] + "|" + row["case_id"]]["diagnosis"]
+ diagnostic_counts[row["provider"]][diagnosis] += 1
+ path = row["artifact_path"]
+ output = shard["blobs"].get(shard["artifacts"].get(path), "")
+ if not row["ok"] or not row["output_chars"] or ERROR_PAGE.search(output):
+ failures.add((row["provider"], row["case_id"]))
+ if reviewed != failures:
+ raise ValueError("Published failure analysis scope is incomplete")
+ cells = guide.select("td[data-diagnostic-provider][data-kind]")
+ expected_cells = {(provider, kind) for provider in providers
+ for kind in ("请求失败", "空输出", "错误页", "低全文相似度")}
+ if ({(cell["data-diagnostic-provider"], cell["data-kind"]) for cell in cells} != expected_cells
+ or len(cells) != len(expected_cells) or any(
+ int(cell.get_text(strip=True)) != diagnostic_counts[cell["data-diagnostic-provider"]][cell["data-kind"]]
+ for cell in cells
+ )):
+ raise ValueError("Interpretation diagnosis matrix differs from measured results")
+ reviewed_by_key = {(row["provider"], row["case_id"]): row for row in failure_analysis["reviewed_failures"]}
+ for row in records:
+ check_result_payload(row, index, artifact_index, shards)
+ bundled = shards[index[row["provider"] + "|" + row["case_id"]]]["results"][row["provider"] + "|" + row["case_id"]]
+ if bundled.get("review") != reviewed_by_key.get((row["provider"], row["case_id"])):
+ raise ValueError("Result detail differs from bound failure analysis: " + row["attempt_id"])
+ for case in dataset["cases"]:
+ case_id = case["id"]
+ for path, expected_hash in (("sources/" + case_id + ".html.txt", case["input_html_sha256"]),
+ ("references/" + case_id + ".md", case["reference_sha256"])):
+ shard_path = artifact_index.get(path)
+ if shard_path not in shards:
+ raise ValueError("Frozen task evidence shard missing: " + path)
+ blob_hash = shards[shard_path]["artifacts"].get(path)
+ content = shards[shard_path]["blobs"].get(blob_hash)
+ if not isinstance(content, str) or hashlib.sha256(content.encode()).hexdigest() != expected_hash:
+ raise ValueError("Frozen task evidence changed: " + path)
+ physical = [json.loads(line) for line in (root / "physical-attempts.jsonl").read_text(encoding="utf-8").splitlines() if line.strip()]
+ for attempt in physical:
+ path = attempt["physical_artifact_path"]
+ shard_path = artifact_index.get(path)
+ if shard_path not in shards:
+ raise ValueError("Physical output shard missing: " + path)
+ digest = shards[shard_path]["artifacts"].get(path)
+ content = shards[shard_path]["blobs"].get(digest)
+ if not isinstance(content, str) or hashlib.sha256(content.encode()).hexdigest() != attempt["output_sha256"]:
+ raise ValueError("Physical output absent or changed: " + path)
+ if attempt not in shards[shard_path]["physical"].get(attempt["track_id"], []):
+ raise ValueError("Physical call missing from result detail: " + attempt["attempt_id"])
+ public_files = [path for path in root.rglob("*") if path.is_file()]
+ if len(public_files) > 2000 or sum(path.stat().st_size for path in public_files) > 314572800:
+ raise ValueError("Report package exceeds Artifact Site deployment limits")
+ viewer = (root / "viewer/index.html").read_text(encoding="utf-8")
+ if "id=\"evidence-content\"" not in viewer or not (root / "assets/report.js").is_file():
+ raise ValueError("On-demand evidence viewer missing")
+
+
+if __name__ == "__main__":
+ parser = argparse.ArgumentParser(description=__doc__)
+ parser.add_argument("report", type=Path)
+ check(parser.parse_args().report)
+ print("Complete WebMainBench report release gate passed")
diff --git a/scripts/correct_webmainbench_teds.py b/scripts/correct_webmainbench_teds.py
index 0047ff157..226a4a44a 100644
--- a/scripts/correct_webmainbench_teds.py
+++ b/scripts/correct_webmainbench_teds.py
@@ -80,9 +80,9 @@ def main():
raise ValueError('Refusing symlinked correction evidence')
expected = {(provider, case['id']) for provider in dataset['coverage_plan']['static_html']
for case in dataset['cases']}
- if len(records) != len(expected) or len(expected) != 200 * len(summaries) or {
+ if len(records) != len(expected) or len(expected) != len(dataset['cases']) * len(summaries) or {
(row['provider'], row['case_id']) for row in records} != expected:
- raise ValueError(f'Correction requires the complete declared 200-by-{len(summaries)} matrix')
+ raise ValueError(f'Correction requires the complete declared {len(dataset["cases"])}-by-{len(summaries)} matrix')
# Verify the exact official implementation, not only its Git HEAD.
for summary in summaries.values():
for relative, expected in summary['scorer']['metric_source_sha256'].items():
@@ -175,7 +175,7 @@ def category(row):
'input_results_sha256': digest(source / 'results.jsonl'),
'output_results_sha256': digest(results_path),
'dataset_sha256': digest(source / 'dataset.json'),
- 'sample_manifest_sha256': dataset['sample_manifest_sha256'],
+ 'sample_manifest_sha256': dataset.get('sample_manifest_sha256'),
'inherited_correction_receipt_sha256': inherited_receipt_sha256,
'inherited_logical_results': len(inherited),
'source_code_sha256': {str(path.relative_to(ROOT)): digest(path) for path in source_files},
diff --git a/scripts/review_webmainbench_teds_correction.py b/scripts/review_webmainbench_teds_correction.py
index 55fba3e02..7c66a45d6 100644
--- a/scripts/review_webmainbench_teds_correction.py
+++ b/scripts/review_webmainbench_teds_correction.py
@@ -40,6 +40,8 @@ def review(source, corrected):
if len({row['attempt_id'] for row in after}) != len(after):
raise ValueError('Duplicate corrected attempt')
for name in ('dataset.json', 'sample-manifest.json', 'physical-attempts.jsonl', 'execution.json', 'result-review.json'):
+ if not (source / name).exists() and not (corrected / name).exists():
+ continue
if digest(source / name) != digest(corrected / name):
raise ValueError('Unexpected change: ' + name)
changed = []
@@ -76,7 +78,7 @@ def review(source, corrected):
impacts = {}
for provider, summary in summaries.items():
records = [row for row in after if row['provider'] == provider]
- if len(records) != 200:
+ if len(records) != len(json.loads((source / 'dataset.json').read_text())['cases']):
raise ValueError('Provider scope mismatch')
current = summary['official_aggregate']
for name in ('text_edit', 'code_edit', 'formula_edit', 'table_edit', 'table_TEDS'):
diff --git a/scripts/run_certified_webmainbench_full.py b/scripts/run_certified_webmainbench_full.py
new file mode 100644
index 000000000..0d539499f
--- /dev/null
+++ b/scripts/run_certified_webmainbench_full.py
@@ -0,0 +1,117 @@
+"""Run the predeclared complete WebMainBench cohort with source and result checks."""
+from __future__ import annotations
+
+import argparse
+import hashlib
+import json
+import subprocess
+import sys
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parents[1]
+sys.path.insert(0, str(ROOT))
+
+from scripts.run_webmainbench_benchmark import load_source
+
+
+def digest(path: Path) -> str:
+ return hashlib.sha256(path.read_bytes()).hexdigest()
+
+
+def read_jsonl(path: Path) -> list[dict]:
+ return [json.loads(line) for line in path.read_text(encoding="utf-8").splitlines() if line.strip()]
+
+
+def source_binding(source: Path, scorer: Path, profile: dict) -> dict:
+ if digest(source) != profile["source_dataset_sha256"]:
+ raise ValueError("WebMainBench source bytes changed")
+ records = load_source(source)
+ case_ids = [record["track_id"] for record in records]
+ case_digest = hashlib.sha256(("\n".join(case_ids) + "\n").encode()).hexdigest()
+ if len(records) != profile["case_count"] or case_digest != profile["case_ids_sha256"]:
+ raise ValueError("WebMainBench case inventory changed")
+ commit = subprocess.check_output(["git", "-C", str(scorer), "rev-parse", "HEAD"], text=True).strip()
+ if commit != profile["official_scorer_commit"]:
+ raise ValueError("Official scorer commit changed")
+ return {"source_sha256": digest(source), "case_ids_sha256": case_digest,
+ "scorer_commit": commit, "case_count": len(records)}
+
+
+def verify_shard(shard: Path, provider: str, case_ids: set[str], profile: dict) -> dict:
+ rows = read_jsonl(shard / "results.jsonl")
+ physical = read_jsonl(shard / "physical-attempts.jsonl")
+ execution = json.loads((shard / "execution.json").read_text(encoding="utf-8"))
+ dataset = json.loads((shard / "dataset.json").read_text(encoding="utf-8"))
+ if (len(rows) != profile["case_count"] or {row["track_id"] for row in rows} != case_ids
+ or len({row["track_id"] for row in rows}) != len(rows)
+ or dataset["case_count"] != profile["case_count"]
+ or set(dataset["case_ids"]) != case_ids
+ or execution["case_count"] != profile["case_count"]
+ or execution["provider"] != provider
+ or execution["source_dataset_sha256"] != profile["source_dataset_sha256"]):
+ raise ValueError(provider + ": incomplete or mismatched cohort")
+ attempt_ids = {row["attempt_id"] for row in physical}
+ if len(attempt_ids) != len(physical) or sum(row["physical_attempt_count"] for row in rows) != len(physical):
+ raise ValueError(provider + ": physical call inventory mismatch")
+ for row in rows:
+ path = shard / row["artifact_path"]
+ if path.is_symlink() or digest(path) != row["output_sha256"]:
+ raise ValueError(provider + ": result artifact mismatch: " + row["track_id"])
+ for row in physical:
+ path = shard / row["physical_artifact_path"]
+ if path.is_symlink() or digest(path) != row["output_sha256"]:
+ raise ValueError(provider + ": physical artifact mismatch: " + row["attempt_id"])
+ return {"provider": provider, "logical_results": len(rows), "physical_calls": len(physical),
+ "request_successes": sum(bool(row["ok"]) for row in rows),
+ "results_sha256": digest(shard / "results.jsonl"),
+ "physical_attempts_sha256": digest(shard / "physical-attempts.jsonl"),
+ "execution_sha256": digest(shard / "execution.json")}
+
+
+def main() -> None:
+ parser = argparse.ArgumentParser(description=__doc__)
+ parser.add_argument("--output-root", type=Path, required=True)
+ parser.add_argument("--source", type=Path, required=True)
+ parser.add_argument("--scorer", type=Path, required=True)
+ parser.add_argument("--profile", type=Path, default=ROOT / "evaluation/webmainbench-full/current-report.json")
+ args = parser.parse_args()
+ if args.output_root.exists():
+ parser.error("Output root already exists; cohort evidence is immutable")
+ profile = json.loads(args.profile.read_text(encoding="utf-8"))
+ if (profile.get("dataset_id") != "webmainbench-545-full-five-methods"
+ or profile.get("case_count") != 545
+ or profile.get("providers") != ["moli", "lightpanda", "obscura", "chromium", "kitesurf"]
+ or profile.get("runs_per_case") != 1 or profile.get("concurrency") != 1):
+ parser.error("Not the complete declared WebMainBench method matrix")
+ profile_sha = digest(args.profile)
+ before = source_binding(args.source, args.scorer, profile)
+ case_ids = {record["track_id"] for record in load_source(args.source)}
+ args.output_root.mkdir(parents=True)
+ (args.output_root / "preflight.json").write_text(
+ json.dumps({"profile_sha256": profile_sha, **before}, indent=2) + "\n", encoding="utf-8")
+ summaries = []
+ for provider in profile["providers"]:
+ print("Starting certified full-cohort method: " + provider, flush=True)
+ command = [sys.executable, str(ROOT / "scripts/run_webmainbench_benchmark.py"),
+ "--provider", provider, "--dataset", str(args.source),
+ "--output", str(args.output_root / provider),
+ "--timeout", str(profile["timeout_seconds"]),
+ "--max-failure-retries", str(profile["max_failure_retries"])]
+ if provider != "kitesurf":
+ version = profile["versions"][provider].split()[-1]
+ command += ["--expected-version", version]
+ subprocess.run(command, cwd=ROOT, check=True)
+ summaries.append(verify_shard(args.output_root / provider, provider, case_ids, profile))
+ if source_binding(args.source, args.scorer, profile) != before or digest(args.profile) != profile_sha:
+ raise ValueError("Source, scorer or profile changed during provider calls")
+ certificate = {"schema_version": 1, "kind": "certified_webmainbench_545_full_v1",
+ "profile_sha256": profile_sha, "preflight": before,
+ "postflight": source_binding(args.source, args.scorer, profile),
+ "providers": summaries}
+ (args.output_root / "cohort-certificate.json").write_text(
+ json.dumps(certificate, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
+ print("Certified complete WebMainBench cohort: " + str(args.output_root), flush=True)
+
+
+if __name__ == "__main__":
+ main()
diff --git a/scripts/run_webmainbench_benchmark.py b/scripts/run_webmainbench_benchmark.py
index ba0728cb5..e0c79ff9e 100644
--- a/scripts/run_webmainbench_benchmark.py
+++ b/scripts/run_webmainbench_benchmark.py
@@ -305,7 +305,9 @@ def run(args: argparse.Namespace) -> None:
else:
records = load_source(args.dataset, selected)
source_digest = DATASET_SHA256
- expected_versions = EXPECTED_VERSIONS
+ expected_versions = {**EXPECTED_VERSIONS}
+ if args.expected_version:
+ expected_versions[args.provider] = args.expected_version
dataset_id = "webmainbench-545"
if args.limit:
records = records[: args.limit]
@@ -461,6 +463,7 @@ def main() -> None:
parser.add_argument("--limit", type=int)
parser.add_argument("--timeout", type=float, default=45.0)
parser.add_argument("--max-failure-retries", type=int, choices=(0, 1), default=1)
+ parser.add_argument("--expected-version", help="Exact release version required for this provider")
args = parser.parse_args()
if args.benchmark == 'wcxb' and args.provider not in WCXB_PROVIDERS:
parser.error(f'--provider {args.provider} is not in the WCXB method matrix')
diff --git a/scripts/score_certified_webmainbench_full.py b/scripts/score_certified_webmainbench_full.py
new file mode 100644
index 000000000..940d4b452
--- /dev/null
+++ b/scripts/score_certified_webmainbench_full.py
@@ -0,0 +1,91 @@
+"""Score every provider in the certified complete WebMainBench cohort."""
+from __future__ import annotations
+
+import argparse
+import json
+import subprocess
+import sys
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parents[1]
+sys.path.insert(0, str(ROOT))
+
+from scripts.run_certified_webmainbench_full import digest, read_jsonl, source_binding
+from scripts.score_webmainbench_results import aggregate
+
+
+def verify_score(shard: Path, provider: str, case_ids: set[str], scorer: Path, profile: dict) -> dict:
+ scored = read_jsonl(shard / "scored-results.jsonl")
+ measured = read_jsonl(shard / "results.jsonl")
+ summary = json.loads((shard / "score-summary.json").read_text(encoding="utf-8"))
+ if (len(scored) != profile["case_count"] or len(measured) != len(scored)
+ or {row["track_id"] for row in scored} != case_ids
+ or len({row["track_id"] for row in scored}) != len(scored)
+ or any(row["provider"] != provider for row in scored)
+ or [{key: value for key, value in row.items() if key != "metrics"} for row in scored] != measured
+ or summary.get("provider") != provider or summary.get("case_count") != len(scored)
+ or summary.get("scorer", {}).get("commit") != profile["official_scorer_commit"]
+ or summary.get("scorer", {}).get("metric_config") != profile["metric_config"]
+ or summary["official_aggregate"] != aggregate(scored)):
+ raise ValueError(provider + ": scored evidence is incomplete or differs from measured calls")
+ for relative, expected in summary["scorer"]["metric_source_sha256"].items():
+ if digest(scorer / relative) != expected:
+ raise ValueError(provider + ": scorer source changed: " + relative)
+ return {"provider": provider, "scored_results_sha256": digest(shard / "scored-results.jsonl"),
+ "score_summary_sha256": digest(shard / "score-summary.json"),
+ "logical_results": len(scored)}
+
+
+def scorer_source_binding(scorer: Path) -> dict[str, str]:
+ files = sorted((scorer / "webmainbench/metrics").glob("*.py"))
+ if not files:
+ raise ValueError("Pinned scorer metrics source is absent")
+ return {str(path.relative_to(scorer)): digest(path) for path in files}
+
+
+def main() -> None:
+ parser = argparse.ArgumentParser(description=__doc__)
+ parser.add_argument("--cohort", type=Path, required=True)
+ parser.add_argument("--source", type=Path, required=True)
+ parser.add_argument("--scorer", type=Path, required=True)
+ parser.add_argument("--profile", type=Path, default=ROOT / "evaluation/webmainbench-full/current-report.json")
+ args = parser.parse_args()
+ if (args.cohort / "scoring-certificate.json").exists():
+ parser.error("Scoring certificate already exists; refusing overwrite")
+ profile = json.loads(args.profile.read_text(encoding="utf-8"))
+ cohort = json.loads((args.cohort / "cohort-certificate.json").read_text(encoding="utf-8"))
+ if (cohort.get("kind") != "certified_webmainbench_545_full_v1"
+ or cohort.get("profile_sha256") != digest(args.profile)):
+ parser.error("Complete certified cohort is required")
+ before = source_binding(args.source, args.scorer, profile)
+ scorer_files = scorer_source_binding(args.scorer)
+ if before != cohort["postflight"]:
+ parser.error("Scoring input differs from the certified cohort")
+ source_rows = read_jsonl(args.source)
+ case_ids = {row["track_id"] for row in source_rows}
+ if len(source_rows) != profile["case_count"] or len(case_ids) != len(source_rows):
+ parser.error("Source case inventory differs from the declared cohort")
+ receipts = []
+ for provider in profile["providers"]:
+ shard = args.cohort / provider
+ print("Scoring " + provider, flush=True)
+ with (shard / "scoring.log").open("w", encoding="utf-8") as stream:
+ subprocess.run([sys.executable, str(ROOT / "scripts/score_webmainbench_results.py"),
+ "--evidence", str(shard), "--dataset", str(args.source),
+ "--scorer-repo", str(args.scorer)], cwd=ROOT, stdout=stream,
+ stderr=subprocess.STDOUT, check=True)
+ receipts.append(verify_score(shard, provider, case_ids, args.scorer, profile))
+ if source_binding(args.source, args.scorer, profile) != before or scorer_source_binding(args.scorer) != scorer_files:
+ raise ValueError("Source or scorer changed during scoring")
+ print(provider + " scored and verified", flush=True)
+ certificate = {"schema_version": 1, "kind": "certified_webmainbench_scoring_v1",
+ "cohort_certificate_sha256": digest(args.cohort / "cohort-certificate.json"),
+ "source_binding": before, "scorer_source_sha256": scorer_files,
+ "providers": receipts}
+ (args.cohort / "scoring-certificate.json").write_text(
+ json.dumps(certificate, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
+ print("Certified complete scoring", flush=True)
+
+
+if __name__ == "__main__":
+ main()
diff --git a/tests/test_certified_webmainbench_full.py b/tests/test_certified_webmainbench_full.py
new file mode 100644
index 000000000..9d32c8dd5
--- /dev/null
+++ b/tests/test_certified_webmainbench_full.py
@@ -0,0 +1,106 @@
+"""Scenario checks for the full-cohort evidence boundary."""
+import hashlib
+import json
+from pathlib import Path
+
+import pytest
+
+from scripts.run_certified_webmainbench_full import verify_shard
+from scripts.assemble_webmainbench_full import public_value
+from scripts.score_certified_webmainbench_full import verify_score
+from scripts.score_webmainbench_results import aggregate
+
+
+def _digest(value: bytes) -> str:
+ return hashlib.sha256(value).hexdigest()
+
+
+def _write_json(path: Path, value: object) -> None:
+ path.write_text(json.dumps(value), encoding="utf-8")
+
+
+def _write_jsonl(path: Path, rows: list[dict]) -> None:
+ path.write_text("".join(json.dumps(row) + "\n" for row in rows), encoding="utf-8")
+
+
+def _cohort(tmp_path: Path) -> Path:
+ shard = tmp_path / "moli"
+ artifact_dir = shard / "artifacts"
+ artifact_dir.mkdir(parents=True)
+ rows = []
+ calls = []
+ for case_id in ("one", "two"):
+ body = f"answer for {case_id}".encode()
+ logical_path = f"artifacts/{case_id}.md"
+ physical_path = f"artifacts/{case_id}-p01.txt"
+ (shard / logical_path).write_bytes(body)
+ (shard / physical_path).write_bytes(body)
+ rows.append({"provider": "moli", "track_id": case_id, "ok": True,
+ "artifact_path": logical_path, "output_sha256": _digest(body),
+ "physical_attempt_count": 1})
+ calls.append({"attempt_id": f"moli-{case_id}-1", "track_id": case_id,
+ "physical_artifact_path": physical_path, "output_sha256": _digest(body)})
+ _write_jsonl(shard / "results.jsonl", rows)
+ _write_jsonl(shard / "physical-attempts.jsonl", calls)
+ _write_json(shard / "dataset.json", {"case_count": 2, "case_ids": ["one", "two"]})
+ _write_json(shard / "execution.json", {"case_count": 2, "provider": "moli",
+ "source_dataset_sha256": "source"})
+ return shard
+
+
+def test_complete_cohort_has_bound_logical_and_physical_evidence(tmp_path):
+ shard = _cohort(tmp_path)
+ result = verify_shard(shard, "moli", {"one", "two"},
+ {"case_count": 2, "source_dataset_sha256": "source"})
+ assert result["logical_results"] == result["physical_calls"] == 2
+
+
+def test_missing_case_cannot_be_certified(tmp_path):
+ shard = _cohort(tmp_path)
+ rows = [json.loads(line) for line in (shard / "results.jsonl").read_text().splitlines()]
+ _write_jsonl(shard / "results.jsonl", rows[:1])
+ with pytest.raises(ValueError, match="incomplete or mismatched"):
+ verify_shard(shard, "moli", {"one", "two"},
+ {"case_count": 2, "source_dataset_sha256": "source"})
+
+
+def test_changed_physical_output_cannot_be_certified(tmp_path):
+ shard = _cohort(tmp_path)
+ (shard / "artifacts/one-p01.txt").write_text("changed")
+ with pytest.raises(ValueError, match="physical artifact mismatch"):
+ verify_shard(shard, "moli", {"one", "two"},
+ {"case_count": 2, "source_dataset_sha256": "source"})
+
+
+def test_public_evidence_hides_local_build_path_without_losing_error():
+ row = {"error_message": "panic at /Users/runner/.cargo/html5ever/mod.rs:254:9",
+ "meta": ["/private/tmp/run-123/output.txt"]}
+ public = public_value(row)
+ assert "/Users/" not in json.dumps(public)
+ assert "/private/tmp/" not in json.dumps(public)
+ assert "html5ever/mod.rs:254:9" in public["error_message"]
+
+
+def test_scored_cohort_keeps_exact_measured_results_and_scorer_source(tmp_path):
+ shard = _cohort(tmp_path)
+ raw = [json.loads(line) for line in (shard / "results.jsonl").read_text().splitlines()]
+ names = ("text_edit", "code_edit", "formula_edit", "table_edit", "table_TEDS")
+ scored = [{**row, "metrics": {name: {"score": .5, "success": True} for name in names}} for row in raw]
+ _write_jsonl(shard / "scored-results.jsonl", scored)
+ scorer = tmp_path / "scorer"
+ scorer.mkdir()
+ (scorer / "metric.py").write_text("frozen scorer")
+ _write_json(shard / "score-summary.json", {
+ "provider": "moli", "case_count": 2, "official_aggregate": aggregate(scored),
+ "scorer": {"commit": "pinned", "metric_config": {"use_llm": False},
+ "metric_source_sha256": {"metric.py": _digest(b"frozen scorer")}},
+ })
+ assert verify_score(shard, "moli", {"one", "two"}, scorer, {
+ "case_count": 2, "official_scorer_commit": "pinned", "metric_config": {"use_llm": False},
+ })["logical_results"] == 2
+ scored[0]["ok"] = False
+ _write_jsonl(shard / "scored-results.jsonl", scored)
+ with pytest.raises(ValueError, match="differs from measured"):
+ verify_score(shard, "moli", {"one", "two"}, scorer, {
+ "case_count": 2, "official_scorer_commit": "pinned", "metric_config": {"use_llm": False},
+ })
diff --git a/tests/test_webmainbench_full_report.py b/tests/test_webmainbench_full_report.py
new file mode 100644
index 000000000..954cc5097
--- /dev/null
+++ b/tests/test_webmainbench_full_report.py
@@ -0,0 +1,109 @@
+"""Reader-facing controls for the complete-cohort report renderer."""
+import json
+import hashlib
+import subprocess
+from pathlib import Path
+from bs4 import BeautifulSoup
+import pytest
+
+from harness.web_fetch.report import generate_webmainbench_full
+from scripts.report_presentation_contract import check_presentation
+from scripts.check_webmainbench_full_report import check_result_payload
+
+
+def test_full_report_renders_five_way_details_and_failure_evidence(tmp_path: Path) -> None:
+ providers = ["moli", "lightpanda", "obscura", "chromium", "kitesurf"]
+ case_id = "case-1"
+ metrics = ("text_edit", "code_edit", "formula_edit", "table_edit", "table_TEDS")
+ (tmp_path / "artifacts").mkdir()
+ rows = []
+ physical = []
+ summaries = {}
+ for index, provider in enumerate(providers):
+ text = "" if provider == "moli" else f"response {provider}"
+ artifact = f"artifacts/{provider}.md"
+ (tmp_path / artifact).write_text(text, encoding="utf-8")
+ physical_path = f"artifacts/{provider}-p01.txt"
+ (tmp_path / physical_path).write_text(text, encoding="utf-8")
+ output_sha = hashlib.sha256(text.encode()).hexdigest()
+ physical.append({"provider": provider, "track_id": case_id,
+ "physical_artifact_path": physical_path, "output_sha256": output_sha,
+ "elapsed_ms": 100 + index})
+ value = (index + 1) / 10
+ scores = {metric: value for metric in metrics}
+ scores["overall"] = value
+ summaries[provider] = {
+ "official_aggregate": {"scores": scores, "denominators": {metric: 1 for metric in metrics}},
+ "request_successes": 0 if provider == "moli" else 1,
+ }
+ rows.append({
+ "attempt_id": provider + "__case-1__r001", "case_id": case_id, "provider": provider,
+ "original_url": "https://example.test/page", "ok": provider != "moli",
+ "output_chars": len(text), "artifact_path": artifact, "output_sha256": output_sha,
+ "elapsed_ms": 100 + index, "total_elapsed_ms": 100 + index,
+ "physical_attempt_count": 1, "provider_metadata": {"stderr": "fixture error"},
+ "error_type": "fixture_error" if provider == "moli" else None,
+ "error_message": "fixture error" if provider == "moli" else None,
+ "metrics": {metric: {"score": value, "success": True} for metric in metrics},
+ })
+ dataset = {"cases": [{"id": case_id, "url": "https://example.test/page",
+ "meta": {"language": "en", "level": "easy"},
+ "input_html_sha256": hashlib.sha256(b"source
").hexdigest(),
+ "reference_sha256": hashlib.sha256(b"answer").hexdigest()}]}
+ (tmp_path / "sources").mkdir()
+ (tmp_path / "sources/case-1.html.txt").write_text("source
")
+ (tmp_path / "references").mkdir()
+ (tmp_path / "references/case-1.md").write_text("answer")
+ profile = {"providers": providers, "versions": {provider: "1.0" for provider in providers},
+ "source_dataset_sha256": "a" * 64, "official_scorer_commit": "b" * 40}
+ content_keys = ("home_title", "home_lead", "score_note", "latency_note", "all_latency_note", "diagnostic_note", "infographic_caption",
+ "dataset_lead", "analysis_lead", "method_lead", "method_input",
+ "method_scoring", "method_correction", "guide_lead", "guide_scope",
+ "guide_outputs", "guide_diagnosis", "guide_example", "reproduce_lead",
+ "guide_method_scope", "guide_conclusion")
+ (tmp_path / "report-content.json").write_text(json.dumps({key: "说明" for key in content_keys}), encoding="utf-8")
+ content = json.loads((tmp_path / "report-content.json").read_text())
+ content["interfaces"] = {provider: "调用接口" for provider in providers}
+ content["adoption_lead"] = "采用证据"
+ content["adoption_limit"] = "采用限制"
+ content["review_scope_failure"] = "失败已复核"
+ content["review_scope_pass"] = "成功未复核"
+ content["kitesurf_version_note"] = "后端版本未披露"
+ for provider in providers:
+ content["guide_method_" + provider] = "逐项观察"
+ (tmp_path / "report-content.json").write_text(json.dumps(content), encoding="utf-8")
+ (tmp_path / "provider-adoption.json").write_text(json.dumps({"included": [], "cutoff": "2026-08-26"}), encoding="utf-8")
+ (tmp_path / "failure-analysis.json").write_text(json.dumps({"reviewed_failures": [
+ {"provider": "moli", "case_id": case_id, "category": "fixture_error", "category_label": "测试环境错误",
+ "cause": "Fixture failed", "treatment": "Investigate", "status": "open"}]}), encoding="utf-8")
+ (tmp_path / "physical-attempts.jsonl").write_text("".join(json.dumps(row) + "\n" for row in physical), encoding="utf-8")
+ generate_webmainbench_full(tmp_path, dataset, rows, summaries, profile)
+ subprocess.run(["node", "--check", str(tmp_path / "assets/report.js")], check=True)
+ home = check_presentation((tmp_path / "index.html").read_text(encoding="utf-8"))
+ assert len(home.select('td[data-metric-direction="max"]')) == 30
+ assert len(home.select('td[data-metric-direction="max"] .comparison-winner')) == 6
+ detail = check_presentation((tmp_path / "results/index.html").read_text(encoding="utf-8"))
+ assert detail.select_one('[data-failure-diagnosis]') is not None
+ comparison = BeautifulSoup((tmp_path / "cases/index.html").read_text(encoding="utf-8"), "html.parser")
+ assert comparison.select_one("#case-results") is not None
+ analysis = BeautifulSoup((tmp_path / "analysis/index.html").read_text(encoding="utf-8"), "html.parser")
+ assert len(analysis.select("tr[data-result]")) == 5
+ assert json.loads((tmp_path / "report-manifest.json").read_text())["languages"] == ["zh"]
+ home_html = (tmp_path / "index.html").read_text(encoding="utf-8")
+ with pytest.raises(ValueError, match="extremum"):
+ check_presentation(home_html.replace("winner comparison-winner", "winner", 1))
+ detail_html = (tmp_path / "results/index.html").read_text(encoding="utf-8")
+ index = json.loads((tmp_path / "result-shard-index.json").read_text())
+ artifact_index = json.loads((tmp_path / "artifact-shard-index.json").read_text())
+ shards = {path: json.loads((tmp_path / path).read_text()) for path in set(index.values())}
+ check_result_payload(rows[0], index, artifact_index, shards)
+ with pytest.raises(ValueError, match="Evidence target"):
+ check_result_payload(rows[0], index, {**artifact_index, rows[0]["artifact_path"]: "shards/missing.json"}, shards)
+ assert not (tmp_path / "artifacts").exists()
+ assert not (tmp_path / "sources").exists()
+ assert not (tmp_path / "references").exists()
+ assert len(list((tmp_path / "shards").glob("*.json"))) == 6
+ with pytest.raises(ValueError, match="exact error"):
+ check_presentation(detail_html.replace('data-report-detail="pending"', 'data-report-detail="failed"').replace('data-kind="error"', 'data-kind="other"'))
+ with pytest.raises(ValueError, match="exactly one summary"):
+ check_presentation(detail_html.replace('data-result-summary', 'data-no-summary'))
diff --git a/tests/test_webmainbench_runner.py b/tests/test_webmainbench_runner.py
index 39333fa90..4d183043c 100644
--- a/tests/test_webmainbench_runner.py
+++ b/tests/test_webmainbench_runner.py
@@ -4,7 +4,7 @@
import tempfile
from pathlib import Path
-from scripts.run_webmainbench_benchmark import DATASET_SHA256, fetch_record, load_selected_ids
+from scripts.run_webmainbench_benchmark import DATASET_SHA256, check_identity, fetch_record, load_selected_ids
RECORD = {
@@ -27,6 +27,11 @@ def fetch(self, case, timeout):
class WebMainBenchRunnerTests(unittest.TestCase):
+ def test_release_identity_must_match_declared_current_version(self):
+ check_identity("moli", {"reported_version": "moli 1.1.9"}, {"moli": "1.1.9"})
+ with self.assertRaisesRegex(ValueError, "expected version 1.1.9"):
+ check_identity("moli", {"reported_version": "moli 1.1.8"}, {"moli": "1.1.9"})
+
def test_loads_and_binds_frozen_sample_manifest(self):
with tempfile.TemporaryDirectory() as folder:
path = Path(folder) / "sample.json"
From a46b91d8663e2697f6da86cd3d4a787d1ac10392 Mon Sep 17 00:00:00 2001
From: lanyue-llk <270302213+lanyue-llk@users.noreply.github.com>
Date: Wed, 23 Sep 2026 21:28:28 +0800
Subject: [PATCH 02/14] Show per-method cause matrix and same-case WebMainBench
outputs
---
.../webmainbench-full/content-binding.json | 10 +-
harness/web_fetch/report.py | 93 ++++++++++++++++---
research/report-changes.json | 11 +++
scripts/check_webmainbench_full_report.py | 17 +++-
tests/test_webmainbench_full_report.py | 4 +
5 files changed, 117 insertions(+), 18 deletions(-)
diff --git a/evaluation/webmainbench-full/content-binding.json b/evaluation/webmainbench-full/content-binding.json
index 8d88a5425..71fb6d818 100644
--- a/evaluation/webmainbench-full/content-binding.json
+++ b/evaluation/webmainbench-full/content-binding.json
@@ -1,9 +1,9 @@
{
"schema_version": 1,
"relative_path": "report-content.json",
- "sha256": "1d0aa4cbd90b52e046062d00ceb97c4f3dd8a19f7c51c0b1547eb826ba9d61e4",
- "infographic": "infographic.png",
- "infographic_sha256": "402f90bb2d664b38e9efb697f910d99d142d4e872cc7d1e76b8ed733f8aa9e62",
- "infographic_mobile": "infographic-mobile.png",
- "infographic_mobile_sha256": "2c60966e07b88fe5d0c7e966af90f4027e61bd22e439f7a4d408b875c4e18eb7"
+ "sha256": "0d59f0345090c422e0c290673ca9350e4704a8dbc51bb5482324c0450a0ac58f",
+ "infographic": "issue-matrix.png",
+ "infographic_sha256": "879c1af9af28a383a1ef6a0bf26e52655a4aaff28857f9d2dee3296f9d3b4dbf",
+ "infographic_mobile": "issue-matrix-mobile.png",
+ "infographic_mobile_sha256": "1d85914f7b4853050635962e1c459278c457a3682fa64b8fdd85027fec48a8c0"
}
diff --git a/harness/web_fetch/report.py b/harness/web_fetch/report.py
index e56fffbb8..dea961819 100644
--- a/harness/web_fetch/report.py
+++ b/harness/web_fetch/report.py
@@ -2739,7 +2739,6 @@ def generate_webmainbench_full(out: Path, dataset: dict, records: list[dict],
content = json.loads((out / "report-content.json").read_text(encoding="utf-8"))
failure_analysis = json.loads((out / "failure-analysis.json").read_text(encoding="utf-8"))
reviewed_failures = {(row["provider"], row["case_id"]): row for row in failure_analysis["reviewed_failures"]}
- reviewed_categories = Counter(row["category_label"] for row in failure_analysis["reviewed_failures"])
machine_issue_count = sum(webmainbench_full_machine_issue(out, row) for row in records)
adoption = json.loads((out / "provider-adoption.json").read_text(encoding="utf-8"))
cloudflare_matches = {(entry["repo"], match["path"]) for entry in adoption["included"]
@@ -2776,13 +2775,56 @@ def classify(row):
categories = {row["attempt_id"]: classify(row) for row in records}
counts = Counter(categories.values())
- css = """*{box-sizing:border-box}body{margin:0;background:#f4f7fb;color:#172637;font:16px/1.55 system-ui,sans-serif}body.dark{background:#101923;color:#edf3fa}a{color:#1859a5}body.dark a{color:#92c7ff}.wrap{width:min(1200px,calc(100% - 32px));margin:auto}header{background:#142d4b;color:white}header .wrap{display:flex;align-items:center;gap:18px;flex-wrap:wrap;padding:15px 0}header a{color:white;text-decoration:none}header strong{margin-right:auto}nav{display:flex;gap:14px;flex-wrap:wrap}button,input,select{font:inherit;border:1px solid #9aacc0;border-radius:7px;padding:7px 10px;background:white;color:#172637}body.dark button,body.dark input,body.dark select{background:#25384b;color:white;border-color:#61768b}h1{font-size:clamp(30px,4vw,48px);line-height:1.18;margin:0 0 15px}h2{font-size:26px;margin:0 0 14px}.hero{padding:48px 0 22px}.hero p{font-size:18px;max-width:900px;color:#506377}body.dark .hero p,body.dark .note{color:#bfd0e1}section{margin:24px 0 38px}figure{margin:24px 0}.panel{background:white;border:1px solid #d8e1eb;border-radius:12px;padding:20px;margin:14px 0}body.dark .panel{background:#1c2b3b;border-color:#38506a}.scroll{overflow:auto;max-height:66vh;border:1px solid #d8e1eb;border-radius:10px;background:white}body.dark .scroll{background:#1c2b3b;border-color:#38506a}table{border-collapse:collapse;width:100%;min-width:820px}th,td{padding:9px 12px;text-align:left;border-bottom:1px solid #dce5ee;vertical-align:top}th{position:sticky;top:0;background:#e9f0f8;color:#152a40;z-index:1}body.dark th{background:#29435d;color:white}tbody tr:hover{background:#edf4fc}body.dark tbody tr:hover{background:#28445f}td small{display:block;color:#597088}body.dark td small{color:#b8cadb}.number{font-variant-numeric:tabular-nums;white-space:nowrap}.winner{color:#116b3c;font-weight:750}body.dark .winner{color:#82e1a7}.filters{display:flex;gap:12px;flex-wrap:wrap;margin:14px 0}.filters label{display:grid;gap:4px;font-size:13px;font-weight:700}.note{color:#52677b;font-size:14px}pre{white-space:pre-wrap;overflow-wrap:anywhere;background:#0e2032;color:#f1f6fb;padding:17px;border-radius:8px;max-height:65vh;overflow:auto;font:13px/1.5 ui-monospace,monospace}details{border:1px solid #dce5ee;border-radius:8px;padding:10px 14px;margin:10px 0}body.dark details{border-color:#38506a}summary{cursor:pointer;font-weight:700}.actions{display:flex;gap:14px;flex-wrap:wrap}.source{overflow-wrap:anywhere}footer{padding:32px 0;color:#52677b}@media(max-width:650px){.panel{padding:14px}header nav{width:100%}th,td{padding:7px 9px}}"""
- js = """function theme(){document.body.classList.toggle('dark');try{localStorage.setItem('wmb545-theme',document.body.classList.contains('dark')?'dark':'light')}catch(e){}}try{if(localStorage.getItem('wmb545-theme')==='dark')document.body.classList.add('dark')}catch(e){}function filter(){let q=(document.getElementById('q')?.value||'').toLowerCase(),p=document.getElementById('provider')?.value||'',k=document.getElementById('kind')?.value||'issue',n=0;document.querySelectorAll('tr[data-result]').forEach(r=>{let show=(!q||r.dataset.search.includes(q))&&(!p||r.dataset.provider===p)&&(k==='all'||(k==='issue'?r.dataset.kind!=='未触发诊断':r.dataset.kind===k));r.hidden=!show;if(show)n++});let c=document.getElementById('count');if(c)c.textContent=n;let e=document.getElementById('empty');if(e)e.hidden=n!==0}document.addEventListener('DOMContentLoaded',()=>{for(let id of ['q','provider','kind'])document.getElementById(id)?.addEventListener('input',filter);let params=new URLSearchParams(location.search);for(let id of ['provider','kind']){let select=document.getElementById(id);if(select&¶ms.has(id))select.value=params.get(id)}filter();let node=document.getElementById('evidence-content');if(node){let path=params.get('path')||'';if(!/^(artifacts|references|sources)\\/[A-Za-z0-9_.\\/-]+$/.test(path)||path.includes('..')){node.textContent='无效证据路径';return}document.getElementById('evidence-path').textContent=path;document.getElementById('evidence-raw').href=path;loadEvidence(path,node)}});"""
+ css = """*{box-sizing:border-box}body{margin:0;background:#f4f7fb;color:#172637;font:16px/1.55 system-ui,sans-serif}body.dark{background:#101923;color:#edf3fa}a{color:#1859a5}body.dark a{color:#92c7ff}.wrap{width:min(1200px,calc(100% - 32px));margin:auto}header{background:#142d4b;color:white}header .wrap{display:flex;align-items:center;gap:18px;flex-wrap:wrap;padding:15px 0}header a{color:white;text-decoration:none}header strong{margin-right:auto}nav{display:flex;gap:14px;flex-wrap:wrap}button,input,select{font:inherit;border:1px solid #9aacc0;border-radius:7px;padding:7px 10px;background:white;color:#172637}body.dark button,body.dark input,body.dark select{background:#25384b;color:white;border-color:#61768b}h1{font-size:clamp(30px,4vw,48px);line-height:1.18;margin:0 0 15px}h2{font-size:26px;margin:0 0 14px}.hero{padding:48px 0 22px}.hero p{font-size:18px;max-width:900px;color:#506377}body.dark .hero p,body.dark .note{color:#bfd0e1}section{margin:24px 0 38px}figure{margin:24px 0}.panel{background:white;border:1px solid #d8e1eb;border-radius:12px;padding:20px;margin:14px 0}body.dark .panel{background:#1c2b3b;border-color:#38506a}.scroll{overflow:auto;max-height:66vh;border:1px solid #d8e1eb;border-radius:10px;background:white}body.dark .scroll{background:#1c2b3b;border-color:#38506a}table{border-collapse:collapse;width:100%;min-width:820px}th,td{padding:9px 12px;text-align:left;border-bottom:1px solid #dce5ee;vertical-align:top}th{position:sticky;top:0;background:#e9f0f8;color:#152a40;z-index:1}body.dark th{background:#29435d;color:white}tbody tr:hover{background:#edf4fc}body.dark tbody tr:hover{background:#28445f}td small{display:block;color:#597088}body.dark td small{color:#b8cadb}.number{font-variant-numeric:tabular-nums;white-space:nowrap}.winner{color:#116b3c;font-weight:750}body.dark .winner{color:#82e1a7}.filters{display:flex;gap:12px;flex-wrap:wrap;margin:14px 0}.filters label{display:grid;gap:4px;font-size:13px;font-weight:700}.note{color:#52677b;font-size:14px}pre{white-space:pre-wrap;overflow-wrap:anywhere;background:#0e2032;color:#f1f6fb;padding:17px;border-radius:8px;max-height:65vh;overflow:auto;font:13px/1.5 ui-monospace,monospace}.compare-grid{display:grid;grid-template-columns:repeat(2,minmax(0,1fr));gap:16px}.compare-grid .panel{min-width:0;margin:0}.compare-grid pre{max-height:70vh}details{border:1px solid #dce5ee;border-radius:8px;padding:10px 14px;margin:10px 0}body.dark details{border-color:#38506a}summary{cursor:pointer;font-weight:700}.actions{display:flex;gap:14px;flex-wrap:wrap}.source{overflow-wrap:anywhere}footer{padding:32px 0;color:#52677b}@media(max-width:800px){.compare-grid{grid-template-columns:1fr}}@media(max-width:650px){.panel{padding:14px}header nav{width:100%}th,td{padding:7px 9px}}"""
+ js = """function theme(){document.body.classList.toggle('dark');try{localStorage.setItem('wmb545-theme',document.body.classList.contains('dark')?'dark':'light')}catch(e){}}try{if(localStorage.getItem('wmb545-theme')==='dark')document.body.classList.add('dark')}catch(e){}function filter(){let q=(document.getElementById('q')?.value||'').toLowerCase(),p=document.getElementById('provider')?.value||'',k=document.getElementById('kind')?.value||'issue',cause=document.getElementById('cause')?.value||'',n=0;document.querySelectorAll('tr[data-result]').forEach(r=>{let show=(!q||r.dataset.search.includes(q))&&(!p||r.dataset.provider===p)&&(!cause||r.dataset.cause===cause)&&(k==='all'||(k==='issue'?r.dataset.kind!=='未触发诊断':r.dataset.kind===k));r.hidden=!show;if(show)n++});let c=document.getElementById('count');if(c)c.textContent=n;let e=document.getElementById('empty');if(e)e.hidden=n!==0}document.addEventListener('DOMContentLoaded',()=>{for(let id of ['q','provider','kind','cause'])document.getElementById(id)?.addEventListener('input',filter);let params=new URLSearchParams(location.search);for(let id of ['provider','kind','cause']){let select=document.getElementById(id);if(select&¶ms.has(id))select.value=params.get(id)}filter();let node=document.getElementById('evidence-content');if(node){let path=params.get('path')||'';if(!/^(artifacts|references|sources)\\/[A-Za-z0-9_.\\/-]+$/.test(path)||path.includes('..')){node.textContent='无效证据路径';return}document.getElementById('evidence-path').textContent=path;document.getElementById('evidence-raw').href=path;loadEvidence(path,node)}});"""
js += """
async function loadJson(path){let response=await fetch(path);if(!response.ok)throw Error('HTTP '+response.status);return response.json()}
async function loadEvidence(path,node){try{let index=await loadJson('artifact-shard-index.json'),shardPath=index[path];if(!shardPath)throw Error('证据索引缺失');let shard=await loadJson(shardPath),hash=shard.artifacts[path],value=shard.blobs[hash];if(typeof value!=='string')throw Error('证据内容缺失');node.textContent=value;let raw=document.getElementById('evidence-raw');raw.href=URL.createObjectURL(new Blob([value],{type:'text/plain;charset=utf-8'}));raw.download=path.split('/').pop()}catch(error){node.textContent='无法读取证据：'+error.message}}
async function renderResult(){let params=new URLSearchParams(location.search),provider=params.get('provider')||'',id=params.get('id')||'';let title=document.getElementById('result-title');if(!/^[a-z0-9-]+$/.test(provider)||!/^[A-Za-z0-9-]+$/.test(id)){title.textContent='无效结果地址';return}try{let index=await loadJson('result-shard-index.json'),key=provider+'|'+id,shardPath=index[key];if(!shardPath)throw Error('结果索引缺失');let shard=await loadJson(shardPath),row=shard.results[key];if(!row)throw Error('结果内容缺失');let names={moli:'Moli',lightpanda:'Lightpanda',obscura:'Obscura',chromium:'Chrome',kitesurf:'Kitesurf'},name=names[provider]||provider;title.textContent=name+' · '+id;document.title=title.textContent+' · WebMainBench 545';document.getElementById('result-source').textContent=row.original_url;let failure=['请求失败','空输出','错误页'].includes(row.diagnosis),detail=document.querySelector('[data-report-detail]');detail.dataset.reportDetail=failure?'failed':'success';document.getElementById('result-summary').textContent='状态：'+(row.ok?'返回成功':'请求失败')+'；诊断：'+row.diagnosis+'；最终调用 '+row.elapsed_ms.toFixed(0)+' ms；物理调用 '+row.physical_attempt_count+' 次，含重试总耗时 '+row.total_elapsed_ms.toFixed(0)+' ms。';let section=document.getElementById('result-failure');section.hidden=!failure;if(failure){document.getElementById('result-stage').textContent='阶段：'+(row.error_type||row.diagnosis);document.getElementById('result-error').textContent='程序错误：'+(row.error_message||'无');document.getElementById('result-log').textContent=row.provider_metadata?.stderr||row.provider_metadata?.browser_stderr||'无日志';document.getElementById('result-cause').textContent='复核原因：'+(row.review?.cause||'未记录');document.getElementById('result-treatment').textContent='处理：'+(row.review?.treatment||'未记录').replace(/[。；]+$/,'')+'；状态：'+(row.review?.status||'未记录')}let labels=[['text_edit','全文'],['code_edit','代码'],['formula_edit','公式'],['table_edit','表格内容'],['table_TEDS','表格结构']],body=document.getElementById('result-metrics');for(let [metric,label] of labels){let tr=document.createElement('tr'),entry=row.metrics?.[metric]||{};for(let value of [label,(!entry.success||entry.score==null)?'—':(entry.score*100).toFixed(1)+'%',entry.success?'参与计分':'不参与计分']){let td=document.createElement('td');td.textContent=value;tr.append(td)}body.append(tr)}document.getElementById('result-case').href='cases/index.html?id='+id;document.getElementById('result-output').href='viewer/index.html?path='+encodeURIComponent(row.artifact_path);document.getElementById('result-reference').href='viewer/index.html?path=references/'+id+'.md';document.getElementById('result-source-link').href='viewer/index.html?path=sources/'+id+'.html.txt';let physical=shard.physical[id]||[],list=document.getElementById('result-physical-list');for(let [number,attempt] of physical.entries()){let li=document.createElement('li'),a=document.createElement('a');a.href='viewer/index.html?path='+encodeURIComponent(attempt.physical_artifact_path);a.textContent='第 '+(number+1)+' 次调用 · '+attempt.elapsed_ms.toFixed(0)+' ms';li.append(a);list.append(li)}document.getElementById('result-physical').hidden=!physical.length}catch(error){title.textContent='无法载入结果：'+error.message}}
-async function renderCase(){let id=new URLSearchParams(location.search).get('id')||'',slot=document.getElementById('case-id');if(!/^[A-Za-z0-9-]+$/.test(id)){slot.textContent='无效任务地址';return}try{let dataset=await loadJson('dataset.json'),item=dataset.cases.find(x=>x.id===id);if(!item)throw Error('任务不存在');slot.textContent=id;document.getElementById('case-url').textContent=item.url;document.title='同页比较 '+id+' · WebMainBench 545';document.getElementById('case-source').href='viewer/index.html?path=sources/'+id+'.html.txt';document.getElementById('case-reference').href='viewer/index.html?path=references/'+id+'.md';let index=await loadJson('result-shard-index.json'),names={moli:'Moli',lightpanda:'Lightpanda',obscura:'Obscura',chromium:'Chrome',kitesurf:'Kitesurf'},metrics=['overall','text_edit','code_edit','formula_edit','table_edit','table_TEDS'],providers=dataset.coverage_plan.static_html,paths=[...new Set(providers.map(p=>index[p+'|'+id]))];if(paths.some(x=>!x))throw Error('同页结果不完整');let loaded=await Promise.all(paths.map(loadJson)),shards=Object.fromEntries(paths.map((p,i)=>[p,loaded[i]])),body=document.getElementById('case-results');for(let provider of providers){let row=shards[index[provider+'|'+id]].results[provider+'|'+id];if(!row)throw Error('同页方法缺失');let tr=document.createElement('tr'),td=document.createElement('td'),link=document.createElement('a');link.href='results/index.html?provider='+provider+'&id='+id;link.textContent=names[provider]||provider;td.append(link);tr.append(td);td=document.createElement('td');td.textContent=row.diagnosis;tr.append(td);for(let metric of metrics){td=document.createElement('td');let score=row.metrics?.[metric]?.score;td.textContent=(!row.metrics?.[metric]?.success||score==null)?'—':(score*100).toFixed(1)+'%';tr.append(td)}body.append(tr)}}catch(error){slot.textContent='无法载入同页结果：'+error.message}}
+async function renderCase(){
+ let id=new URLSearchParams(location.search).get('id')||'',slot=document.getElementById('case-id');
+ if(!/^[A-Za-z0-9-]+$/.test(id)){slot.textContent='无效任务地址';return}
+ try{
+ let dataset=await loadJson('dataset.json'),item=dataset.cases.find(x=>x.id===id);
+ if(!item)throw Error('任务不存在');
+ slot.textContent=id;document.getElementById('case-url').textContent=item.url;
+ document.title='同页比较 '+id+' · WebMainBench 545';
+ document.getElementById('case-source').href='viewer/index.html?path=sources/'+id+'.html.txt';
+ document.getElementById('case-reference').href='viewer/index.html?path=references/'+id+'.md';
+ let resultIndex=await loadJson('result-shard-index.json'),artifactIndex=await loadJson('artifact-shard-index.json');
+ let names={moli:'Moli',lightpanda:'Lightpanda',obscura:'Obscura',chromium:'Chrome',kitesurf:'Kitesurf'};
+ let metrics=['overall','text_edit','code_edit','formula_edit','table_edit','table_TEDS'];
+ let providers=dataset.coverage_plan.static_html,referencePath='references/'+id+'.md';
+ let paths=[...new Set([...providers.map(p=>resultIndex[p+'|'+id]),artifactIndex[referencePath]])];
+ if(paths.some(x=>!x))throw Error('同页证据不完整');
+ let loaded=await Promise.all(paths.map(loadJson)),shards=Object.fromEntries(paths.map((p,i)=>[p,loaded[i]]));
+ let referenceShard=shards[artifactIndex[referencePath]],reference=referenceShard.blobs[referenceShard.artifacts[referencePath]];
+ if(typeof reference!=='string')throw Error('人工答案缺失');
+ let grid=document.getElementById('case-comparison'),referencePanel=document.createElement('article');
+ referencePanel.className='panel';let referenceTitle=document.createElement('h2');referenceTitle.textContent='人工主内容答案';
+ let referenceNote=document.createElement('p');referenceNote.className='note';referenceNote.textContent='人工答案是评测参考，判断正文与格式仍需结合冻结网页和各方法输出。';
+ let referenceBody=document.createElement('pre');referenceBody.textContent=reference;
+ referencePanel.append(referenceTitle,referenceNote,referenceBody);grid.append(referencePanel);
+ let body=document.getElementById('case-results');
+ for(let provider of providers){
+ let shard=shards[resultIndex[provider+'|'+id]],row=shard.results[provider+'|'+id];
+ if(!row)throw Error('同页方法缺失');
+ let output=shard.blobs[shard.artifacts[row.artifact_path]];
+ if(typeof output!=='string')throw Error('方法输出缺失：'+provider);
+ let tr=document.createElement('tr'),td=document.createElement('td'),link=document.createElement('a');
+ link.href='results/index.html?provider='+provider+'&id='+id;link.textContent=names[provider]||provider;
+ td.append(link);tr.append(td);td=document.createElement('td');td.textContent=row.diagnosis;tr.append(td);
+ for(let metric of metrics){td=document.createElement('td');let entry=row.metrics?.[metric];td.textContent=(!entry?.success||entry.score==null)?'—':(entry.score*100).toFixed(1)+'%';tr.append(td)}
+ body.append(tr);
+ let panel=document.createElement('article');panel.className='panel result-output';
+ let title=document.createElement('h2');title.textContent=names[provider]||provider;
+ let note=document.createElement('p');note.className='note';note.textContent='诊断：'+row.diagnosis+'；正文 '+row.output_chars+' 字。';
+ let detail=document.createElement('a');detail.href=link.href;detail.textContent='查看评分、错误与调用记录';
+ let raw=document.createElement('pre');raw.textContent=output||'（空输出）';
+ panel.append(title,note,detail,raw);grid.append(panel);
+ }
+ }catch(error){slot.textContent='无法载入同页结果：'+error.message}
+}
document.addEventListener('DOMContentLoaded',()=>{if(document.getElementById('result-title'))renderResult();if(document.getElementById('case-id'))renderCase()});
"""
(out / "assets").mkdir(exist_ok=True)
@@ -2867,7 +2909,7 @@ def all_call_latency_table():
return table(['方法', '全部逻辑调用', '均值', 'P50', 'P95'], rows)
home = ('' + prose('home_title') + ' ' + prose('home_lead') + '
' + prose('infographic_caption') + ' ' + prose('infographic_caption') + ' 核心结果 ' + metric_table() + '' + prose('score_note') + '
全部页面的调用耗时 ' + all_call_latency_table() + '' + prose('all_latency_note') + '
成功页面的调用耗时与重试成本 ' + latency_table() + '' + prose('latency_note') + '
冻结网页与人工答案 ' + prose('dataset_lead') + '
搜索任务或网址 方法全部方法 ' +
''.join('' + esc(labels[provider]) + " " for provider in providers) +
' 诊断' +
''.join('' + esc("全部问题" if kind == "issue" else "全部结果" if kind == "all" else kind) + " " for kind in options) +
+ ' 已复核原因全部原因 ' +
+ ''.join('' + esc(kind) + ' ' for kind in cause_order) +
"
")
result_rows = []
for row in records:
case_id, provider = row["case_id"], row["provider"]
kind = categories[row["attempt_id"]]
+ review = reviewed_failures.get((provider, case_id))
result_rows.append('' +
- anchor("results/index.html?provider=" + provider + "&id=" + case_id, case_id) + " " +
- esc(labels[provider]) + " " + esc(kind) + " " +
+ anchor("cases/index.html?id=" + case_id, case_id) + " " +
+ anchor("results/index.html?provider=" + provider + "&id=" + case_id, labels[provider]) + " " + esc(kind) + " " +
percent(score(row, "text_edit")) + " " + f'{row["elapsed_ms"]:,.0f} ms' +
" " + anchor("cases/index.html?id=" + case_id, "同页") + " 逐项结果与问题定位 ' + prose('analysis_lead') + '
当前显示 当前条件无结果。
' +
table(["任务", "方法", "诊断", "全文分", "最终调用", "同页"], result_rows))
page("cases/index.html", "同页比较",
- ''
+ ''
'冻结网页 人工答案 ' + anchor("dataset/index.html", "返回数据集") + '
'
'')
+ ''
'
'
@@ -3011,6 +3060,28 @@ def bound_artifact(path, expected, shard_path):
anchor("analysis/index.html?provider=" + provider + "&kind=" + kind,
str(sum(categories[row["attempt_id"]] == kind for row in selected))) + ""
for kind in diagnostic_kinds) + "")
+ cause_owners = {
+ "Moli 解析器崩溃": "Moli 引擎",
+ "Kitesurf 服务错误": "Kitesurf 服务",
+ "Kitesurf 请求超时": "Kitesurf 服务或传输，内部阶段未定位",
+ "Kitesurf 空正文": "Kitesurf 接口，内部原因未知",
+ "Obscura 空响应": "Obscura",
+ "Obscura 导航循环": "Obscura",
+ "冻结页跳转受阻": "共享输入与隔离环境",
+ }
+ cause_counts = Counter((row["category_label"], row["provider"])
+ for row in failure_analysis["reviewed_failures"])
+ cause_rows = []
+ for cause in cause_order:
+ cells = []
+ for provider in providers:
+ count = cause_counts[cause, provider]
+ label = str(count) if count else "—"
+ target = "analysis/index.html?provider=" + provider + "&cause=" + cause
+ cells.append('
' +
+ (anchor(target, label) if count else label) + ' ')
+ cause_rows.append('
' + esc(cause) + ' ' + ''.join(cells) +
+ '' + esc(cause_owners[cause]) + ' ')
page("guide/index.html", "结果解读",
'
结果解读 ' + esc(outcome) + " " + prose('guide_lead') + '
质量与样本 ' +
metric_table() + '' + prose('guide_scope') + '
核对完整输出 ' + prose('guide_outputs') + '
' +
@@ -3021,8 +3092,8 @@ def bound_artifact(path, expected, shard_path):
'' + prose('guide_method_scope') + '
' +
table(["方法", *diagnostic_kinds], diagnostic_rows) + '错误类别 ' +
table(["类别", "结果数"], ["" + esc(kind) + " " + str(count) + " ' + prose('guide_diagnosis') + '
已复核的机器问题原因 ' +
- table(["原因", "结果数"], ["" + esc(kind) + " " + str(count) + " ' + prose('guide_diagnosis') + '
已复核问题：错误类型 × 方法 ' +
+ table(["错误类型", *[labels[provider] for provider in providers], "归属或定位"], cause_rows) +
'' + str(len(reviewed_failures)) + '/' + str(machine_issue_count) + ' ' + prose('review_scope_failure') +
'；' + str(len(records) - len(reviewed_failures)) + ' ' + prose('review_scope_pass') + '
' +
anchor('failure-analysis.json', '查看逐条失败原因与处理') +
diff --git a/research/report-changes.json b/research/report-changes.json
index 5aefe4b58..6b0e21395 100644
--- a/research/report-changes.json
+++ b/research/report-changes.json
@@ -1275,6 +1275,17 @@
}
],
"releases": [
+ {
+ "id": "2026-09-23-webmainbench-error-attribution-and-case-comparison",
+ "comparison_kinds": ["webmainbench_frozen_html_to_markdown"],
+ "date": "2026-09-23",
+ "title": "按方法定位机器问题并展示同页完整输出",
+ "title_i18n": {"en": "Attribute machine issues by method and show complete same-page outputs"},
+ "summary": "机器问题图与解读页交叉列出错误类型、方法和可确认的归属。逐题对照页在同一处显示人工主内容答案与五种方法的完整输出，结果清单可按复核原因筛选。",
+ "summary_i18n": {"en": "The issue graphic and interpretation cross-tabulate error type, method, and evidenced attribution. Each case comparison displays the human main-content reference beside all five complete outputs, and the result inventory can filter by reviewed cause."},
+ "changes": [["错误归属与逐题阅读", "汇总数不能直接显示每种错误发生在哪个方法，逐题页只列分数而需分别打开答案和输出。", "增加错误类型与方法矩阵、归属说明及原因筛选；同页显示参考答案和五种完整结果。", "545 题输入、2725 条结果、分数、耗时和已复核的 28 条机器问题均不变。"]],
+ "changes_i18n": {"en": [["Issue attribution and case reading", "Aggregate counts did not show which method encountered each issue, and case pages listed scores without the reference and outputs in one view.", "Add an error-type-by-method matrix, attribution notes, and cause filtering; display the reference and five full outputs on each case page.", "The 545 inputs, 2,725 results, scores, timings, and 28 reviewed machine issues remain unchanged."]]}
+ },
{
"id": "2026-09-23-webmainbench-complete-545-five-methods",
"comparison_kinds": ["webmainbench_frozen_html_to_markdown"],
diff --git a/scripts/check_webmainbench_full_report.py b/scripts/check_webmainbench_full_report.py
index 630ef7226..7b26da97e 100644
--- a/scripts/check_webmainbench_full_report.py
+++ b/scripts/check_webmainbench_full_report.py
@@ -83,14 +83,16 @@ def check(root: Path) -> None:
if not home.select_one('source[media="(max-width:650px)"][srcset="assets/infographic-mobile.png"]'):
raise ValueError("Mobile infographic route missing")
analysis = BeautifulSoup((root / "analysis/index.html").read_text(encoding="utf-8"), "html.parser")
- if len(analysis.select("tr[data-result]")) != 2725 or not analysis.select_one("#kind"):
+ if (len(analysis.select("tr[data-result]")) != 2725 or not analysis.select_one("#kind")
+ or not analysis.select_one("#cause")):
raise ValueError("Result inventory incomplete")
if not any(row.get("data-kind") != "未触发诊断" for row in analysis.select("tr[data-result]")):
raise ValueError("Default diagnostic population is empty")
if not (root / "en/index.html").is_file() or "返回中文报告" not in (root / "en/index.html").read_text(encoding="utf-8"):
raise ValueError("English not-generated route missing")
case_template = BeautifulSoup((root / "cases/index.html").read_text(encoding="utf-8"), "html.parser")
- if not case_template.select_one("#case-results") or not case_template.select_one("#case-source"):
+ if (not case_template.select_one("#case-results") or not case_template.select_one("#case-source")
+ or not case_template.select_one("#case-reference") or not case_template.select_one("#case-comparison")):
raise ValueError("Same-case comparison template is incomplete")
result_template = check_presentation((root / "results/index.html").read_text(encoding="utf-8"))
if not result_template.select_one('[data-result-summary]') or not result_template.select_one('#result-logs pre[data-kind="error"]'):
@@ -122,6 +124,17 @@ def check(root: Path) -> None:
for cell in cells
)):
raise ValueError("Interpretation diagnosis matrix differs from measured results")
+ cause_counts = Counter((entry["category_label"], entry["provider"])
+ for entry in failure_analysis["reviewed_failures"])
+ cause_cells = guide.select("td[data-cause-provider][data-cause-kind]")
+ expected_cause_cells = {(kind, provider) for kind, _ in cause_counts for provider in providers}
+ if ({(cell["data-cause-kind"], cell["data-cause-provider"]) for cell in cause_cells} != expected_cause_cells
+ or len(cause_cells) != len(expected_cause_cells)
+ or any(cell.get_text(strip=True) !=
+ (str(cause_counts[cell["data-cause-kind"], cell["data-cause-provider"]])
+ if cause_counts[cell["data-cause-kind"], cell["data-cause-provider"]] else "—")
+ for cell in cause_cells)):
+ raise ValueError("Per-method cause matrix differs from reviewed evidence")
reviewed_by_key = {(row["provider"], row["case_id"]): row for row in failure_analysis["reviewed_failures"]}
for row in records:
check_result_payload(row, index, artifact_index, shards)
diff --git a/tests/test_webmainbench_full_report.py b/tests/test_webmainbench_full_report.py
index 954cc5097..024045b15 100644
--- a/tests/test_webmainbench_full_report.py
+++ b/tests/test_webmainbench_full_report.py
@@ -86,8 +86,12 @@ def test_full_report_renders_five_way_details_and_failure_evidence(tmp_path: Pat
assert detail.select_one('[data-failure-diagnosis]') is not None
comparison = BeautifulSoup((tmp_path / "cases/index.html").read_text(encoding="utf-8"), "html.parser")
assert comparison.select_one("#case-results") is not None
+ assert comparison.select_one("#case-comparison") is not None
+ assert comparison.select_one("#case-reference") is not None
analysis = BeautifulSoup((tmp_path / "analysis/index.html").read_text(encoding="utf-8"), "html.parser")
assert len(analysis.select("tr[data-result]")) == 5
+ assert analysis.select_one("#cause") is not None
+ assert analysis.select_one('tr[data-cause="测试环境错误"]') is not None
assert json.loads((tmp_path / "report-manifest.json").read_text())["languages"] == ["zh"]
home_html = (tmp_path / "index.html").read_text(encoding="utf-8")
with pytest.raises(ValueError, match="extremum"):
From dc920eff0f8769aa7cd2d6c2c6d98439394deecf Mon Sep 17 00:00:00 2001
From: lanyue-llk <270302213+lanyue-llk@users.noreply.github.com>
Date: Wed, 23 Sep 2026 21:53:49 +0800
Subject: [PATCH 03/14] Bind published report and repair release checks
---
.../webmainbench-full/publication-target.json | 8 +++++++
harness/web_fetch/report.py | 5 +++-
research/report-changes.json | 8 ++-----
scripts/assemble_webmainbench_full.py | 2 ++
.../review_webmainbench_teds_correction.py | 6 ++++-
tests/test_certified_webmainbench_full.py | 5 +++-
tests/test_report_changes.py | 2 +-
tests/test_webmainbench_full_report.py | 4 ++--
tests/test_webmainbench_review_guards.py | 24 +++++++++++++++++++
9 files changed, 52 insertions(+), 12 deletions(-)
create mode 100644 evaluation/webmainbench-full/publication-target.json
diff --git a/evaluation/webmainbench-full/publication-target.json b/evaluation/webmainbench-full/publication-target.json
new file mode 100644
index 000000000..0c5bab881
--- /dev/null
+++ b/evaluation/webmainbench-full/publication-target.json
@@ -0,0 +1,8 @@
+{
+ "schema_version": 1,
+ "profile_sha256": "8cbceaf9694e88e7c1ddbba3263573dcccf69b1ac89d7e9ea5aa098f2a3bab02",
+ "host": "https://artifact-site.lexmount.cn",
+ "site_slug": "zrqPjO672dgg",
+ "share_url": "https://artifact-site.lexmount.cn/v/Avz3qzo6EEujI0iln1LAzCUF93I47OEC",
+ "expected_remote_version": "ver_773460c631394cc79b919753de576645"
+}
diff --git a/harness/web_fetch/report.py b/harness/web_fetch/report.py
index dea961819..c3ff7de51 100644
--- a/harness/web_fetch/report.py
+++ b/harness/web_fetch/report.py
@@ -2909,7 +2909,7 @@ def all_call_latency_table():
return table(['方法', '全部逻辑调用', '均值', 'P50', 'P95'], rows)
home = ('' + prose('home_title') + ' ' + prose('home_lead') + '
' + prose('infographic_caption') + ' ' + prose('infographic_caption') + ' 核心结果 ' + metric_table() + '' + prose('score_note') + '
全部页面的调用耗时 ' + all_call_latency_table() + '' + prose('all_latency_note') + '
成功页面的调用耗时与重试成本 ' + latency_table() + '' + prose('latency_note') + '
' + prose('home_title') + ' ' + prose('home_lead') + '
' + prose('infographic_caption') + ' 核心结果 ' + metric_table() + '' + prose('score_note') + '
修复后 Moli 的完整复测 ' + prose('followup_home') + '
' +
+ anchor('moli-update/index.html', '查看复测结果与逐页证据') + '全部页面的调用耗时 ' + all_call_latency_table() + '' + prose('all_latency_note') + '
成功页面的调用耗时与重试成本 ' + latency_table() + '' + prose('latency_note') + '
问题分布 ' + table(["自动诊断", "结果数"], ['' + anchor("analysis/index.html?kind=" + kind, kind) + ' ' + str(count) + ' ' + prose('diagnostic_note') + '
' + label +
+ ' ' + f"{item['baseline'] * 100:.2f}%" + '' + str(item['baseline_n']) +
+ ' 页计分 ' + f"{item['candidate'] * 100:.2f}%" + '' +
+ str(item['candidate_n']) + ' 页计分 ' + anchor('cases/index.html?id=' + case_id, case_id) +
+ ' 解析器崩溃 返回成功；全文分 ' + f"{item['text_score'] * 100:.1f}%" +
+ ' ' + anchor('moli-followup/' + case_id + '.txt', '修复后完整输出') + ' ' + anchor('cases/index.html?id=' + item['case_id'], item['case_id']) +
+ ' 原站跳转被隔离环境阻断 Moli 修复后的 545 页复测 ' +
+ prose('followup_lead') + '
修复了什么 ' +
+ prose('followup_mechanism') + '
完整范围的结果 ' +
+ '请求返回成功由 ' + str(followup['baseline_successes']) + '/545 升至 ' +
+ str(followup['candidate_successes']) + '/545；修复后共 ' +
+ str(followup['candidate_physical_calls']) + ' 次物理调用。
' +
+ table(['指标', '五方法基线中的 Moli', '修复后 Moli 单方法复测'], followup_metric_rows) +
+ '' + prose('followup_metric_scope') + '
恢复的页面与剩余失败 ' +
+ table(['任务', '原结果', '复测结果', '证据'], recovered_rows) +
+ '' + prose('followup_quality') + '
' +
+ table(['任务', '剩余问题'], remaining_rows) + '证据与适用范围 ' + prose('followup_limit') + '
' +
+ anchor('moli-followup.json', '完整范围与哈希绑定') + ' · ' +
+ anchor('https://github.com/lexmount/moli/pull/785', 'Moli 修复代码与 CI') +
+ '
冻结网页与人工答案 ' + prose('dataset_lead') + '
搜索任务或网址 方法全部方法 ' +
''.join('' + esc(labels[provider]) + " " for provider in providers) +
' 诊断' +
@@ -3060,15 +3108,6 @@ def bound_artifact(path, expected, shard_path):
anchor("analysis/index.html?provider=" + provider + "&kind=" + kind,
str(sum(categories[row["attempt_id"]] == kind for row in selected))) + ""
for kind in diagnostic_kinds) + "")
- cause_owners = {
- "Moli 解析器崩溃": "Moli 引擎",
- "Kitesurf 服务错误": "Kitesurf 服务",
- "Kitesurf 请求超时": "Kitesurf 服务或传输，内部阶段未定位",
- "Kitesurf 空正文": "Kitesurf 接口，内部原因未知",
- "Obscura 空响应": "Obscura",
- "Obscura 导航循环": "Obscura",
- "冻结页跳转受阻": "共享输入与隔离环境",
- }
cause_counts = Counter((row["category_label"], row["provider"])
for row in failure_analysis["reviewed_failures"])
unknown_causes = {cause for cause, _ in cause_counts} - set(cause_owners)
@@ -3107,6 +3146,14 @@ def bound_artifact(path, expected, shard_path):
page("reproduce/index.html", "复现依据",
'复现依据 ' + prose('reproduce_lead') + '
' +
" · ".join(anchor(name, name) for name in evidence_links) + "
")
+ releases = json.loads((Path(__file__).resolve().parents[2] / 'research/report-changes.json').read_text(encoding='utf-8'))['releases']
+ webmainbench_releases = [item for item in releases if 'webmainbench_frozen_html_to_markdown' in item.get('comparison_kinds', [])]
+ history_rows = ['' + esc(item['date']) +
+ ' ' + esc(item['title']) + ' ' + esc(item['summary']) + ' 报告更新记录 按发布记录说明读者可见的结果与内容变化。
输出 ')
+ metrics = {name: {'baseline': 0.4, 'candidate': 0.5, 'baseline_n': 542,
+ 'candidate_n': 544} for name in ('text_edit', 'code_edit', 'formula_edit', 'table_edit')}
+ receipt = {'kind': 'webmainbench_moli_full_diagnostic_followup', 'case_count': 545,
+ 'baseline_successes': 542, 'candidate_successes': 544, 'recovered': recovered,
+ 'remaining': [{'case_id': 'still-failed'}], 'metrics_original_scorer': metrics}
+ receipt_path = tmp_path / 'moli-followup.json'
+ receipt_path.write_text(json.dumps(receipt))
+ binding = {'relative_path': 'moli-followup.json',
+ 'sha256': hashlib.sha256(receipt_path.read_bytes()).hexdigest()}
+ (tmp_path / 'index.html').write_text('复测 ')
+ rows = ''.join('40.00% 542 50.00% 544 542/545 → 544/545
' + rows + ''.join(links)
+ followup_page = tmp_path / 'moli-update/index.html'
+ followup_page.write_text(page)
+ check_moli_followup(tmp_path, binding)
+ followup_page.write_text(page.replace('50.00%', '51.00%', 1))
+ with pytest.raises(ValueError, match='score differs'):
+ check_moli_followup(tmp_path, binding)
+ followup_page.write_text(page)
+ (tmp_path / 'moli-followup/restored-one.txt').write_text('Changed output')
+ with pytest.raises(ValueError, match='output changed'):
+ check_moli_followup(tmp_path, binding)
From 56751154c00b48899e086f3e1aff23d07c2eb00e Mon Sep 17 00:00:00 2001
From: lanyue-llk <270302213+lanyue-llk@users.noreply.github.com>
Date: Thu, 24 Sep 2026 12:54:51 +0800
Subject: [PATCH 05/14] Open Moli follow-up outputs in report viewer
---
evaluation/webmainbench-full/publication-target.json | 2 +-
harness/web_fetch/report.py | 6 +++---
scripts/check_webmainbench_full_report.py | 2 +-
tests/test_webmainbench_full_report.py | 2 +-
4 files changed, 6 insertions(+), 6 deletions(-)
diff --git a/evaluation/webmainbench-full/publication-target.json b/evaluation/webmainbench-full/publication-target.json
index 61bd00487..a945003aa 100644
--- a/evaluation/webmainbench-full/publication-target.json
+++ b/evaluation/webmainbench-full/publication-target.json
@@ -4,5 +4,5 @@
"host": "https://artifact-site.lexmount.cn",
"site_slug": "zrqPjO672dgg",
"share_url": "https://artifact-site.lexmount.cn/v/Avz3qzo6EEujI0iln1LAzCUF93I47OEC",
- "expected_remote_version": "ver_2e197dd714c84426a7fe309de4635f93"
+ "expected_remote_version": "ver_91722963e35744c7920e47bc2578f090"
}
diff --git a/harness/web_fetch/report.py b/harness/web_fetch/report.py
index 4da52c360..ae8e60a11 100644
--- a/harness/web_fetch/report.py
+++ b/harness/web_fetch/report.py
@@ -2777,10 +2777,10 @@ def classify(row):
categories = {row["attempt_id"]: classify(row) for row in records}
counts = Counter(categories.values())
css = """*{box-sizing:border-box}body{margin:0;background:#f4f7fb;color:#172637;font:16px/1.55 system-ui,sans-serif}body.dark{background:#101923;color:#edf3fa}a{color:#1859a5}body.dark a{color:#92c7ff}.wrap{width:min(1200px,calc(100% - 32px));margin:auto}header{background:#142d4b;color:white}header .wrap{display:flex;align-items:center;gap:18px;flex-wrap:wrap;padding:15px 0}header a{color:white;text-decoration:none}header strong{margin-right:auto}nav{display:flex;gap:14px;flex-wrap:wrap}button,input,select{font:inherit;border:1px solid #9aacc0;border-radius:7px;padding:7px 10px;background:white;color:#172637}body.dark button,body.dark input,body.dark select{background:#25384b;color:white;border-color:#61768b}h1{font-size:clamp(30px,4vw,48px);line-height:1.18;margin:0 0 15px}h2{font-size:26px;margin:0 0 14px}.hero{padding:48px 0 22px}.hero p{font-size:18px;max-width:900px;color:#506377}body.dark .hero p,body.dark .note{color:#bfd0e1}section{margin:24px 0 38px}figure{margin:24px 0}.panel{background:white;border:1px solid #d8e1eb;border-radius:12px;padding:20px;margin:14px 0}body.dark .panel{background:#1c2b3b;border-color:#38506a}.scroll{overflow:auto;max-height:66vh;border:1px solid #d8e1eb;border-radius:10px;background:white}body.dark .scroll{background:#1c2b3b;border-color:#38506a}table{border-collapse:collapse;width:100%;min-width:820px}th,td{padding:9px 12px;text-align:left;border-bottom:1px solid #dce5ee;vertical-align:top}th{position:sticky;top:0;background:#e9f0f8;color:#152a40;z-index:1}body.dark th{background:#29435d;color:white}tbody tr:hover{background:#edf4fc}body.dark tbody tr:hover{background:#28445f}td small{display:block;color:#597088}body.dark td small{color:#b8cadb}.number{font-variant-numeric:tabular-nums;white-space:nowrap}.winner{color:#116b3c;font-weight:750}body.dark .winner{color:#82e1a7}.filters{display:flex;gap:12px;flex-wrap:wrap;margin:14px 0}.filters label{display:grid;gap:4px;font-size:13px;font-weight:700}.note{color:#52677b;font-size:14px}pre{white-space:pre-wrap;overflow-wrap:anywhere;background:#0e2032;color:#f1f6fb;padding:17px;border-radius:8px;max-height:65vh;overflow:auto;font:13px/1.5 ui-monospace,monospace}.compare-grid{display:grid;grid-template-columns:repeat(2,minmax(0,1fr));gap:16px}.compare-grid .panel{min-width:0;margin:0}.compare-grid pre{max-height:70vh}details{border:1px solid #dce5ee;border-radius:8px;padding:10px 14px;margin:10px 0}body.dark details{border-color:#38506a}summary{cursor:pointer;font-weight:700}.actions{display:flex;gap:14px;flex-wrap:wrap}.source{overflow-wrap:anywhere}footer{padding:32px 0;color:#52677b}@media(max-width:800px){.compare-grid{grid-template-columns:1fr}}@media(max-width:650px){.panel{padding:14px}header nav{width:100%}th,td{padding:7px 9px}}"""
- js = """function theme(){document.body.classList.toggle('dark');try{localStorage.setItem('wmb545-theme',document.body.classList.contains('dark')?'dark':'light')}catch(e){}}try{if(localStorage.getItem('wmb545-theme')==='dark')document.body.classList.add('dark')}catch(e){}function filter(){let q=(document.getElementById('q')?.value||'').toLowerCase(),p=document.getElementById('provider')?.value||'',k=document.getElementById('kind')?.value||'issue',cause=document.getElementById('cause')?.value||'',n=0;document.querySelectorAll('tr[data-result]').forEach(r=>{let show=(!q||r.dataset.search.includes(q))&&(!p||r.dataset.provider===p)&&(!cause||r.dataset.cause===cause)&&(k==='all'||(k==='issue'?r.dataset.kind!=='未触发诊断':r.dataset.kind===k));r.hidden=!show;if(show)n++});let c=document.getElementById('count');if(c)c.textContent=n;let e=document.getElementById('empty');if(e)e.hidden=n!==0}document.addEventListener('DOMContentLoaded',()=>{for(let id of ['q','provider','kind','cause'])document.getElementById(id)?.addEventListener('input',filter);let params=new URLSearchParams(location.search);for(let id of ['provider','kind','cause']){let select=document.getElementById(id);if(select&¶ms.has(id))select.value=params.get(id)}filter();let node=document.getElementById('evidence-content');if(node){let path=params.get('path')||'';if(!/^(artifacts|references|sources)\\/[A-Za-z0-9_.\\/-]+$/.test(path)||path.includes('..')){node.textContent='无效证据路径';return}document.getElementById('evidence-path').textContent=path;document.getElementById('evidence-raw').href=path;loadEvidence(path,node)}});"""
+ js = """function theme(){document.body.classList.toggle('dark');try{localStorage.setItem('wmb545-theme',document.body.classList.contains('dark')?'dark':'light')}catch(e){}}try{if(localStorage.getItem('wmb545-theme')==='dark')document.body.classList.add('dark')}catch(e){}function filter(){let q=(document.getElementById('q')?.value||'').toLowerCase(),p=document.getElementById('provider')?.value||'',k=document.getElementById('kind')?.value||'issue',cause=document.getElementById('cause')?.value||'',n=0;document.querySelectorAll('tr[data-result]').forEach(r=>{let show=(!q||r.dataset.search.includes(q))&&(!p||r.dataset.provider===p)&&(!cause||r.dataset.cause===cause)&&(k==='all'||(k==='issue'?r.dataset.kind!=='未触发诊断':r.dataset.kind===k));r.hidden=!show;if(show)n++});let c=document.getElementById('count');if(c)c.textContent=n;let e=document.getElementById('empty');if(e)e.hidden=n!==0}document.addEventListener('DOMContentLoaded',()=>{for(let id of ['q','provider','kind','cause'])document.getElementById(id)?.addEventListener('input',filter);let params=new URLSearchParams(location.search);for(let id of ['provider','kind','cause']){let select=document.getElementById(id);if(select&¶ms.has(id))select.value=params.get(id)}filter();let node=document.getElementById('evidence-content');if(node){let path=params.get('path')||'';if(!/^(artifacts|references|sources|moli-followup)\\/[A-Za-z0-9_.\\/-]+$/.test(path)||path.includes('..')){node.textContent='无效证据路径';return}document.getElementById('evidence-path').textContent=path;document.getElementById('evidence-raw').href=path;loadEvidence(path,node)}});"""
js += """
async function loadJson(path){let response=await fetch(path);if(!response.ok)throw Error('HTTP '+response.status);return response.json()}
-async function loadEvidence(path,node){try{let index=await loadJson('artifact-shard-index.json'),shardPath=index[path];if(!shardPath)throw Error('证据索引缺失');let shard=await loadJson(shardPath),hash=shard.artifacts[path],value=shard.blobs[hash];if(typeof value!=='string')throw Error('证据内容缺失');node.textContent=value;let raw=document.getElementById('evidence-raw');raw.href=URL.createObjectURL(new Blob([value],{type:'text/plain;charset=utf-8'}));raw.download=path.split('/').pop()}catch(error){node.textContent='无法读取证据：'+error.message}}
+async function loadEvidence(path,node){try{let value;if(path.startsWith('moli-followup/')){let response=await fetch(path);if(!response.ok)throw Error('HTTP '+response.status);value=await response.text()}else{let index=await loadJson('artifact-shard-index.json'),shardPath=index[path];if(!shardPath)throw Error('证据索引缺失');let shard=await loadJson(shardPath),hash=shard.artifacts[path];value=shard.blobs[hash]}if(typeof value!=='string')throw Error('证据内容缺失');node.textContent=value;let raw=document.getElementById('evidence-raw');raw.href=URL.createObjectURL(new Blob([value],{type:'text/plain;charset=utf-8'}));raw.download=path.split('/').pop()}catch(error){node.textContent='无法读取证据：'+error.message}}
async function renderResult(){let params=new URLSearchParams(location.search),provider=params.get('provider')||'',id=params.get('id')||'';let title=document.getElementById('result-title');if(!/^[a-z0-9-]+$/.test(provider)||!/^[A-Za-z0-9-]+$/.test(id)){title.textContent='无效结果地址';return}try{let index=await loadJson('result-shard-index.json'),key=provider+'|'+id,shardPath=index[key];if(!shardPath)throw Error('结果索引缺失');let shard=await loadJson(shardPath),row=shard.results[key];if(!row)throw Error('结果内容缺失');let names={moli:'Moli',lightpanda:'Lightpanda',obscura:'Obscura',chromium:'Chrome',kitesurf:'Kitesurf'},name=names[provider]||provider;title.textContent=name+' · '+id;document.title=title.textContent+' · WebMainBench 545';document.getElementById('result-source').textContent=row.original_url;let failure=['请求失败','空输出','错误页'].includes(row.diagnosis),detail=document.querySelector('[data-report-detail]');detail.dataset.reportDetail=failure?'failed':'success';document.getElementById('result-summary').textContent='状态：'+(row.ok?'返回成功':'请求失败')+'；诊断：'+row.diagnosis+'；最终调用 '+row.elapsed_ms.toFixed(0)+' ms；物理调用 '+row.physical_attempt_count+' 次，含重试总耗时 '+row.total_elapsed_ms.toFixed(0)+' ms。';let section=document.getElementById('result-failure');section.hidden=!failure;if(failure){document.getElementById('result-stage').textContent='阶段：'+(row.error_type||row.diagnosis);document.getElementById('result-error').textContent='程序错误：'+(row.error_message||'无');document.getElementById('result-log').textContent=row.provider_metadata?.stderr||row.provider_metadata?.browser_stderr||'无日志';document.getElementById('result-cause').textContent='复核原因：'+(row.review?.cause||'未记录');document.getElementById('result-treatment').textContent='处理：'+(row.review?.treatment||'未记录').replace(/[。；]+$/,'')+'；状态：'+(row.review?.status||'未记录')}let labels=[['text_edit','全文'],['code_edit','代码'],['formula_edit','公式'],['table_edit','表格内容'],['table_TEDS','表格结构']],body=document.getElementById('result-metrics');for(let [metric,label] of labels){let tr=document.createElement('tr'),entry=row.metrics?.[metric]||{};for(let value of [label,(!entry.success||entry.score==null)?'—':(entry.score*100).toFixed(1)+'%',entry.success?'参与计分':'不参与计分']){let td=document.createElement('td');td.textContent=value;tr.append(td)}body.append(tr)}document.getElementById('result-case').href='cases/index.html?id='+id;document.getElementById('result-output').href='viewer/index.html?path='+encodeURIComponent(row.artifact_path);document.getElementById('result-reference').href='viewer/index.html?path=references/'+id+'.md';document.getElementById('result-source-link').href='viewer/index.html?path=sources/'+id+'.html.txt';let physical=shard.physical[id]||[],list=document.getElementById('result-physical-list');for(let [number,attempt] of physical.entries()){let li=document.createElement('li'),a=document.createElement('a');a.href='viewer/index.html?path='+encodeURIComponent(attempt.physical_artifact_path);a.textContent='第 '+(number+1)+' 次调用 · '+attempt.elapsed_ms.toFixed(0)+' ms';li.append(a);list.append(li)}document.getElementById('result-physical').hidden=!physical.length}catch(error){title.textContent='无法载入结果：'+error.message}}
async function renderCase(){
let id=new URLSearchParams(location.search).get('id')||'',slot=document.getElementById('case-id');
@@ -2936,7 +2936,7 @@ def all_call_latency_table():
case_id = item['case_id']
recovered_rows.append('' + anchor('cases/index.html?id=' + case_id, case_id) +
' 解析器崩溃 返回成功；全文分 ' + f"{item['text_score'] * 100:.1f}%" +
- ' ' + anchor('moli-followup/' + case_id + '.txt', '修复后完整输出') + ' ' + anchor('viewer/index.html?path=moli-followup/' + case_id + '.txt', '修复后完整输出') + ' ')
remaining_rows = ['' + anchor('cases/index.html?id=' + item['case_id'], item['case_id']) +
' 原站跳转被隔离环境阻断 输出 ')
+ links.append('输出 ')
metrics = {name: {'baseline': 0.4, 'candidate': 0.5, 'baseline_n': 542,
'candidate_n': 544} for name in ('text_edit', 'code_edit', 'formula_edit', 'table_edit')}
receipt = {'kind': 'webmainbench_moli_full_diagnostic_followup', 'case_count': 545,
From b0e893c1b612826339a4368474b4004c69060369 Mon Sep 17 00:00:00 2001
From: lanyue-llk <270302213+lanyue-llk@users.noreply.github.com>
Date: Thu, 24 Sep 2026 13:00:20 +0800
Subject: [PATCH 06/14] Keep WebMainBench release and cause checks data-driven
---
harness/web_fetch/report.py | 21 ++++++++++++---------
scripts/check_webmainbench_full_report.py | 4 ++--
tests/test_report_changes.py | 5 ++++-
3 files changed, 18 insertions(+), 12 deletions(-)
diff --git a/harness/web_fetch/report.py b/harness/web_fetch/report.py
index ae8e60a11..3280d49fa 100644
--- a/harness/web_fetch/report.py
+++ b/harness/web_fetch/report.py
@@ -2733,6 +2733,17 @@ def webmainbench_full_machine_issue(root: Path, row: dict) -> bool:
return bool(ERROR_PAGE.search(contained_path(root, row["artifact_path"]).read_text(encoding="utf-8")))
+WEBMAINBENCH_FULL_CAUSE_OWNERS = {
+ "Moli 解析器崩溃": "Moli 引擎",
+ "Kitesurf 服务错误": "Kitesurf 服务",
+ "Kitesurf 请求超时": "Kitesurf 服务或传输，内部阶段未定位",
+ "Kitesurf 空正文": "Kitesurf 接口，内部原因未知",
+ "Obscura 空响应": "Obscura",
+ "Obscura 导航循环": "Obscura",
+ "冻结页跳转受阻": "共享输入与隔离环境",
+}
+
+
def generate_webmainbench_full(out: Path, dataset: dict, records: list[dict],
summaries: dict, profile: dict) -> None:
"""Render the complete frozen WebMainBench cohort inside the shared report module."""
@@ -2968,15 +2979,7 @@ def all_call_latency_table():
page("dataset/index.html", "数据集", '冻结网页与人工答案 ' + prose('dataset_lead') + '
搜索任务或网址 方法全部方法 ' +
''.join('' + esc(labels[provider]) + " " for provider in providers) +
diff --git a/scripts/check_webmainbench_full_report.py b/scripts/check_webmainbench_full_report.py
index a33e7abb0..8beed3df4 100644
--- a/scripts/check_webmainbench_full_report.py
+++ b/scripts/check_webmainbench_full_report.py
@@ -10,7 +10,7 @@
from scripts.report_presentation_contract import check_presentation
from harness.web_fetch.safety import contained_path
-from harness.web_fetch.report import ERROR_PAGE
+from harness.web_fetch.report import ERROR_PAGE, WEBMAINBENCH_FULL_CAUSE_OWNERS
def check_result_payload(row: dict, index: dict, artifact_index: dict, shards: dict) -> None:
@@ -173,7 +173,7 @@ def check(root: Path) -> None:
cause_counts = Counter((entry["category_label"], entry["provider"])
for entry in failure_analysis["reviewed_failures"])
cause_cells = guide.select("td[data-cause-provider][data-cause-kind]")
- expected_cause_cells = {(kind, provider) for kind, _ in cause_counts for provider in providers}
+ expected_cause_cells = {(kind, provider) for kind in WEBMAINBENCH_FULL_CAUSE_OWNERS for provider in providers}
if ({(cell["data-cause-kind"], cell["data-cause-provider"]) for cell in cause_cells} != expected_cause_cells
or len(cause_cells) != len(expected_cause_cells)
or any(cell.get_text(strip=True) !=
diff --git a/tests/test_report_changes.py b/tests/test_report_changes.py
index eb674a2eb..6389e75cf 100644
--- a/tests/test_report_changes.py
+++ b/tests/test_report_changes.py
@@ -11,7 +11,10 @@ def test_webmainbench_history_is_localized_from_one_record(self):
from harness.web_fetch.report_changes import ROOT
source = json.loads((ROOT / 'research/report-changes.json').read_text(encoding='utf-8'))
releases = webmainbench_history(source)
- self.assertEqual(len(releases), 17)
+ expected = [row for row in source['releases']
+ if 'webmainbench_frozen_html_to_markdown' in row.get('comparison_kinds', [])]
+ self.assertEqual([row['id'] for row in releases], [row['id'] for row in expected])
+ self.assertEqual(releases[0]['id'], '2026-09-24-webmainbench-moli-repair-545-followup')
for english in (False, True):
page = render_webmainbench_history(
{'releases': releases}, english,
From e42ce4e7d5ed490e992d3e06c08694014b509c46 Mon Sep 17 00:00:00 2001
From: lanyue-llk <270302213+lanyue-llk@users.noreply.github.com>
Date: Thu, 24 Sep 2026 13:06:08 +0800
Subject: [PATCH 07/14] Bind WebMainBench publication separately from frozen
cohort
---
AGENTS.md | 1 +
scripts/check_webmainbench_full_report.py | 8 ++++++++
2 files changed, 9 insertions(+)
diff --git a/AGENTS.md b/AGENTS.md
index 78165bfed..b62e23757 100644
--- a/AGENTS.md
+++ b/AGENTS.md
@@ -21,6 +21,7 @@ These requirements apply to **all** report tracks; browser-specific rules below
- Run repository Python only through `make` targets or `uv run --frozen python`. `.python-version`, `uv.lock`, the Makefile and CI are the runtime contract; never fall back to system Python.
- Run every publishable provider cohort through `scripts/run_certified_benchmark.py`, which freezes references before calls and verifies them again afterward. Direct `run_benchmark.py` output and incomplete cohorts are diagnostic evidence only; never assemble or publish them.
- For the default Web Fetch report, treat `evaluation/current-report.json` as the matrix and publication-target authority. The independent WCXB report uses only `evaluation/wcxb/current-report.json` for its matrix and target. Browser Use and Browser Trajectory Replay have separate contracts under their own `evaluation/` namespaces; never reuse a publication target across reports or tracks. Publishing must use the relevant bound site and expected remote version; do not copy target IDs from prose, old receipts or earlier reports.
+- For the complete WebMainBench report, `evaluation/webmainbench-full/current-report.json` is the immutable pre-call profile bound by the cohort certificate. Its `publication_state` records that frozen pre-call state. The live site, share URL, and expected remote version are bound in `evaluation/webmainbench-full/publication-target.json` together with the profile hash; update this target after publication without changing or re-certifying historical calls.
- For the current WCXB report, the source-qualified 504-case manifest is the entire evaluation population for quality, delivery, and latency. Keep source-invalid cases outside the current manifest, scored evidence, public package, and future provider calls; preserve earlier physical calls only in private historical evidence. The dataset page owns frozen task inputs, source URLs and human reference scope; it must not show measured browser scores. The home page owns aggregate comparisons, the result analysis owns the filterable method-case inventory, and case result pages own full responses. Keep a direct link from each dataset row to a safe frozen-page preview and its human reference. Default result filters must show a nonempty, clearly labeled diagnostic population when one exists, and filtered tables must shrink rather than leave a large blank viewport.
- Persist provider calls outside Git under `~/.browser-eval/` (or `BROWSER_EVAL_DATA_HOME`). Git may contain compact benchmark definitions, annotations, independent source evidence and public release metadata, but never measured outputs, execution manifests, provider artifacts, rerun directories, credentials or internal work logs.
- Web Fetch keeps one current authoritative logical result per method and task. A declared failure retry retains every physical call, retains total retry execution cost separately from the profile-defined primary latency and uses the final response, never the best response. A proven wrong command, version, scorer, reference or timing is a correction: preserve the prior evidence, supersede the complete declared scope and disclose score, denominator and latency effects.
diff --git a/scripts/check_webmainbench_full_report.py b/scripts/check_webmainbench_full_report.py
index 8beed3df4..25f482afa 100644
--- a/scripts/check_webmainbench_full_report.py
+++ b/scripts/check_webmainbench_full_report.py
@@ -65,6 +65,14 @@ def check_moli_followup(root: Path, binding: dict) -> None:
def check(root: Path) -> None:
+ repo = Path(__file__).resolve().parents[1]
+ profile_path = repo / 'evaluation/webmainbench-full/current-report.json'
+ target = json.loads((repo / 'evaluation/webmainbench-full/publication-target.json').read_text(encoding='utf-8'))
+ if (target.get('profile_sha256') != hashlib.sha256(profile_path.read_bytes()).hexdigest()
+ or not str(target.get('expected_remote_version', '')).startswith('ver_')
+ or not str(target.get('share_url', '')).startswith(str(target.get('host', '')).rstrip('/') + '/v/')
+ or not target.get('site_slug')):
+ raise ValueError('WebMainBench publication target is not bound to the certified profile')
followup_binding = json.loads((Path(__file__).resolve().parents[1] /
'evaluation/webmainbench-full/moli-followup-binding.json').read_text(encoding='utf-8'))
check_moli_followup(root, followup_binding)
From 19c84ddc9cbd53b514ded58b91c01398567483d2 Mon Sep 17 00:00:00 2001
From: lanyue-llk <270302213+lanyue-llk@users.noreply.github.com>
Date: Thu, 24 Sep 2026 14:50:36 +0800
Subject: [PATCH 08/14] Publish certified six-method WebMainBench results
---
AGENTS.md | 2 +-
.../webmainbench-full/content-binding.json | 6 +-
.../webmainbench-full/moli-fixed-report.json | 28 ++++
.../webmainbench-full/publication-target.json | 1 +
harness/web_fetch/report.py | 111 +++++++--------
harness/web_fetch/report_build.py | 77 ++++-------
research/report-changes.json | 9 ++
scripts/assemble_webmainbench_six_methods.py | 126 ++++++++++++++++++
scripts/check_webmainbench_full_report.py | 88 ++++++------
scripts/report_presentation_contract.py | 6 +-
scripts/run_certified_webmainbench_full.py | 43 +++++-
tests/test_certified_webmainbench_full.py | 27 +++-
tests/test_report_changes.py | 1 -
tests/test_webmainbench_full_report.py | 61 +++------
14 files changed, 371 insertions(+), 215 deletions(-)
create mode 100644 evaluation/webmainbench-full/moli-fixed-report.json
create mode 100644 scripts/assemble_webmainbench_six_methods.py
diff --git a/AGENTS.md b/AGENTS.md
index b62e23757..641eac0f1 100644
--- a/AGENTS.md
+++ b/AGENTS.md
@@ -21,7 +21,7 @@ These requirements apply to **all** report tracks; browser-specific rules below
- Run repository Python only through `make` targets or `uv run --frozen python`. `.python-version`, `uv.lock`, the Makefile and CI are the runtime contract; never fall back to system Python.
- Run every publishable provider cohort through `scripts/run_certified_benchmark.py`, which freezes references before calls and verifies them again afterward. Direct `run_benchmark.py` output and incomplete cohorts are diagnostic evidence only; never assemble or publish them.
- For the default Web Fetch report, treat `evaluation/current-report.json` as the matrix and publication-target authority. The independent WCXB report uses only `evaluation/wcxb/current-report.json` for its matrix and target. Browser Use and Browser Trajectory Replay have separate contracts under their own `evaluation/` namespaces; never reuse a publication target across reports or tracks. Publishing must use the relevant bound site and expected remote version; do not copy target IDs from prose, old receipts or earlier reports.
-- For the complete WebMainBench report, `evaluation/webmainbench-full/current-report.json` is the immutable pre-call profile bound by the cohort certificate. Its `publication_state` records that frozen pre-call state. The live site, share URL, and expected remote version are bound in `evaluation/webmainbench-full/publication-target.json` together with the profile hash; update this target after publication without changing or re-certifying historical calls.
+- For the complete WebMainBench report, `evaluation/webmainbench-full/current-report.json` is the immutable five-method pre-call profile bound by its cohort certificate. The repaired Moli sixth version has its own pre-call profile and certificate at `evaluation/webmainbench-full/moli-fixed-report.json`; the report joins both complete 545-page cohorts by a hash-bound six-method receipt. Their `publication_state` records frozen pre-call state. The live site, share URL, and expected remote version are bound in `evaluation/webmainbench-full/publication-target.json` together with both profile hashes; update this target after publication without changing or re-certifying historical calls.
- For the current WCXB report, the source-qualified 504-case manifest is the entire evaluation population for quality, delivery, and latency. Keep source-invalid cases outside the current manifest, scored evidence, public package, and future provider calls; preserve earlier physical calls only in private historical evidence. The dataset page owns frozen task inputs, source URLs and human reference scope; it must not show measured browser scores. The home page owns aggregate comparisons, the result analysis owns the filterable method-case inventory, and case result pages own full responses. Keep a direct link from each dataset row to a safe frozen-page preview and its human reference. Default result filters must show a nonempty, clearly labeled diagnostic population when one exists, and filtered tables must shrink rather than leave a large blank viewport.
- Persist provider calls outside Git under `~/.browser-eval/` (or `BROWSER_EVAL_DATA_HOME`). Git may contain compact benchmark definitions, annotations, independent source evidence and public release metadata, but never measured outputs, execution manifests, provider artifacts, rerun directories, credentials or internal work logs.
- Web Fetch keeps one current authoritative logical result per method and task. A declared failure retry retains every physical call, retains total retry execution cost separately from the profile-defined primary latency and uses the final response, never the best response. A proven wrong command, version, scorer, reference or timing is a correction: preserve the prior evidence, supersede the complete declared scope and disclose score, denominator and latency effects.
diff --git a/evaluation/webmainbench-full/content-binding.json b/evaluation/webmainbench-full/content-binding.json
index 14c4c3118..bd431e4a3 100644
--- a/evaluation/webmainbench-full/content-binding.json
+++ b/evaluation/webmainbench-full/content-binding.json
@@ -1,9 +1,5 @@
{
"schema_version": 1,
"relative_path": "report-content.json",
- "sha256": "95d2e1f6f1aa8db094ba61f1766d971c604bd394ee843314f7b5036c17a9dee0",
- "infographic": "issue-matrix.png",
- "infographic_sha256": "879c1af9af28a383a1ef6a0bf26e52655a4aaff28857f9d2dee3296f9d3b4dbf",
- "infographic_mobile": "issue-matrix-mobile.png",
- "infographic_mobile_sha256": "1d85914f7b4853050635962e1c459278c457a3682fa64b8fdd85027fec48a8c0"
+ "sha256": "8671c9830d305c1c02c5648a8ff755624fbd04ab7078cd9644b0a4cbca197a8c"
}
diff --git a/evaluation/webmainbench-full/moli-fixed-report.json b/evaluation/webmainbench-full/moli-fixed-report.json
new file mode 100644
index 000000000..d3a31e445
--- /dev/null
+++ b/evaluation/webmainbench-full/moli-fixed-report.json
@@ -0,0 +1,28 @@
+{
+ "schema_version": 1,
+ "dataset_id": "webmainbench-545-moli-fixed",
+ "comparison_kind": "webmainbench_frozen_html_to_markdown",
+ "case_count": 545,
+ "case_ids_sha256": "5c0cf5810602b70e8f7c1e16723e24a49c3ea207d54865df5c03c38f3d88ffd5",
+ "source_dataset_sha256": "0efaa4b49a45e320a27fe6e5a0b6aad5b57259fc3321ac3448519cacc74c537e",
+ "providers": ["moli"],
+ "versions": {"moli": "1.1.9"},
+ "binary_sha256": "48f45a608ce41d3c6600ca88df964e580aecf353289e028e8d9f7703224ab3b1",
+ "source_commit": "a50182cbe4eef20b230d3f01e452d9de4f3146ba",
+ "timeout_seconds": 45,
+ "runs_per_case": 1,
+ "concurrency": 1,
+ "max_failure_retries": 1,
+ "reference_binding_mode": "complete_webmainbench_545_official_frozen_html_and_groundtruth_content",
+ "official_scorer_commit": "9d991bdc00c57b57521499494d96be85c31317ba",
+ "metric_config": {"use_llm": false},
+ "official_metrics": ["overall", "text_edit", "code_edit", "formula_edit", "table_edit", "table_TEDS"],
+ "input_contract": {
+ "body": "official html field encoded as UTF-8 without mutation",
+ "execution": "inline scripts may run; non-loopback subresources and data access are blocked; local engine receives the same body from an OS-isolated loopback HTTPS fixture with a per-run CA",
+ "original_url": "retained as evidence metadata only"
+ },
+ "publication_state": "pending",
+ "require_independent_result_review": false,
+ "conversion": "Native Markdown from Moli fixed build"
+}
diff --git a/evaluation/webmainbench-full/publication-target.json b/evaluation/webmainbench-full/publication-target.json
index a945003aa..333669567 100644
--- a/evaluation/webmainbench-full/publication-target.json
+++ b/evaluation/webmainbench-full/publication-target.json
@@ -1,6 +1,7 @@
{
"schema_version": 1,
"profile_sha256": "8cbceaf9694e88e7c1ddbba3263573dcccf69b1ac89d7e9ea5aa098f2a3bab02",
+ "moli_fixed_profile_sha256": "db8e7fa408edf3dc2e19d2b0f35645448d0e1efd3ac86926d0bcc9e1a3fb3848",
"host": "https://artifact-site.lexmount.cn",
"site_slug": "zrqPjO672dgg",
"share_url": "https://artifact-site.lexmount.cn/v/Avz3qzo6EEujI0iln1LAzCUF93I47OEC",
diff --git a/harness/web_fetch/report.py b/harness/web_fetch/report.py
index 3280d49fa..7b30ce4fe 100644
--- a/harness/web_fetch/report.py
+++ b/harness/web_fetch/report.py
@@ -2748,7 +2748,6 @@ def generate_webmainbench_full(out: Path, dataset: dict, records: list[dict],
summaries: dict, profile: dict) -> None:
"""Render the complete frozen WebMainBench cohort inside the shared report module."""
content = json.loads((out / "report-content.json").read_text(encoding="utf-8"))
- followup = json.loads((out / "moli-followup.json").read_text(encoding="utf-8")) if (out / "moli-followup.json").is_file() else None
failure_analysis = json.loads((out / "failure-analysis.json").read_text(encoding="utf-8"))
reviewed_failures = {(row["provider"], row["case_id"]): row for row in failure_analysis["reviewed_failures"]}
machine_issue_count = sum(webmainbench_full_machine_issue(out, row) for row in records)
@@ -2763,7 +2762,7 @@ def prose(key):
grouped = defaultdict(dict)
for row in records:
grouped[row["case_id"]][row["provider"]] = row
- labels = {p: (profile["versions"][p] if p == "chromium" else p.capitalize() +
+ labels = {p: ("Moli 1.1.9（修复后）" if p == "moli-fixed" else profile["versions"][p] if p == "chromium" else p.capitalize() +
(" " + profile["versions"][p] if p != "kitesurf" else "")) for p in providers}
measures = [("overall", "综合分"), ("text_edit", "全文"), ("code_edit", "代码"),
("formula_edit", "公式"), ("table_edit", "表格内容"), ("table_TEDS", "表格结构")]
@@ -2788,11 +2787,11 @@ def classify(row):
categories = {row["attempt_id"]: classify(row) for row in records}
counts = Counter(categories.values())
css = """*{box-sizing:border-box}body{margin:0;background:#f4f7fb;color:#172637;font:16px/1.55 system-ui,sans-serif}body.dark{background:#101923;color:#edf3fa}a{color:#1859a5}body.dark a{color:#92c7ff}.wrap{width:min(1200px,calc(100% - 32px));margin:auto}header{background:#142d4b;color:white}header .wrap{display:flex;align-items:center;gap:18px;flex-wrap:wrap;padding:15px 0}header a{color:white;text-decoration:none}header strong{margin-right:auto}nav{display:flex;gap:14px;flex-wrap:wrap}button,input,select{font:inherit;border:1px solid #9aacc0;border-radius:7px;padding:7px 10px;background:white;color:#172637}body.dark button,body.dark input,body.dark select{background:#25384b;color:white;border-color:#61768b}h1{font-size:clamp(30px,4vw,48px);line-height:1.18;margin:0 0 15px}h2{font-size:26px;margin:0 0 14px}.hero{padding:48px 0 22px}.hero p{font-size:18px;max-width:900px;color:#506377}body.dark .hero p,body.dark .note{color:#bfd0e1}section{margin:24px 0 38px}figure{margin:24px 0}.panel{background:white;border:1px solid #d8e1eb;border-radius:12px;padding:20px;margin:14px 0}body.dark .panel{background:#1c2b3b;border-color:#38506a}.scroll{overflow:auto;max-height:66vh;border:1px solid #d8e1eb;border-radius:10px;background:white}body.dark .scroll{background:#1c2b3b;border-color:#38506a}table{border-collapse:collapse;width:100%;min-width:820px}th,td{padding:9px 12px;text-align:left;border-bottom:1px solid #dce5ee;vertical-align:top}th{position:sticky;top:0;background:#e9f0f8;color:#152a40;z-index:1}body.dark th{background:#29435d;color:white}tbody tr:hover{background:#edf4fc}body.dark tbody tr:hover{background:#28445f}td small{display:block;color:#597088}body.dark td small{color:#b8cadb}.number{font-variant-numeric:tabular-nums;white-space:nowrap}.winner{color:#116b3c;font-weight:750}body.dark .winner{color:#82e1a7}.filters{display:flex;gap:12px;flex-wrap:wrap;margin:14px 0}.filters label{display:grid;gap:4px;font-size:13px;font-weight:700}.note{color:#52677b;font-size:14px}pre{white-space:pre-wrap;overflow-wrap:anywhere;background:#0e2032;color:#f1f6fb;padding:17px;border-radius:8px;max-height:65vh;overflow:auto;font:13px/1.5 ui-monospace,monospace}.compare-grid{display:grid;grid-template-columns:repeat(2,minmax(0,1fr));gap:16px}.compare-grid .panel{min-width:0;margin:0}.compare-grid pre{max-height:70vh}details{border:1px solid #dce5ee;border-radius:8px;padding:10px 14px;margin:10px 0}body.dark details{border-color:#38506a}summary{cursor:pointer;font-weight:700}.actions{display:flex;gap:14px;flex-wrap:wrap}.source{overflow-wrap:anywhere}footer{padding:32px 0;color:#52677b}@media(max-width:800px){.compare-grid{grid-template-columns:1fr}}@media(max-width:650px){.panel{padding:14px}header nav{width:100%}th,td{padding:7px 9px}}"""
- js = """function theme(){document.body.classList.toggle('dark');try{localStorage.setItem('wmb545-theme',document.body.classList.contains('dark')?'dark':'light')}catch(e){}}try{if(localStorage.getItem('wmb545-theme')==='dark')document.body.classList.add('dark')}catch(e){}function filter(){let q=(document.getElementById('q')?.value||'').toLowerCase(),p=document.getElementById('provider')?.value||'',k=document.getElementById('kind')?.value||'issue',cause=document.getElementById('cause')?.value||'',n=0;document.querySelectorAll('tr[data-result]').forEach(r=>{let show=(!q||r.dataset.search.includes(q))&&(!p||r.dataset.provider===p)&&(!cause||r.dataset.cause===cause)&&(k==='all'||(k==='issue'?r.dataset.kind!=='未触发诊断':r.dataset.kind===k));r.hidden=!show;if(show)n++});let c=document.getElementById('count');if(c)c.textContent=n;let e=document.getElementById('empty');if(e)e.hidden=n!==0}document.addEventListener('DOMContentLoaded',()=>{for(let id of ['q','provider','kind','cause'])document.getElementById(id)?.addEventListener('input',filter);let params=new URLSearchParams(location.search);for(let id of ['provider','kind','cause']){let select=document.getElementById(id);if(select&¶ms.has(id))select.value=params.get(id)}filter();let node=document.getElementById('evidence-content');if(node){let path=params.get('path')||'';if(!/^(artifacts|references|sources|moli-followup)\\/[A-Za-z0-9_.\\/-]+$/.test(path)||path.includes('..')){node.textContent='无效证据路径';return}document.getElementById('evidence-path').textContent=path;document.getElementById('evidence-raw').href=path;loadEvidence(path,node)}});"""
+ js = """function theme(){document.body.classList.toggle('dark');try{localStorage.setItem('wmb545-theme',document.body.classList.contains('dark')?'dark':'light')}catch(e){}}try{if(localStorage.getItem('wmb545-theme')==='dark')document.body.classList.add('dark')}catch(e){}function filter(){let q=(document.getElementById('q')?.value||'').toLowerCase(),p=document.getElementById('provider')?.value||'',k=document.getElementById('kind')?.value||'issue',cause=document.getElementById('cause')?.value||'',n=0;document.querySelectorAll('tr[data-result]').forEach(r=>{let show=(!q||r.dataset.search.includes(q))&&(!p||r.dataset.provider===p)&&(!cause||r.dataset.cause===cause)&&(k==='all'||(k==='issue'?r.dataset.kind!=='未触发诊断':r.dataset.kind===k));r.hidden=!show;if(show)n++});let c=document.getElementById('count');if(c)c.textContent=n;let e=document.getElementById('empty');if(e)e.hidden=n!==0}document.addEventListener('DOMContentLoaded',()=>{for(let id of ['q','provider','kind','cause'])document.getElementById(id)?.addEventListener('input',filter);let params=new URLSearchParams(location.search);for(let id of ['provider','kind','cause']){let select=document.getElementById(id);if(select&¶ms.has(id))select.value=params.get(id)}filter();let node=document.getElementById('evidence-content');if(node){let path=params.get('path')||'';if(!/^(artifacts|references|sources)\\/[A-Za-z0-9_.\\/-]+$/.test(path)||path.includes('..')){node.textContent='无效证据路径';return}document.getElementById('evidence-path').textContent=path;document.getElementById('evidence-raw').href=path;loadEvidence(path,node)}});"""
js += """
async function loadJson(path){let response=await fetch(path);if(!response.ok)throw Error('HTTP '+response.status);return response.json()}
-async function loadEvidence(path,node){try{let value;if(path.startsWith('moli-followup/')){let response=await fetch(path);if(!response.ok)throw Error('HTTP '+response.status);value=await response.text()}else{let index=await loadJson('artifact-shard-index.json'),shardPath=index[path];if(!shardPath)throw Error('证据索引缺失');let shard=await loadJson(shardPath),hash=shard.artifacts[path];value=shard.blobs[hash]}if(typeof value!=='string')throw Error('证据内容缺失');node.textContent=value;let raw=document.getElementById('evidence-raw');raw.href=URL.createObjectURL(new Blob([value],{type:'text/plain;charset=utf-8'}));raw.download=path.split('/').pop()}catch(error){node.textContent='无法读取证据：'+error.message}}
-async function renderResult(){let params=new URLSearchParams(location.search),provider=params.get('provider')||'',id=params.get('id')||'';let title=document.getElementById('result-title');if(!/^[a-z0-9-]+$/.test(provider)||!/^[A-Za-z0-9-]+$/.test(id)){title.textContent='无效结果地址';return}try{let index=await loadJson('result-shard-index.json'),key=provider+'|'+id,shardPath=index[key];if(!shardPath)throw Error('结果索引缺失');let shard=await loadJson(shardPath),row=shard.results[key];if(!row)throw Error('结果内容缺失');let names={moli:'Moli',lightpanda:'Lightpanda',obscura:'Obscura',chromium:'Chrome',kitesurf:'Kitesurf'},name=names[provider]||provider;title.textContent=name+' · '+id;document.title=title.textContent+' · WebMainBench 545';document.getElementById('result-source').textContent=row.original_url;let failure=['请求失败','空输出','错误页'].includes(row.diagnosis),detail=document.querySelector('[data-report-detail]');detail.dataset.reportDetail=failure?'failed':'success';document.getElementById('result-summary').textContent='状态：'+(row.ok?'返回成功':'请求失败')+'；诊断：'+row.diagnosis+'；最终调用 '+row.elapsed_ms.toFixed(0)+' ms；物理调用 '+row.physical_attempt_count+' 次，含重试总耗时 '+row.total_elapsed_ms.toFixed(0)+' ms。';let section=document.getElementById('result-failure');section.hidden=!failure;if(failure){document.getElementById('result-stage').textContent='阶段：'+(row.error_type||row.diagnosis);document.getElementById('result-error').textContent='程序错误：'+(row.error_message||'无');document.getElementById('result-log').textContent=row.provider_metadata?.stderr||row.provider_metadata?.browser_stderr||'无日志';document.getElementById('result-cause').textContent='复核原因：'+(row.review?.cause||'未记录');document.getElementById('result-treatment').textContent='处理：'+(row.review?.treatment||'未记录').replace(/[。；]+$/,'')+'；状态：'+(row.review?.status||'未记录')}let labels=[['text_edit','全文'],['code_edit','代码'],['formula_edit','公式'],['table_edit','表格内容'],['table_TEDS','表格结构']],body=document.getElementById('result-metrics');for(let [metric,label] of labels){let tr=document.createElement('tr'),entry=row.metrics?.[metric]||{};for(let value of [label,(!entry.success||entry.score==null)?'—':(entry.score*100).toFixed(1)+'%',entry.success?'参与计分':'不参与计分']){let td=document.createElement('td');td.textContent=value;tr.append(td)}body.append(tr)}document.getElementById('result-case').href='cases/index.html?id='+id;document.getElementById('result-output').href='viewer/index.html?path='+encodeURIComponent(row.artifact_path);document.getElementById('result-reference').href='viewer/index.html?path=references/'+id+'.md';document.getElementById('result-source-link').href='viewer/index.html?path=sources/'+id+'.html.txt';let physical=shard.physical[id]||[],list=document.getElementById('result-physical-list');for(let [number,attempt] of physical.entries()){let li=document.createElement('li'),a=document.createElement('a');a.href='viewer/index.html?path='+encodeURIComponent(attempt.physical_artifact_path);a.textContent='第 '+(number+1)+' 次调用 · '+attempt.elapsed_ms.toFixed(0)+' ms';li.append(a);list.append(li)}document.getElementById('result-physical').hidden=!physical.length}catch(error){title.textContent='无法载入结果：'+error.message}}
+async function loadEvidence(path,node){try{let index=await loadJson('artifact-shard-index.json'),shardPath=index[path];if(!shardPath)throw Error('证据索引缺失');let shard=await loadJson(shardPath),hash=shard.artifacts[path],value=shard.blobs[hash];if(typeof value!=='string')throw Error('证据内容缺失');node.textContent=value;let raw=document.getElementById('evidence-raw');raw.href=URL.createObjectURL(new Blob([value],{type:'text/plain;charset=utf-8'}));raw.download=path.split('/').pop()}catch(error){node.textContent='无法读取证据：'+error.message}}
+async function renderResult(){let params=new URLSearchParams(location.search),provider=params.get('provider')||'',id=params.get('id')||'';let title=document.getElementById('result-title');if(!/^[a-z0-9-]+$/.test(provider)||!/^[A-Za-z0-9-]+$/.test(id)){title.textContent='无效结果地址';return}try{let index=await loadJson('result-shard-index.json'),key=provider+'|'+id,shardPath=index[key];if(!shardPath)throw Error('结果索引缺失');let shard=await loadJson(shardPath),row=shard.results[key];if(!row)throw Error('结果内容缺失');let names={moli:'Moli 1.1.9', 'moli-fixed':'Moli 1.1.9（修复后）',lightpanda:'Lightpanda',obscura:'Obscura',chromium:'Chrome',kitesurf:'Kitesurf'},name=names[provider]||provider;title.textContent=name+' · '+id;document.title=title.textContent+' · WebMainBench 545';document.getElementById('result-source').textContent=row.original_url;let failure=['请求失败','空输出','错误页'].includes(row.diagnosis),detail=document.querySelector('[data-report-detail]');detail.dataset.reportDetail=failure?'failed':'success';document.getElementById('result-summary').textContent='状态：'+(row.ok?'返回成功':'请求失败')+'；诊断：'+row.diagnosis+'；最终调用 '+row.elapsed_ms.toFixed(0)+' ms；物理调用 '+row.physical_attempt_count+' 次，含重试总耗时 '+row.total_elapsed_ms.toFixed(0)+' ms。';let section=document.getElementById('result-failure');section.hidden=!failure;if(failure){document.getElementById('result-stage').textContent='阶段：'+(row.error_type||row.diagnosis);document.getElementById('result-error').textContent='程序错误：'+(row.error_message||'无');document.getElementById('result-log').textContent=row.provider_metadata?.stderr||row.provider_metadata?.browser_stderr||'无日志';document.getElementById('result-cause').textContent='复核原因：'+(row.review?.cause||'未记录');document.getElementById('result-treatment').textContent='处理：'+(row.review?.treatment||'未记录').replace(/[。；]+$/,'')+'；状态：'+(row.review?.status||'未记录')}let labels=[['text_edit','全文'],['code_edit','代码'],['formula_edit','公式'],['table_edit','表格内容'],['table_TEDS','表格结构']],body=document.getElementById('result-metrics');for(let [metric,label] of labels){let tr=document.createElement('tr'),entry=row.metrics?.[metric]||{};for(let value of [label,(!entry.success||entry.score==null)?'—':(entry.score*100).toFixed(1)+'%',entry.success?'参与计分':'不参与计分']){let td=document.createElement('td');td.textContent=value;tr.append(td)}body.append(tr)}document.getElementById('result-case').href='cases/index.html?id='+id;document.getElementById('result-output').href='viewer/index.html?path='+encodeURIComponent(row.artifact_path);document.getElementById('result-reference').href='viewer/index.html?path=references/'+id+'.md';document.getElementById('result-source-link').href='viewer/index.html?path=sources/'+id+'.html.txt';let physical=shard.physical[id]||[],list=document.getElementById('result-physical-list');for(let [number,attempt] of physical.entries()){let li=document.createElement('li'),a=document.createElement('a');a.href='viewer/index.html?path='+encodeURIComponent(attempt.physical_artifact_path);a.textContent='第 '+(number+1)+' 次调用 · '+attempt.elapsed_ms.toFixed(0)+' ms';li.append(a);list.append(li)}document.getElementById('result-physical').hidden=!physical.length}catch(error){title.textContent='无法载入结果：'+error.message}}
async function renderCase(){
let id=new URLSearchParams(location.search).get('id')||'',slot=document.getElementById('case-id');
if(!/^[A-Za-z0-9-]+$/.test(id)){slot.textContent='无效任务地址';return}
@@ -2804,7 +2803,7 @@ def classify(row):
document.getElementById('case-source').href='viewer/index.html?path=sources/'+id+'.html.txt';
document.getElementById('case-reference').href='viewer/index.html?path=references/'+id+'.md';
let resultIndex=await loadJson('result-shard-index.json'),artifactIndex=await loadJson('artifact-shard-index.json');
- let names={moli:'Moli',lightpanda:'Lightpanda',obscura:'Obscura',chromium:'Chrome',kitesurf:'Kitesurf'};
+ let names={moli:'Moli 1.1.9','moli-fixed':'Moli 1.1.9（修复后）',lightpanda:'Lightpanda',obscura:'Obscura',chromium:'Chrome',kitesurf:'Kitesurf'};
let metrics=['overall','text_edit','code_edit','formula_edit','table_edit','table_TEDS'];
let providers=dataset.coverage_plan.static_html,referencePath='references/'+id+'.md';
let paths=[...new Set([...providers.map(p=>resultIndex[p+'|'+id]),artifactIndex[referencePath]])];
@@ -2839,6 +2838,11 @@ def classify(row):
}
document.addEventListener('DOMContentLoaded',()=>{if(document.getElementById('result-title'))renderResult();if(document.getElementById('case-id'))renderCase()});
"""
+ js += """
+async function openDiagnosis(button){let tr=button.closest('tr[data-result]'),dialog=document.getElementById('diagnosis-dialog'),provider=tr?.dataset.provider,id=tr?.dataset.caseId;if(!dialog||!provider||!id)return;let put=(name,value)=>{document.getElementById('diagnosis-'+name).textContent=value};put('title','正在读取诊断…');put('error','');put('log','');put('cause','');put('treatment','');dialog.showModal();try{let index=await loadJson('result-shard-index.json'),key=provider+'|'+id,shard=await loadJson(index[key]),row=shard.results[key];if(!row)throw Error('结果证据缺失');put('title',id+' · '+row.diagnosis);put('stage','失败阶段：'+(row.error_type||row.diagnosis));put('error','程序错误：'+(row.error_message||'无'));put('log',row.provider_metadata?.stderr||row.provider_metadata?.browser_stderr||row.error_message||'无日志');put('cause','复核原因：'+(row.review?.cause||'未独立复核'));put('treatment','处理：'+(row.review?.treatment||'未记录')+'；状态：'+(row.review?.status||'未记录'));document.getElementById('diagnosis-result').href='results/index.html?provider='+provider+'&id='+id;document.getElementById('diagnosis-output').href='viewer/index.html?path='+encodeURIComponent(row.artifact_path);document.getElementById('diagnosis-case').href='cases/index.html?id='+id}catch(error){put('title','诊断读取失败');put('error',error.message)}}
+document.addEventListener('DOMContentLoaded',()=>{let dialog=document.getElementById('diagnosis-dialog');if(!dialog)return;document.getElementById('diagnosis-close').addEventListener('click',()=>dialog.close());document.addEventListener('click',event=>{let button=event.target.closest('button[data-diagnose]');if(button)openDiagnosis(button)})});
+"""
+ css += "code,#result-error{overflow-wrap:anywhere;word-break:break-word}dialog{width:min(760px,calc(100% - 24px));max-height:85vh;overflow:auto;border:1px solid #9aacc0;border-radius:12px;background:white;color:#172637;padding:22px}dialog::backdrop{background:#0b172bc9}body.dark dialog{background:#1c2b3b;color:#edf3fa;border-color:#61768b}dialog .actions{margin-top:16px}dialog pre{max-height:28vh}td button[data-diagnose]{font-size:14px;padding:2px 6px}"
(out / "assets").mkdir(exist_ok=True)
(out / "assets/style.css").write_text(css, encoding="utf-8")
(out / "assets/report.js").write_text(js, encoding="utf-8")
@@ -2849,7 +2853,7 @@ def anchor(path, label):
def table(headers, rows):
return ''
- navigation = [("总览", "index.html"), ("Moli 修复复测", "moli-update/index.html")] if followup else [("总览", "index.html")]
+ navigation = [("总览", "index.html")]
navigation += [("数据集", "dataset/index.html"),
("结果分析", "analysis/index.html"), ("解读", "guide/index.html"),
("方法", "methodology/index.html"), ("复现", "reproduce/index.html"),
@@ -2862,7 +2866,7 @@ def page(route, title, body):
(prefix or "./") + '">' + esc(title) + ' · WebMainBench 545 ' +
anchor("index.html", "WebMainBench · 545") + ' ' + links + ' ' + anchor("en/index.html", "English") +
'切换主题
' + body +
- ' WebMainBench 545 例 · 5 种方法 · ' +
+ 'WebMainBench 545 例 · ' + str(len(providers)) + ' 种方法 · ' +
anchor("https://github.com/opendatalab/WebMainBench", "公开数据与评分器") + ' 