From 49d25138541f1a3012034aad20863a7f2f82ebe9 Mon Sep 17 00:00:00 2001 From: llk Date: Thu, 8 Oct 2026 21:08:27 +0800 Subject: [PATCH 001/172] Use native Web Fetch interfaces and frozen benchmark inputs Rebase the session changes onto main 03c9a5e9 and extend its shared certification, execution, evidence, assembly and reporting owners. Keep command entry points thin, package the participant registry, remove retired runners and historical qualification recovery, and enforce task-selected native outputs without external answer repair. Preserve measured evidence and require complete declared correction scopes. Add functional controls for native documents, media URL parsing, interrupted evidence recovery, fixed versus explicitly live references, scoring regressions, report bindings and release presentation. --- .agents/skills/benchmark-report/SKILL.md | 4 +- .../references/audit-and-revision.md | 3 +- .../references/evaluation-design.md | 4 + .../references/report-and-release.md | 8 +- .dockerignore | 6 + .env.example | 4 + AGENTS.md | 2 +- configs/providers.headless.json | 9 + .../browser-native-430/current-report.json | 515 +++++++++++ evaluation/risk-control/README.md | 11 +- evaluation/scoring-regressions.json | 159 +++- examples/web_fetch/docker/resource.Dockerfile | 52 +- harness/reporting/links.py | 6 +- harness/web_fetch/HEADLESS.md | 120 ++- harness/web_fetch/README.md | 76 +- harness/web_fetch/assembly.py | 11 +- harness/web_fetch/certification.py | 75 +- harness/web_fetch/evidence.py | 108 ++- harness/web_fetch/methods.py | 38 +- harness/web_fetch/metrics.py | 9 +- harness/web_fetch/models.py | 2 +- harness/web_fetch/providers/base.py | 77 +- harness/web_fetch/providers/cloudflare.py | 13 +- harness/web_fetch/providers/headless.py | 324 ++++++- .../web_fetch/providers/headless_worker.py | 245 ++++- harness/web_fetch/providers/moli.py | 853 +++--------------- harness/web_fetch/publication_privacy.py | 78 +- harness/web_fetch/reference_freeze.py | 24 +- harness/web_fetch/reference_lifecycle.py | 51 +- harness/web_fetch/reference_quality.py | 51 +- harness/web_fetch/report.py | 118 ++- harness/web_fetch/report_build.py | 64 +- harness/web_fetch/report_capabilities.py | 50 +- harness/web_fetch/report_changes.py | 16 +- harness/web_fetch/report_check.py | 64 +- harness/web_fetch/report_components.py | 17 +- harness/web_fetch/report_diagnostic.py | 4 +- harness/web_fetch/report_explainer.py | 331 ++++++- harness/web_fetch/report_i18n.py | 23 +- harness/web_fetch/report_interfaces.py | 186 +++- harness/web_fetch/report_methods.py | 22 +- harness/web_fetch/report_resources.py | 2 +- harness/web_fetch/report_style.py | 1 + harness/web_fetch/retrieval_hints.py | 26 +- harness/web_fetch/risk_control.py | 214 ++--- harness/web_fetch/scoring.py | 11 +- harness/web_fetch/scoring_regressions.py | 18 + harness/web_fetch/scoring_revision.py | 37 + harness/web_fetch/stable_tasks.py | 32 +- harness/web_fetch/storage.py | 55 ++ harness/web_fetch/task_outputs.py | 230 +++++ pyproject.toml | 2 +- ...-111-three-cohort-evaluation-revision.json | 398 +++++++- research/provider-pricing.json | 4 +- research/report-changes.json | 231 ++++- scripts/build_report.py | 2 +- tests/package_smoke.py | 2 +- tests/report_check_fixture.py | 10 +- tests/test_apt_uri_download.py | 24 + tests/test_assembly_evidence.py | 2 +- tests/test_dynamic_100_moli_119.py | 8 + tests/test_dynamic_provider_waits.py | 17 + tests/test_fixed_native_policy.py | 23 + tests/test_freeze_headless_references.py | 33 +- tests/test_headless.py | 177 +++- tests/test_headless_access.py | 37 +- tests/test_headless_cancellation.py | 199 ++++ tests/test_headless_completion.py | 65 ++ tests/test_headless_files.py | 290 ++++++ tests/test_headless_projection.py | 3 +- tests/test_headless_report.py | 244 ++++- tests/test_interrupted_evidence.py | 143 +++ tests/test_jsonl_evidence.py | 18 + tests/test_moli_provider.py | 548 +++-------- tests/test_native_browser_variants.py | 97 ++ tests/test_native_network_dns.py | 6 +- tests/test_native_task_outputs.py | 281 ++++++ tests/test_provider_evidence_boundary.py | 2 +- tests/test_providers.py | 34 + tests/test_publication_privacy.py | 31 + tests/test_real_smoke.py | 2 +- tests/test_reference_inputs.py | 6 +- tests/test_reference_lifecycle.py | 31 +- tests/test_reference_quality.py | 32 + tests/test_report_body_scope.py | 48 +- tests/test_report_capabilities.py | 20 + tests/test_report_changes.py | 17 + tests/test_report_inventory.py | 2 +- tests/test_report_links.py | 12 + tests/test_report_presentation.py | 47 +- tests/test_report_presentation_contract.py | 14 + tests/test_report_ui.mjs | 18 + tests/test_resource_build_context.py | 31 + tests/test_retrieval_hints.py | 49 +- tests/test_risk_control_dataset.py | 72 +- tests/test_risk_control_report.py | 2 +- tests/test_run_certified_benchmark.py | 100 +- tests/test_scoring.py | 27 +- tests/test_scoring_regressions.py | 61 +- tests/test_storage.py | 100 ++ 100 files changed, 6229 insertions(+), 1852 deletions(-) create mode 100644 evaluation/browser-native-430/current-report.json create mode 100644 harness/web_fetch/task_outputs.py create mode 100644 tests/test_headless_cancellation.py create mode 100644 tests/test_headless_completion.py create mode 100644 tests/test_headless_files.py create mode 100644 tests/test_interrupted_evidence.py create mode 100644 tests/test_jsonl_evidence.py create mode 100644 tests/test_native_browser_variants.py create mode 100644 tests/test_native_task_outputs.py create mode 100644 tests/test_resource_build_context.py diff --git a/.agents/skills/benchmark-report/SKILL.md b/.agents/skills/benchmark-report/SKILL.md index c9861c70a..6321ffbb9 100644 --- a/.agents/skills/benchmark-report/SKILL.md +++ b/.agents/skills/benchmark-report/SKILL.md @@ -5,7 +5,7 @@ description: Design, run, audit, revise, and release evidence-backed benchmark r # Benchmark report -Produce a report whose conclusions can be traced to frozen inputs, measured outputs, scoring rules, and an explicit result review. Optimize for a reader deciding what the benchmark means, not for preserving the author's workflow. +Produce a report whose conclusions can be traced to frozen inputs, measured outputs, scoring rules, and the declared result-review coverage. Optimize for a reader deciding what the benchmark means, not for preserving the author's workflow. Read the repository's `AGENTS.md` first. Project rules override this general skill. Before changing a benchmark, read [evaluation design](references/evaluation-design.md). Before reviewing scores or deciding whether to rerun, read [audit and revision](references/audit-and-revision.md). Before building or publishing a report, read [report and release](references/report-and-release.md). @@ -18,7 +18,7 @@ An explicitly requested diagnostic report may be delivered separately from a cer Public report CLIs default to English where supported and expose `--language en|zh`. Preserve the selected language on rebuild. English builds retain the matching Chinese edition; keep measured material in its source language. Historical Chinese-only renderers must disclose that limit and reject unsupported requests. Bind generated languages and the selected entry in report metadata. English navigation opens a not-generated notice when absent; never publish stale English pages. Validate each generated edition and retain route, value and evidence-link parity checks when both exist. 1. Establish the decision being supported, the measured unit, cohort, versions, execution location, freshness policy, and intended deliverable. Inspect the actual dataset, runner, outputs, scorer, prior review evidence, and current report. Do not infer them from screenshots or prose. -2. Freeze a machine-readable evaluation contract before expensive calls. It must identify the exact cases and methods, run count, timeout, concurrency policy, input and output contract, scoring provenance, reference state, and publication target when one exists. For source-dependent tasks, fail closed: verify and freeze references immediately before calls, verify the same declared scope afterward, and certify the cohort only when both checks pass. Partial or uncertified cohorts remain diagnostic evidence and cannot enter a release. +2. Freeze a machine-readable evaluation contract before expensive calls. It must identify the exact cases and methods, run count, timeout, concurrency policy, input and output contract, scoring provenance, reference state, and publication target when one exists. Existing compiled reference answers are frozen by default. Source metadata, capture dates, and absent newer acceptance-contract fields do not imply a live-reference task. Only tasks explicitly declaring `source_verification.reference_mode: live_exact` require live checks. For those tasks, fail closed: verify and freeze references immediately before calls, verify the same declared scope afterward, and certify the cohort only when both checks pass. An unchanged compiled acceptance-contract dataset is already the benchmark input: freeze its task and rule bytes for execution, without requiring historical source-qualification raw files or re-auditing cohort admission. Qualify sources when adding tasks or changing references/acceptance rules; do not turn an ordinary version rerun into dataset reconstruction. Partial or uncertified cohorts remain diagnostic evidence and cannot enter a release. 3. Let each method use its strongest documented route to produce the required output. Equalize the task, inputs, output contract, call budget, and scoring. Do not require identical internals. Record material asymmetries such as local versus remote execution, cold versus warm start, bundled conversion, network transit, account tier, and billable time. 4. Preserve evidence as an append-only chain: task definition, reference, provider call, raw artifact, normalized artifact, score, review, revision, and release. Keep provider output and call identity immutable. A presentation or scoring change must not masquerade as a new provider result. 5. Separate retrieval success, task acceptance, content quality, latency, cost, and freshness. A missing or partial reference makes the affected metric unavailable or scoped; it does not prove success or failure. Do not derive binary acceptance from an unrelated aggregate similarity score. diff --git a/.agents/skills/benchmark-report/references/audit-and-revision.md b/.agents/skills/benchmark-report/references/audit-and-revision.md index 5ad926e2e..37f5a9b3e 100644 --- a/.agents/skills/benchmark-report/references/audit-and-revision.md +++ b/.agents/skills/benchmark-report/references/audit-and-revision.md @@ -18,7 +18,7 @@ release manifest + deployed bytes Hash and bind transitions where accidental mixing is plausible. Validate exact cardinality and uniqueness, not only totals. Treat raw outputs, timestamps, request status, method version, and call IDs as immutable measured evidence. -Source-dependent calls require a fail-closed boundary: capture and freeze the declared reference scope immediately before execution, repeat the independent observation after execution, and bind both checks to the complete cohort. A process exit, partial attempt count, changed reference, or missing post-check makes the cohort diagnostic rather than publishable; never salvage its successful rows into the release matrix. +Live exact-reference calls require a fail-closed boundary: capture and freeze the declared reference scope immediately before execution, repeat the independent observation after execution, and bind both checks to the complete cohort. A process exit, partial attempt count, changed reference, or missing post-check makes the cohort diagnostic rather than publishable; never salvage its successful rows into the release matrix. ## Independent result review @@ -78,3 +78,4 @@ private evidence. Never rescue an empty declared body from a nonempty display message. Reproject retained full responses without new product calls when only this selection changed; missing historical envelopes require fresh calls across the full declared correction scope before claiming revised benchmark scores. +Reruns of unchanged compiled acceptance-contract tasks freeze the existing executable dataset. They do not require recovering historical qualification raw files or repeating admission review. Source qualification is part of task authoring and reference/rule correction, not a prerequisite for every provider-version comparison. Preserve historical receipts without applying their evidence-file requirements to a new execution protocol. diff --git a/.agents/skills/benchmark-report/references/evaluation-design.md b/.agents/skills/benchmark-report/references/evaluation-design.md index ec68c72ce..ea412c7ea 100644 --- a/.agents/skills/benchmark-report/references/evaluation-design.md +++ b/.agents/skills/benchmark-report/references/evaluation-design.md @@ -43,3 +43,7 @@ Ground truth must be independent of the provider under test. Retain source locat ## Machine-readable current profile Keep one current profile that a fresh session can validate before running, scoring, rebuilding, or publishing. Include exact case IDs or their canonical digest, the ordered method/version set, run count, timeout, concurrency policy, output contract, reference state, review requirement, and release target. Fail closed on drift instead of reconstructing intent from old reports. + +For native-content browser comparisons, use task-appropriate official interfaces and score the returned deliverable. A final URL or HTML language tag is diagnostic unless the benchmark explicitly evaluates that metadata. Do not repair missing text using JSON DOM conversion or external PDF/DOCX parsers. The declared Chromium HTML-to-Markdown exception does not authorize document parsers. Bind native script versions, generate the interface matrix from the runner route selector, and require route validation at aggregation and publication. Keep negative controls for wrong content alongside positive controls for correct content with missing metadata. + +Keep browser engines distinct from execution configurations. Released and source-build variants use separate method identities in the execution registry and share the engine's native route constraints. Derive interface columns and participant counts from that registry. A strict native-mode marker selects default completion; it supplies no per-case selector or wait and must survive both dataset conversion and runner validation. Resource projection must preserve single URL values and tokenize only srcset candidates, including literal commas inside URLs. Protect these boundaries with runner, report and actual-browser controls. diff --git a/.agents/skills/benchmark-report/references/report-and-release.md b/.agents/skills/benchmark-report/references/report-and-release.md index 0501b5a2a..3caed87d1 100644 --- a/.agents/skills/benchmark-report/references/report-and-release.md +++ b/.agents/skills/benchmark-report/references/report-and-release.md @@ -12,7 +12,7 @@ Diagnostic failure analysis derives unmet requirements and expected/actual value Fresh diagnostic cohorts can supply a `webfetch_diagnostic_inputs/1` JSON file to the same build entry. It names complete saved run directories, independent whole-page browser observations and per-case reviews bound to dataset/results/observation hashes. The owner validates call and output identities and adapts these inputs to the existing model; no separate renderer is needed. Public copies may redact sensitive fields using the shared publication projector, with original/published hashes and an explicit local scope note; never alter the scored originals or present a redacted view as byte-identical. Review all requested passes as well as failures, retain access restrictions as observed limitations, and use actual requirements and source errors in interpretation specimens rather than storage metadata. -For live stable-contract cohorts, invoke `scripts/run_certified_benchmark.py --diagnostic`: invariant scoring rules alone do not qualify the current source. The generic entry rejects an unqualified stable-contract cohort before external work unless this diagnostic state is explicit. `--retrieval-mode native` selects ordinary WebFetch completion for a native experiment while preserving scenario categories and every acceptance requirement; it must reject declared retrieval hints instead of dropping them. Independent whole-page auditing uses the existing browser capturer with `--observation-scope visible_page --checkpoint-dir ...`; checkpoint identities include capture inputs and observer code, so resumption cannot silently reuse observations from another protocol. +For reruns of unchanged compiled stable-contract datasets, invoke `scripts/run_certified_benchmark.py` normally. Freeze existing task definitions and acceptance rules; do not require historical qualification raw files or repeat admission review. Source qualification belongs to adding tasks or changing references/acceptance rules. Live exact-reference cases retain pre/post reference checks. Reserve `--diagnostic` for explicitly diagnostic runs, whose outputs cannot be upgraded to certified evidence. `--retrieval-mode native` selects ordinary WebFetch completion while preserving scenario categories and every acceptance requirement; it rejects selector-guided retrieval hints instead of dropping them. Independent whole-page auditing, when requested, uses the existing browser capturer with `--observation-scope visible_page --checkpoint-dir ...`; checkpoint identities include capture inputs and observer code so observations from another protocol cannot be reused silently. For a diagnostic release, freeze the complete intended population in the track profile's `cohort_plan` (group IDs, case counts and SHA-256 of sorted case IDs joined by newlines with a trailing newline), independently of the report's selected inputs. Bind a uniform implementation when required. Pass that profile with `--profile` to both `scripts/build_report.py --diagnostic-cohorts` and `scripts/check_diagnostic_report.py` before publishing. A self-consistent subtotal is not evidence that the requested matrix is complete; reject missing groups, substituted cases and mixed implementations. Template examples without a deployment profile remain local examples, not a substitute for this release gate. @@ -66,6 +66,10 @@ Never publish user prompts, assistant reasoning, plans, internal notes, conversa Write every title and paragraph for the reader, not as a reply to author feedback. Convert corrections into neutral artifact facts; never mention who requested a change, quote review conversation, or use a rebuttal-style heading when a descriptive heading works. Review history may record an objective finding and resolution, while release history records only reader-visible effects. +For a fresh browser cohort, extend the maintained Web Fetch interpretation mode with `comparison.report_interpretation: {path, sha256}`. Keep its JSON prose outside Git in the evidence bundle. The `webfetch_browser_interpretation/1` schema binds the normalized task/result input hash, a title, introduction, scoring explanation, conclusions, and one explanation plus specimen case ID for every participant. The owner computes aggregate and family results, all-call and successful-call latency, observed versions, complete program-error distributions, and evidence links. A participant with failures must show a failed specimen. Historical prose bindings remain scoped to their original evidence; do not replace their hashes to reuse old conclusions. `scripts/check_report.py` verifies current-cohort numerical cells, populations, unique highlights, chapters and live result targets on both the interpretation and standalone routes, with negative controls in `tests/test_headless_report.py`. + +A composed cohort can use a compact current-report profile with `frozen_inputs` from `harness.web_fetch.storage.frozen_input_bindings` and an ordered per-track `case_inventory`. This binds the frozen dataset bytes and existing source/list-reference asset tree without duplicating whole datasets into a new repository folder. Both the build CLI and release checker validate these bindings and the complete provider-by-case matrix. The report package preserves original dataset bytes. These are publication-integrity checks on current inputs, not historical source-qualification requirements. + ## Claims and rankings Pair every ranking with its task family, denominator, acceptance meaning, a typical input-to-output route, and limits. Keep task acceptance separate from continuous quality, latency, cost, and title/body diagnostics. Do not call conditional averages global quality or explain a correlation as a verified mechanism. @@ -121,3 +125,5 @@ These rules apply to every public benchmark report, regardless of provider, task Repository adapter: `harness/reporting/presentation.py` provides track-independent checks over numeric cells marked with `data-metric`, `data-metric-direction`, and `data-sort`, and detail wrappers marked with `data-report-detail`. The current WebMainBench release gate invokes it. New or substantially rewritten tracks must integrate this checker (or equivalent semantic checks) in their own release command; adding a rule to this document is not sufficient. Existing tracks must not be claimed as covered until their gates actually invoke the checks. The local-audit variant also supports a new uniform native local measurement. Bind the exact committed case file, scorer, providers, one run, 45-second outer timeout and zero retries before calls. Preserve product-default retrieval without per-case wait/extraction hints. Validate completeness and every output/log hash before rendering through the existing report owner. Complete retained debug logs may be exposed through sanitized local report assets; do not publish raw private diagnostics or claim inherited independent review. + +Keep unpublished correction notes outside public release history. Schema 3 requires an explicit publication state for every release; an unmarked record fails the build. When a structured release is staged, mark it `publication_state: unpublished`; the public history selector and revision-binding gate must exclude it. Preserve its private correction receipt and measured calls. diff --git a/.dockerignore b/.dockerignore index 365cc8cb0..10ee128cf 100644 --- a/.dockerignore +++ b/.dockerignore @@ -15,3 +15,9 @@ configs browser_harness_eval.egg-info **/__pycache__ **/*.pyc +.env +.env.* +**/.env +**/.env.* +node_modules +**/node_modules diff --git a/.env.example b/.env.example index 1507f9e05..3cde476e2 100644 --- a/.env.example +++ b/.env.example @@ -13,6 +13,10 @@ MOLI_BINARY=moli # Optional compiled Darwin resolver for explicitly frozen public_doh runs. MOLI_NATIVE_DNS_LIBRARY= MOLI_PR643_BINARY= +# Source-build Moli participant: pin executable, reported version and hash before calls. +MOLI_PR919_BINARY= +MOLI_PR919_REPORTED_VERSION= +MOLI_PR919_BINARY_SHA256= MOLI_1_1_11_BINARY=moli-1.1.11 MOLI_1_1_7_BINARY=moli-1.1.7 MOLI_1_0_6_BINARY=moli-1.0.6 diff --git a/AGENTS.md b/AGENTS.md index c1312079e..a26afa971 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -36,7 +36,7 @@ These requirements apply to **all** report tracks; browser-specific rules below - For benchmark design, execution, scoring, result audit, report generation or release, read `.agents/skills/benchmark-report/SKILL.md`. Use `.agents/skills/benchmark-editorial/SKILL.md` additionally only for a new or substantially rewritten standalone interpretation article, official blog or public-account article. - Keep one report implementation in `harness/web_fetch/report.py`. The default `scripts/build_report.py` build uses persistent measured evidence (`~/.browser-eval/evidence/current/` by default, or `WEB_FETCH_EVIDENCE_DIR`) and writes `reports/webfetch/`; the independently bound WCXB build uses `--wcxb` and writes `reports/wcxb/`. Do not create versioned report implementations, compatibility renderers or redirect builds. - Run repository Python only through `make` targets or `uv run --frozen python`. `.python-version`, `uv.lock`, the Makefile and CI are the runtime contract; never fall back to system Python. -- Run every publishable provider cohort through `scripts/run_certified_benchmark.py`, which freezes references before calls and verifies them again afterward. Direct `run_benchmark.py` output and incomplete cohorts are diagnostic evidence only; never assemble or publish them. +- Run every publishable provider cohort through `scripts/run_certified_benchmark.py`, which freezes executable task inputs. Existing compiled reference answers stay frozen; pre/post source checks require an explicit `source_verification.reference_mode: live_exact` task declaration. Missing newer acceptance-contract fields never imply a live-reference task. Rerunning an unchanged compiled dataset must not require historical source-qualification raw files or reopen cohort admission; source qualification belongs to adding tasks or changing references/acceptance rules. Direct `run_benchmark.py` output and incomplete cohorts are diagnostic evidence only; never assemble or publish them. - For the default Web Fetch report, treat `evaluation/current-report.json` as the matrix and publication-target authority. The independent WCXB report uses only `evaluation/wcxb/current-report.json` for its matrix and target. Browser Use and Browser Trajectory Replay have separate contracts under their own `evaluation/` namespaces; never reuse a publication target across reports or tracks. Publishing must use the relevant bound site and expected remote version; do not copy target IDs from prose, old receipts or earlier reports. - For the complete WebMainBench report, `evaluation/webmainbench-full/current-report.json` is the immutable baseline pre-call profile bound by its cohort certificate. Independently certified variant cohorts are registered in `evaluation/webmainbench-full/`; discover the current variant registry and each variant's profile and certificate from that directory rather than hard-coding names. The method-binding file joins the baseline and all certified variant cohorts through locked profile hashes. Their `publication_state` records frozen pre-call state. The live site, share URL, and expected remote version are bound in `evaluation/webmainbench-full/publication-target.json` together with the baseline and all variant profile hashes; update this target after publication without changing or re-certifying historical calls. - For the current WCXB report, the source-qualified manifest is the entire evaluation population for quality, delivery, and latency. Keep source-invalid cases outside the current manifest, scored evidence, public package, and future provider calls; preserve earlier physical calls only in private historical evidence. The dataset page owns frozen task inputs, source URLs and human reference scope; it must not show measured browser scores. The home page owns aggregate comparisons, the result analysis owns the filterable method-case inventory, and case result pages own full responses. Keep a direct link from each dataset row to a safe frozen-page preview and its human reference. Default result filters must show a nonempty, clearly labeled diagnostic population when one exists, and filtered tables must shrink rather than leave a large blank viewport. diff --git a/configs/providers.headless.json b/configs/providers.headless.json index 1aac71c17..97a510af6 100644 --- a/configs/providers.headless.json +++ b/configs/providers.headless.json @@ -42,6 +42,15 @@ "engine": "moli", "binary": "${MOLI_PR643_BINARY}" }, + "moli-pr919": { + "type": "moli_webfetch", + "engine": "moli", + "label": "Moli · PR #919", + "source_url": "https://github.com/lexmount/moli/pull/919", + "binary": "${MOLI_PR919_BINARY}", + "expected_reported_version": "${MOLI_PR919_REPORTED_VERSION}", + "expected_binary_sha256": "${MOLI_PR919_BINARY_SHA256}" + }, "moli-1.1.3": { "type": "headless_markdown", "engine": "moli", diff --git a/evaluation/browser-native-430/current-report.json b/evaluation/browser-native-430/current-report.json new file mode 100644 index 000000000..ef0898668 --- /dev/null +++ b/evaluation/browser-native-430/current-report.json @@ -0,0 +1,515 @@ +{ + "schema_version": 1, + "dataset_id": "webfetch-five-browsers-seven-families-430", + "comparison_kind": "headless_markdown", + "case_count": 430, + "case_ids_sha256": "a81e5406477b6b2d705cedf32cf79d5ba65242fa3e0a839cb1def5ecdab7e05b", + "providers": [ + "moli", + "lightpanda", + "obscura", + "chromium", + "kitesurf" + ], + "timeout_seconds": 45, + "runs_per_case": 1, + "concurrency": 1, + "max_failure_retries": 0, + "conversion": null, + "reference_binding_mode": "equivalent_reference_bytes_with_independent_batch_captures", + "publication_state": "verified", + "unverified_reference_cases": [], + "require_independent_result_review": false, + "case_inventory": { + "web": [ + "static-simple", + "stable-wikipedia-ai-revision", + "cowagent-doc", + "lexmount-doc", + "github-repository", + "ecommerce-list", + "javascript-spa", + "html-table", + "redirect", + "web-retail-returns", + "lm-feishu-help", + "lm-python-tutorial", + "stable-rfc-8259-json", + "lm-node-intro", + "stable-rfc-6455-websocket", + "lm-rust-learn", + "live-001-python-pathlib-documentation", + "live-002-http-http-modules", + "live-003-the-rust-programming-language", + "live-004-deployments", + "live-005-python-asyncio-documentation", + "live-006-python-typing-documentation", + "live-007-html-hypertext-markup-language-s", + "live-008-service", + "live-009-run-a-stateless-application-usin", + "live-010-navigation-menu", + "stable-browser-use-readme", + "live-012-navigation-menu", + "live-013-navigation-menu", + "live-014-navigation-menu", + "live-015-configmaps", + "live-016-navigation-menu", + "live-017-navigation-menu", + "stable-openclaw-readme", + "live-019-navigation-menu", + "stable-nodejs-readme", + "live-021-python-common-pathname-manipulations", + "live-022-until-september-10-get-pycharm-f", + "stable-wcag21-recommendation", + "live-024-cascading-style-sheets-level-2-r", + "live-025-api-reference", + "live-026-getting-started", + "live-027-sponsors", + "live-028-getting-started-with-rails", + "live-029-fetch", + "live-030-rfc-9110", + "stable-xml10-recommendation", + "live-032-api-reference", + "live-033-nasa-science", + "live-034-travel", + "live-035-part-i-tutorial", + "live-036-build-on-the-openai-api-platform", + "live-037-breadcrumb", + "live-038-mystery", + "live-039-welcome-to-redis-docs", + "live-040-documentation-overview", + "live-041-python-programming-language", + "live-042-document-object-model-dom", + "live-043-syntax", + "live-044-history-api", + "live-045-cache", + "live-046-file-api", + "live-047-push-api", + "live-048-websocket-api-websockets", + "live-049-functools-higher-order-functions", + "live-050-re-regular-expression-operations", + "live-051-email-an-email-and-mime-handling", + "live-052-objects-in-kubernetes", + "live-053-dns-for-services-and-pods", + "live-054-navigation-menu", + "live-055-navigation-menu", + "live-056-navigation-menu", + "live-057-navigation-menu", + "live-058-navigation-menu", + "live-059-navigation-menu", + "live-060-database", + "live-061-version-control", + "live-062-natural-language-processing", + "live-063-navigation-menu", + "live-064-navigation-menu", + "live-065-navigation-menu", + "live-066-navigation-menu", + "live-067-navigation-menu", + "live-068-navigation-menu", + "live-069-navigation-menu", + "live-070-navigation-menu", + "live-071-science", + "live-072-python-itertools-documentation", + "live-073-python-json-documentation", + "live-074-python-sqlite3-documentation", + "live-075-python-logging-documentation", + "live-076-python-unittest-documentation", + "live-077-python-dataclasses-documentation", + "live-078-python-collections-documentation", + "live-079-python-hashlib-documentation", + "live-080-python-urllib-parse-documentatio", + "live-081-python-csv-documentation", + "live-082-python-decimal-documentation", + "live-083-python-datetime-documentation", + "live-084-python-concurrent-futures-docume" + ], + "document": [ + "document-01", + "document-02", + "document-03", + "document-04", + "document-05", + "document-06", + "document-07", + "document-08", + "document-09", + "document-10" + ], + "image": [ + "image-01", + "image-02", + "image-03", + "image-04", + "image-nasa-article", + "image-w3-trulli", + "image-unsplash-detail", + "image-pexels-detail", + "image-earth-main", + "image-10" + ], + "video": [ + "video-01", + "video-02", + "video-03", + "video-04", + "video-05", + "video-06", + "video-blender-release", + "video-08", + "video-09", + "video-bilibili-current" + ], + "dynamic": [ + "dynamic-062-zh-sspai-home", + "dynamic-071-es-cl-falabella-audio", + "dynamic-072-ko-lotteon-home", + "dynamic-076-es-es-nintendo-mario-search", + "dynamic-078-es-es-nintendo-switch2-search", + "dynamic-080-es-es-nintendo-switch2-edition-search", + "dynamic-091-pt-br-kabum-hardware", + "dynamic-023-en-mui-data-grid-pagination", + "dynamic-061-061-juejin-ranking", + "dynamic-068-ja-nintendo-switch2-software", + "dynamic-069-ja-nintendo-search", + "dynamic-057-nutui-jd-com-h5", + "dynamic-017-en-stable-eui-table", + "dynamic-041-en-stable-heroui-table", + "dynamic-019-selenium-dev-selenium", + "dynamic-020-webscraper-io-test", + "dynamic-026-bilibili-rank", + "dynamic-027-ej2-syncfusion-com-react", + "dynamic-028-ej2-syncfusion-com-react", + "dynamic-036-yonhap-home", + "dynamic-037-quasar-dev-vue", + "dynamic-038-material-angular-dev-components", + "dynamic-040-gridjs-io-docs", + "dynamic-051-youzan-github-io-vant", + "dynamic-052-bilibili-com-v", + "dynamic-054-ant-protable-api", + "dynamic-055-arco-design-vue", + "dynamic-056-vant-ui-github-io-vant", + "dynamic-059-layui-vue-com-zh", + "dynamic-063-s-weibo-com-top", + "dynamic-065-franceinfo-home", + "dynamic-070-nintendo-feature", + "dynamic-074-melon-com-chart", + "dynamic-082-allocine-films", + "dynamic-086-joyn-prosieben", + "dynamic-087-joyn-filme", + "dynamic-092-play-uol-com-br", + "dynamic-095-joyn-home", + "dynamic-098-shahid-mbc-net-ar", + "dynamic-013-en-stable-patternfly-table", + "dynamic-014-en-stable-rsuite-table", + "dynamic-015-015-testpages-iframe-list", + "dynamic-016-en-stable-angular-tree", + "dynamic-018-en-stable-varlet-home", + "dynamic-021-021-scalar-get-operation", + "dynamic-022-en-stable-syncfusion-grid", + "dynamic-024-en-stable-gridjs-pagination", + "dynamic-025-025-tabulator-vertical-virtual-dom", + "dynamic-029-029-primevue-basic-treetable", + "dynamic-030-030-primevue-basic-virtualscroller", + "dynamic-031-031-primeng-basic-table", + "dynamic-032-032-primeng-basic-virtualscroller", + "dynamic-033-033-vuetify-usage-table", + "dynamic-034-034-vuetify-server-table", + "dynamic-035-035-vuetify-virtual-table", + "dynamic-039-039-angular-autocomplete-examples", + "dynamic-043-en-stable-grommet-table", + "dynamic-046-en-real-hn-algolia", + "dynamic-050-en-real-usgs-earthquakes", + "dynamic-058-zh-stable-tdesign-table", + "dynamic-096-ar-real-cnn-world", + "dynamic-quotes-api", + "dynamic-products-api", + "dynamic-catalog", + "dynamic-delayed", + "dynamic-reviews-graphql", + "dynamic-react-cart", + "dynamic-angular-phones", + "dynamic-swagger", + "dynamic-product-reviews", + "dynamic-iframe", + "dynamic-104-mui-data-grid", + "dynamic-105-blueprint-docs", + "dynamic-107-redoc-cafe-api", + "dynamic-108-swagger-petstore3", + "replacement-en-syncfusion-filtering", + "replacement-en-gridjs-search", + "replacement-en-quasar-table", + "replacement-en-gridjs-sort", + "replacement-en-mui-sorting", + "replacement-ko-yonhap-economy", + "replacement-en-mui-filtering", + "replacement-en-syncfusion-sorting", + "replacement-zh-ant-descriptions", + "replacement-zh-ant-form", + "replacement-ja-nintendo-mario", + "replacement-ja-nintendo-zelda", + "replacement-ko-yonhap-sports", + "replacement-fix6-es-mx-lokus-home", + "replacement-fix6-es-mx-lokus-tutorials", + "replacement-fix3-fr-fr-datapass-stats", + "replacement-pt-kabum-computers", + "replacement-fix3-fr-fr-mobilites-durables", + "replacement-pt-kabum-gamer", + "replacement-fix3-de-dashboard-deutschland", + "replacement-pt-kabum-peripherals", + "replacement-pt-kabum-smartphones", + "replacement-fix2-ar-mantine-rtl", + "replacement-fix3-ar-antd-rtl", + "replacement-fix2-ar-shadcn-rtl" + ], + "challenge": [ + "global-homepage-001-google-com", + "global-homepage-002-youtube-com", + "global-homepage-005-chatgpt-com", + "china-homepage-089-yuque-com", + "global-homepage-008-whatsapp-com", + "global-homepage-010-bing-com", + "global-homepage-011-wikipedia-org", + "global-homepage-012-gemini-google-com", + "global-homepage-013-yahoo-co-jp", + "global-homepage-015-yahoo-com", + "global-homepage-016-amazon-com", + "global-homepage-018-baidu-com", + "global-homepage-020-naver-com", + "global-homepage-021-netflix-com", + "global-homepage-023-cloud-microsoft", + "global-homepage-025-bilibili-com", + "global-homepage-027-dzen-ru", + "global-homepage-028-microsoft-com", + "global-homepage-029-twitch-tv", + "global-homepage-030-weather-com", + "global-homepage-032-canva-com", + "global-homepage-034-news-yahoo-co-jp", + "global-homepage-035-fandom-com", + "global-homepage-036-globo-com", + "global-homepage-037-mail-ru", + "global-homepage-038-samsung-com", + "global-homepage-039-duckduckgo-com", + "global-homepage-041-discord-com", + "global-homepage-042-github-com", + "global-homepage-044-spotify-com", + "global-homepage-045-nytimes-com", + "global-homepage-046-zoom-us", + "global-homepage-049-brave-com", + "global-homepage-056-apple-com", + "global-homepage-060-espn-com", + "global-homepage-062-bbc-com", + "global-homepage-063-paypal-com", + "china-homepage-088-figma-com", + "global-homepage-067-deepseek-com", + "global-homepage-068-indeed-com", + "global-homepage-072-rakuten-co-jp", + "global-homepage-075-telegram-org", + "global-homepage-080-cnn-com", + "global-homepage-081-namu-wiki", + "global-homepage-082-shein-com", + "global-homepage-084-qq-com", + "global-homepage-085-douyin-com", + "global-homepage-086-usps-com", + "global-homepage-087-instructure-com", + "global-homepage-090-rutube-ru", + "china-homepage-087-gamersky-com", + "challenge-02", + "challenge-03", + "challenge-04", + "challenge-05", + "challenge-06", + "china-homepage-090-toutiao-com", + "challenge-10-justwatch", + "china-homepage-008-doubao-com", + "china-homepage-012-feishu-cn", + "china-homepage-018-163-com", + "china-homepage-019-taobao-com", + "china-homepage-026-aliyun-com", + "china-homepage-028-weibo-com", + "china-homepage-029-360-com", + "china-homepage-033-dingtalk-com", + "china-homepage-036-hao123-com", + "china-homepage-039-1688-com", + "china-homepage-040-quark-cn", + "china-homepage-042-kdocs-cn", + "china-homepage-044-kimi-com", + "china-homepage-045-tmall-com", + "china-homepage-046-wedoany-com", + "china-homepage-047-qianwen-com", + "china-homepage-048-douyu-com", + "china-homepage-050-openai-com", + "china-homepage-052-alipay-com", + "china-homepage-053-sohu-com", + "china-homepage-055-huya-com", + "china-homepage-056-cnki-net", + "china-homepage-057-nih-gov", + "china-homepage-058-huaban-com", + "china-homepage-059-jianying-com", + "china-homepage-060-eastmoney-com", + "china-homepage-063-sina-com-cn", + "china-homepage-064-gamer520-com", + "china-homepage-065-lingxing-com", + "china-homepage-067-alibaba-com", + "china-homepage-068-youdao-com", + "china-homepage-069-3dmgame-com", + "china-homepage-071-2345-com", + "china-homepage-072-cctv-com", + "china-homepage-073-so-com", + "china-homepage-075-volcengine-com", + "china-homepage-078-nga-cn", + "china-homepage-080-yandex-com", + "china-homepage-081-elsevier-com", + "china-homepage-082-huawei-com", + "china-homepage-083-grok-com", + "china-homepage-085-shopify-com" + ], + "risk_control": [ + "risk-diverse-002", + "risk-001", + "risk-006", + "risk-011", + "risk-061", + "risk-101", + "risk-102", + "risk-diverse-271", + "risk-diverse-027", + "risk-new-102", + "risk-diverse-030", + "risk-diverse-031", + "risk-diverse-032", + "risk-diverse-033", + "risk-diverse-034", + "risk-diverse-036", + "risk-diverse-038", + "risk-diverse-039", + "risk-diverse-040", + "risk-diverse-041", + "risk-diverse-042", + "risk-diverse-045", + "risk-diverse-047", + "risk-diverse-048", + "risk-diverse-049", + "risk-diverse-050", + "risk-diverse-051", + "risk-diverse-055", + "risk-diverse-057", + "risk-diverse-059", + "risk-diverse-074", + "risk-diverse-101", + "risk-diverse-104", + "risk-diverse-106", + "risk-diverse-113", + "risk-diverse-114", + "risk-diverse-117", + "risk-diverse-118", + "risk-diverse-121", + "risk-diverse-122", + "risk-diverse-123", + "risk-diverse-129", + "risk-diverse-134", + "risk-diverse-135", + "risk-diverse-138", + "risk-diverse-139", + "risk-diverse-153", + "risk-diverse-158", + "risk-diverse-159", + "risk-diverse-160", + "risk-diverse-170", + "risk-diverse-177", + "risk-diverse-181", + "risk-diverse-182", + "risk-diverse-185", + "risk-diverse-192", + "risk-diverse-193", + "risk-diverse-219", + "risk-diverse-220", + "risk-diverse-221", + "risk-diverse-222", + "risk-diverse-223", + "risk-diverse-225", + "risk-diverse-226", + "risk-diverse-011", + "risk-diverse-012", + "risk-diverse-241", + "risk-diverse-242", + "risk-diverse-244", + "risk-diverse-245", + "risk-diverse-248", + "risk-diverse-262", + "risk-diverse-263", + "risk-diverse-267", + "risk-diverse-268", + "risk-new-006", + "risk-diverse-199", + "risk-diverse-200", + "risk-diverse-201", + "risk-diverse-202", + "risk-diverse-204", + "risk-diverse-205", + "risk-diverse-208", + "risk-diverse-209", + "risk-diverse-210", + "risk-diverse-211", + "risk-diverse-212", + "risk-diverse-213", + "risk-diverse-214", + "risk-diverse-215", + "risk-diverse-216", + "risk-diverse-228", + "risk-diverse-230", + "risk-diverse-231", + "risk-diverse-234", + "risk-diverse-236", + "risk-diverse-237", + "risk-diverse-238", + "risk-diverse-239", + "risk-diverse-250" + ] + }, + "frozen_inputs": { + "dataset_sha256": "7ac804c896f9af55fa5a01c052aad0dce28a5d1835bdc0559ea230a4b45fca2d", + "auxiliary_sha256": "d687f5114e52c030630399797af2f4add2061dfb17c83fb58a81cbfd1af1e312" + }, + "report_output": "reports/browser-native-430", + "source_native_dataset_sha256": "d636aa8af567da8a2e0ff8caf319dd6369528d5f65f0edfe47db1ac0d9890d64", + "generated_languages": [ + "zh" + ], + "resource_profile": { + "required": true, + "comparison_kind": "headless_client_resource_profile", + "case_count": 20, + "providers": [ + "moli", + "lightpanda", + "obscura", + "chromium" + ], + "concurrency": 1, + "sample_interval_ms": 50, + "backend": "docker_cgroup_v2", + "cpus_per_call": 2, + "memory_per_call": "2g", + "network_isolation": "one dedicated bridge with no peer containers per call", + "controller_excluded": true, + "required_metrics": [ + "measurement_duration_ms", + "cpu_ms", + "cpu_average_utilization_pct", + "cpu_sampled_utilization_p95_pct", + "memory_mean_bytes", + "memory_p95_bytes", + "memory_peak_bytes", + "memory_mib_seconds", + "network_rx_bytes", + "network_tx_bytes", + "network_rx_bytes_per_second", + "network_tx_bytes_per_second" + ] + }, + "native_task_route_contract": 4, + "artifact_hub_base_url": "https://artifact-site.lexmount.cn", + "artifact_hub_site": "jGXOOJwkdBK_", + "artifact_hub_expected_version": "ver_840272a174e344269e4ebbcc9d114e6a", + "artifact_hub_share_url": "https://artifact-site.lexmount.cn/v/IqGpMUYHWat-dw4n8usQttTptoPzboAo" +} diff --git a/evaluation/risk-control/README.md b/evaluation/risk-control/README.md index ebe2f8895..0f89651a1 100644 --- a/evaluation/risk-control/README.md +++ b/evaluation/risk-control/README.md @@ -25,7 +25,16 @@ uv run --frozen python scripts/run_certified_benchmark.py risk-control \ --output "$BROWSER_EVAL_DATA_HOME/evidence/risk-control/" ``` -Output must be a new directory under the data root. The certified entry point is `scripts/run_certified_benchmark.py risk-control`; direct `--execute` on the execution script is rejected. A new run under the current profile has exactly 100 logical results and no automatic retries, retaining complete hash-bound debug stderr outside Git. Historical evidence additionally contains three user-authorized calls after DNS failures: 103 physical calls in total. The final response is authoritative, all call costs remain recorded and no best-response selection is allowed. +输出目录必须是新的，并位于数据目录下。正式入口固定为 +`scripts/run_certified_benchmark.py risk-control`；直接对执行脚本使用 `--execute` 会被拒绝。 +当前 profile 的新正式运行恰有 100 个逻辑结果，且不自动重试，并为每次调用在 Git 外保留完整、 +哈希绑定的 debug stderr。保留的历史证据另有 3 个 DNS 失败后的用户授权补调，因此保留 103 次 +物理调用；每例固定采用最后一次响应，全部调用耗时均计入，不能择优。运行前后 Moli 身份必须 +一致；profile、已编译的数据集、规则和 provider 配置也会在调用前后核验。 +普通重跑不读取历史资格原文，也不重做入选审核；修改题目、参考或规则时，才由构建器 +核验相应来源。新运行生成 `frozen-input-verification.json` 和 v2 认证回执。 +生成报告前校验 `certification.json` 对 profile、dataset、results、manifest、execution、 +冻结输入核验回执和完整日志的绑定。历史 v1 回执继续按原证据契约读取，不改写或升级。 Moli identity must remain unchanged before and after calls. Each of the 100 declared references is checked before and after execution; missing, changed or incomplete scope stops certification. Report builds verify `certification.json` bindings to the profile, dataset, results, manifest, execution, reference-verification receipts and complete logs. diff --git a/evaluation/scoring-regressions.json b/evaluation/scoring-regressions.json index 82663f684..d7c001a1d 100644 --- a/evaluation/scoring-regressions.json +++ b/evaluation/scoring-regressions.json @@ -21777,6 +21777,163 @@ "expected_metrics": { "stable_unique_record_count": 0 } +}, +{ + "id": "unmatched-markdown-brackets-retains-required-prose", + "category": "format-normalization", + "root_cause": "Repeated suffix scans for unclosed Markdown delimiters make ordinary prose quadratic; indexing delimiters must preserve visible text and anchor decisions.", + "failure_mode": "false-negative", + "case": { + "url": "https://example.test/body", + "acceptance_contract": { + "version": 1, + "schema": "body_text", + "minimum_records": 0, + "stable_text_anchors": [ + { + "id": "body", + "alternatives": [ + "Source prose" + ] + } + ] + } + }, + "artifact": "Source prose [ [ [ [ [ [ [ [ [ [ [ [ [ [ [ [ [ [ [ [ [ [ [ [ [ [ [ [ [ [ [ [ [label]( [label]( [label]( [label]( [label]( [label]( [label]( [label](", + "ok": true, + "expected_task_pass": true, + "expected_metrics": { + "stable_task_pass": true + } +}, +{ + "id": "unmatched-markdown-brackets-cannot-supply-missing-prose", + "category": "format-normalization", + "root_cause": "Repeated suffix scans for unclosed Markdown delimiters make ordinary prose quadratic; indexing delimiters must preserve visible text and anchor decisions.", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/body", + "acceptance_contract": { + "version": 1, + "schema": "body_text", + "minimum_records": 0, + "stable_text_anchors": [ + { + "id": "body", + "alternatives": [ + "Source prose" + ] + } + ] + } + }, + "artifact": "Unrelated prose [ [ [ [ [ [ [ [ [ [ [ [ [ [ [ [ [ [ [ [ [ [ [ [ [ [ [ [ [ [ [ [ [label]( [label]( [label]( [label]( [label]( [label]( [label]( [label](", + "ok": true, + "expected_task_pass": false, + "expected_metrics": { + "stable_task_pass": false + } +}, +{ + "id": "required-final-url-missing-is-a-measured-failure", + "category": "task-predicate", + "root_cause": "missing_provider_observation_is_not_missing_benchmark_reference", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/start", + "track": "risk_control", + "task_kind": "body", + "source_benchmark": { + "risk_control": { + "navigation_contract": { + "requested_url": "https://example.test/start", + "observed_final_url": "https://example.test/final" + } + } + }, + "acceptance_contract": { + "version": 1, + "schema": "body_text", + "minimum_records": 0, + "stable_text_anchors": [ + { + "id": "body", + "alternatives": [ + "Required body" + ] + } + ] + } + }, + "artifact": "Required body", + "ok": true, + "expected_task_pass": false, + "expected_task_measured": true, + "expected_metrics": { + "page_identity_required": true, + "page_identity_status": "unavailable" + } +}, +{ + "id": "required-final-url-present-keeps-valid-body", + "category": "task-predicate", + "root_cause": "missing_provider_observation_is_not_missing_benchmark_reference", + "failure_mode": "false-negative", + "case": { + "url": "https://example.test/start", + "track": "risk_control", + "task_kind": "body", + "source_benchmark": { + "risk_control": { + "navigation_contract": { + "requested_url": "https://example.test/start", + "observed_final_url": "https://example.test/final" + } + } + }, + "acceptance_contract": { + "version": 1, + "schema": "body_text", + "minimum_records": 0, + "stable_text_anchors": [ + { + "id": "body", + "alternatives": [ + "Required body" + ] + } + ] + } + }, + "artifact": "Required body", + "ok": true, + "expected_task_pass": true, + "expected_task_measured": true, + "expected_metrics": { + "page_identity_required": true, + "page_identity_status": "verified" + }, + "final_url": "https://example.test/final" +}, + {"id":"content-policy-dynamic-final-page-mismatch-cannot-be-counted-as-a-pass","category":"acceptance-contract","root_cause":"dynamic_page_identity_alignment","failure_mode":"false-negative","case":{"url":"https://example.test/start","evaluation_url":"https://example.test/start","expected_final_url":"https://example.test/en/catalog","target_language":"en-US","target_region":"US","track":"dynamic","task_kind":"body","acceptance_contract":{"version":1,"schema":"body_text","minimum_records":0,"stable_text_anchors":[{"id":"identity","alternatives":["Catalog"]}]}},"artifact":"Catalog","ok":true,"final_url":"https://example.test/fr/catalog","metadata":{"route":"dynamic_selected_text","document_language":"fr"},"expected_task_pass":true,"expected_metrics":{"page_identity_status":"mismatch","final_url_consistent":false,"language_consistent":false,"page_identity_required":false}}, + {"id":"content-policy-dynamic-search-query-mismatch-is-a-different-page","category":"acceptance-contract","root_cause":"dynamic_page_query_identity","failure_mode":"false-negative","case":{"url":"https://example.test/search?q=Mario","evaluation_url":"https://example.test/search?q=Mario","expected_final_url":"https://example.test/search?q=Mario","target_language":"ja-JP","target_region":"JP","track":"dynamic","task_kind":"body","acceptance_contract":{"version":1,"schema":"body_text","minimum_records":0,"stable_text_anchors":[{"id":"identity","alternatives":["Search results"]}]}},"artifact":"Search results","ok":true,"final_url":"https://example.test/search?q=Zelda","metadata":{"route":"dynamic_selected_text","document_language":"ja"},"expected_task_pass":true,"expected_metrics":{"page_identity_status":"mismatch","final_url_consistent":false,"page_identity_required":false}}, + {"id":"content-policy-dynamic-source-document-language-mismatch-is-not-verified","category":"acceptance-contract","root_cause":"declared_document_language_vs_visible_language","failure_mode":"false-negative","case":{"url":"https://example.test/rtl","evaluation_url":"https://example.test/rtl","expected_final_url":"https://example.test/rtl","target_language":"ar","expected_document_language":"en","target_region":"US","track":"dynamic","task_kind":"body","acceptance_contract":{"version":1,"schema":"body_text","minimum_records":0,"stable_text_anchors":[{"id":"arabic-content","alternatives":["الشخص"]}]}},"artifact":"الشخص","ok":true,"final_url":"https://example.test/rtl","metadata":{"route":"dynamic_selected_text","document_language":"fr"},"expected_task_pass":true,"expected_metrics":{"page_identity_status":"mismatch","language_consistent":false,"page_identity_required":false}}, + {"id":"content-policy-dynamic-undeclared-source-language-rejects-declared-language","category":"acceptance-contract","root_cause":"source_document_language_absent","failure_mode":"false-negative","case":{"url":"https://example.test/untagged","evaluation_url":"https://example.test/untagged","expected_final_url":"https://example.test/untagged","target_language":"en","target_region":"US","expected_document_language_present":false,"track":"dynamic","task_kind":"body","acceptance_contract":{"version":1,"schema":"body_text","minimum_records":0,"stable_text_anchors":[{"id":"content","alternatives":["Visible content"]}]}},"artifact":"Visible content","ok":true,"final_url":"https://example.test/untagged","metadata":{"route":"dynamic_selected_text","document_language":"en"},"expected_task_pass":true,"expected_metrics":{"page_identity_status":"mismatch","language_consistent":false,"page_identity_required":false}}, + {"id":"content-policy-required-final-url-missing-is-a-measured-failure","category":"task-predicate","root_cause":"body_fetch_does_not_require_final_url","failure_mode":"false-negative","case":{"url":"https://example.test/start","track":"risk_control","task_kind":"body","source_benchmark":{"risk_control":{"navigation_contract":{"requested_url":"https://example.test/start","observed_final_url":"https://example.test/final"}}},"acceptance_contract":{"version":1,"schema":"body_text","minimum_records":0,"stable_text_anchors":[{"id":"body","alternatives":["Required body"]}]}},"artifact":"Required body","ok":true,"expected_task_pass":true,"expected_task_measured":true,"expected_metrics":{"page_identity_required":false,"page_identity_status":"unavailable"}}, + {"id":"content-policy-required-final-url-present-keeps-valid-body","category":"task-predicate","root_cause":"body_fetch_does_not_require_final_url","failure_mode":"false-negative","case":{"url":"https://example.test/start","track":"risk_control","task_kind":"body","source_benchmark":{"risk_control":{"navigation_contract":{"requested_url":"https://example.test/start","observed_final_url":"https://example.test/final"}}},"acceptance_contract":{"version":1,"schema":"body_text","minimum_records":0,"stable_text_anchors":[{"id":"body","alternatives":["Required body"]}]}},"artifact":"Required body","ok":true,"expected_task_pass":true,"expected_task_measured":true,"expected_metrics":{"page_identity_required":false,"page_identity_status":"verified"},"final_url":"https://example.test/final"}, + {"category":"acceptance-contract","root_cause":"body_fetch_does_not_require_final_url","case":{"url":"https://example.test/start","track":"risk_control","task_kind":"body","source_benchmark":{"risk_control":{"navigation_contract":{"requested_url":"https://example.test/start","observed_final_url":"https://example.test/final"}}},"acceptance_contract":{"version":1,"schema":"body_text","minimum_records":0,"stable_text_anchors":[{"id":"body","alternatives":["Target article content"]}]}},"ok":true,"expected_task_measured":true,"expected_metrics":{"page_identity_required":false},"id":"body-without-final-url-passes","artifact":"Target article content","expected_task_pass":true,"failure_mode":"false-negative"}, + {"category":"acceptance-contract","root_cause":"body_fetch_does_not_require_final_url","case":{"url":"https://example.test/start","track":"risk_control","task_kind":"body","source_benchmark":{"risk_control":{"navigation_contract":{"requested_url":"https://example.test/start","observed_final_url":"https://example.test/final"}}},"acceptance_contract":{"version":1,"schema":"body_text","minimum_records":0,"stable_text_anchors":[{"id":"body","alternatives":["Target article content"]}]}},"ok":true,"expected_task_measured":true,"expected_metrics":{"page_identity_required":false},"id":"verification-page-without-final-url-fails","artifact":"Please sign in to continue","expected_task_pass":false,"failure_mode":"false-positive"}, + {"id":"wrong-body-content-policy-dynamic-final-page-mismatch-cannot-be-counted-as-a-pass","category":"acceptance-contract","root_cause":"dynamic_page_identity_alignment","failure_mode":"false-positive","case":{"url":"https://example.test/start","evaluation_url":"https://example.test/start","expected_final_url":"https://example.test/en/catalog","target_language":"en-US","target_region":"US","track":"dynamic","task_kind":"body","acceptance_contract":{"version":1,"schema":"body_text","minimum_records":0,"stable_text_anchors":[{"id":"identity","alternatives":["Catalog"]}]}},"artifact":"Unrelated page text","ok":true,"final_url":"https://example.test/fr/catalog","metadata":{"route":"dynamic_selected_text","document_language":"fr"},"expected_task_pass":false,"expected_metrics":{"page_identity_status":"mismatch","final_url_consistent":false,"language_consistent":false,"page_identity_required":false}}, + {"id":"wrong-body-content-policy-dynamic-search-query-mismatch-is-a-different-page","category":"acceptance-contract","root_cause":"dynamic_page_query_identity","failure_mode":"false-positive","case":{"url":"https://example.test/search?q=Mario","evaluation_url":"https://example.test/search?q=Mario","expected_final_url":"https://example.test/search?q=Mario","target_language":"ja-JP","target_region":"JP","track":"dynamic","task_kind":"body","acceptance_contract":{"version":1,"schema":"body_text","minimum_records":0,"stable_text_anchors":[{"id":"identity","alternatives":["Search results"]}]}},"artifact":"Unrelated page text","ok":true,"final_url":"https://example.test/search?q=Zelda","metadata":{"route":"dynamic_selected_text","document_language":"ja"},"expected_task_pass":false,"expected_metrics":{"page_identity_status":"mismatch","final_url_consistent":false,"page_identity_required":false}}, + {"id":"wrong-body-content-policy-dynamic-source-document-language-mismatch-is-not-verified","category":"acceptance-contract","root_cause":"declared_document_language_vs_visible_language","failure_mode":"false-positive","case":{"url":"https://example.test/rtl","evaluation_url":"https://example.test/rtl","expected_final_url":"https://example.test/rtl","target_language":"ar","expected_document_language":"en","target_region":"US","track":"dynamic","task_kind":"body","acceptance_contract":{"version":1,"schema":"body_text","minimum_records":0,"stable_text_anchors":[{"id":"arabic-content","alternatives":["الشخص"]}]}},"artifact":"Unrelated page text","ok":true,"final_url":"https://example.test/rtl","metadata":{"route":"dynamic_selected_text","document_language":"fr"},"expected_task_pass":false,"expected_metrics":{"page_identity_status":"mismatch","language_consistent":false,"page_identity_required":false}}, + {"id":"wrong-body-content-policy-dynamic-undeclared-source-language-rejects-declared-language","category":"acceptance-contract","root_cause":"source_document_language_absent","failure_mode":"false-positive","case":{"url":"https://example.test/untagged","evaluation_url":"https://example.test/untagged","expected_final_url":"https://example.test/untagged","target_language":"en","target_region":"US","expected_document_language_present":false,"track":"dynamic","task_kind":"body","acceptance_contract":{"version":1,"schema":"body_text","minimum_records":0,"stable_text_anchors":[{"id":"content","alternatives":["Visible content"]}]}},"artifact":"Unrelated page text","ok":true,"final_url":"https://example.test/untagged","metadata":{"route":"dynamic_selected_text","document_language":"en"},"expected_task_pass":false,"expected_metrics":{"page_identity_status":"mismatch","language_consistent":false,"page_identity_required":false}} +], + "superseded_cases": { + "dynamic-final-page-mismatch-cannot-be-counted-as-a-pass": "content-policy-dynamic-final-page-mismatch-cannot-be-counted-as-a-pass", + "dynamic-search-query-mismatch-is-a-different-page": "content-policy-dynamic-search-query-mismatch-is-a-different-page", + "dynamic-source-document-language-mismatch-is-not-verified": "content-policy-dynamic-source-document-language-mismatch-is-not-verified", + "dynamic-undeclared-source-language-rejects-declared-language": "content-policy-dynamic-undeclared-source-language-rejects-declared-language", + "required-final-url-missing-is-a-measured-failure": "content-policy-required-final-url-missing-is-a-measured-failure", + "required-final-url-present-keeps-valid-body": "content-policy-required-final-url-present-keeps-valid-body" } -] } diff --git a/examples/web_fetch/docker/resource.Dockerfile b/examples/web_fetch/docker/resource.Dockerfile index 96bc1f3e5..31fd266f8 100644 --- a/examples/web_fetch/docker/resource.Dockerfile +++ b/examples/web_fetch/docker/resource.Dockerfile @@ -10,14 +10,19 @@ ARG LIGHTPANDA_SHA_AMD64=4200e6dd7b7dd4ab60224c83f5d5f3688e7fd8cfe9389155f3bc5d6 ARG OBSCURA_VERSION=v0.2.1 ARG OBSCURA_SHA_ARM64=0297c26d583f598f0126a7271cc40750598a9a9cbd86d1d6f79b2b99097d5244 ARG OBSCURA_SHA_AMD64=6a1a66b3f1ab118fa7d31330894a868617aea68c06d75436d851356c39df1ed3 -ARG MOLI_1_1_1_SHA_ARM64=549484765476b8dd3fd93ebf59a089e4424425a961c14874974a88bba6d8b5b4 -ARG MOLI_1_1_1_SHA_AMD64=7b3eb9cbbf2cc8bd5ea9ef4a5bdb24cee2df35d26da621216a8b69c2aff3ebaa +ARG MOLI_VERSION=1.1.1 +ARG MOLI_SHA_ARM64=549484765476b8dd3fd93ebf59a089e4424425a961c14874974a88bba6d8b5b4 +ARG CHROMIUM_URL_ARM64= +ARG CHROMIUM_SHA_ARM64= +ARG CHROMIUM_URL_AMD64= +ARG CHROMIUM_SHA_AMD64= +ARG MOLI_SHA_AMD64=7b3eb9cbbf2cc8bd5ea9ef4a5bdb24cee2df35d26da621216a8b69c2aff3ebaa COPY --from=uv /uv /usr/local/bin/uv RUN apt-get update \ && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ ca-certificates chromium curl fonts-liberation fonts-noto-cjk \ - libcurl4 libgcc-s1 libstdc++6 nodejs npm python3 python3-venv tar \ + libcurl4 libgcc-s1 libstdc++6 nodejs npm python3 python3-venv tar unzip \ && rm -rf /var/lib/apt/lists/* \ && useradd --create-home --uid 10001 evaluator @@ -56,21 +61,39 @@ RUN target_arch="${TARGETARCH:-$(dpkg --print-architecture)}" \ RUN target_arch="${TARGETARCH:-$(dpkg --print-architecture)}" \ && case "$target_arch" in \ - arm64) archive_arch=aarch64; sha="$MOLI_1_1_1_SHA_ARM64" ;; \ - amd64) archive_arch=x86_64; sha="$MOLI_1_1_1_SHA_AMD64" ;; \ + arm64) archive_arch=aarch64; sha="$MOLI_SHA_ARM64" ;; \ + amd64) archive_arch=x86_64; sha="$MOLI_SHA_AMD64" ;; \ *) echo "unsupported TARGETARCH=$target_arch" >&2; exit 2 ;; \ esac \ && aria2c --allow-overwrite=true --check-certificate=true --console-log-level=warn \ --max-connection-per-server=8 --split=8 --min-split-size=1M \ - --dir=/tmp --out=moli-1.1.1.tar.gz \ - "https://github.com/lexmount/moli/releases/download/v1.1.1/moli-${archive_arch}-unknown-linux-gnu.tar.gz" \ - && printf '%s %s\n' "$sha" /tmp/moli-1.1.1.tar.gz | sha256sum -c - \ - && mkdir -p /tmp/moli-1.1.1 \ - && tar -xzf /tmp/moli-1.1.1.tar.gz -C /tmp/moli-1.1.1 \ - && find /tmp/moli-1.1.1 -type f -name moli -exec install -m 0755 '{}' /usr/local/bin/moli-1.1.1 \; \ - && test -x /usr/local/bin/moli-1.1.1 \ + --dir=/tmp --out=moli-${MOLI_VERSION}.tar.gz \ + "https://github.com/lexmount/moli/releases/download/v${MOLI_VERSION}/moli-${archive_arch}-unknown-linux-gnu.tar.gz" \ + && printf '%s %s\n' "$sha" /tmp/moli-${MOLI_VERSION}.tar.gz | sha256sum -c - \ + && mkdir -p /tmp/moli-${MOLI_VERSION} \ + && tar -xzf /tmp/moli-${MOLI_VERSION}.tar.gz -C /tmp/moli-${MOLI_VERSION} \ + && find /tmp/moli-${MOLI_VERSION} -type f -name moli -exec install -m 0755 '{}' /usr/local/bin/moli-${MOLI_VERSION} \; \ + && test -x /usr/local/bin/moli-${MOLI_VERSION} \ && rm -rf /tmp/moli-* +# Optional official Chrome for Testing archive. The caller binds its exact URL +# and digest; the default image retains the existing Debian Chromium route. +RUN mkdir -p /opt/chrome-for-testing \ + && target_arch="${TARGETARCH:-$(dpkg --print-architecture)}" \ + && case "$target_arch" in \ + arm64) url="$CHROMIUM_URL_ARM64"; sha="$CHROMIUM_SHA_ARM64" ;; \ + amd64) url="$CHROMIUM_URL_AMD64"; sha="$CHROMIUM_SHA_AMD64" ;; \ + *) echo "unsupported TARGETARCH=$target_arch" >&2; exit 2 ;; \ + esac \ + && if [ -n "$url" ]; then \ + test -n "$sha" \ + && curl --fail --location --proto '=https' --tlsv1.2 "$url" -o /tmp/chromium.zip \ + && printf '%s %s\n' "$sha" /tmp/chromium.zip | sha256sum -c - \ + && unzip -q /tmp/chromium.zip -d /opt/chrome-for-testing \ + && test -n "$(find /opt/chrome-for-testing -type f -name chrome -print -quit)" \ + && rm /tmp/chromium.zip; \ + else test -z "$sha"; fi + FROM python:3.11-slim-trixie@sha256:9534e5a8e315485d4061ed659af0fd78a284c015f9b73661b41d6bab25604534 ARG DEBIAN_MIRROR=https://mirrors.aliyun.com/debian @@ -82,7 +105,7 @@ RUN sed -i "s|http://deb.debian.org/debian-security|${DEBIAN_SECURITY_MIRROR}|g; /etc/apt/sources.list.d/debian.sources \ && apt-get update \ && packages="ca-certificates chromium curl fonts-liberation fonts-noto-cjk libcurl4 libgcc-s1 libstdc++6 tar" \ - && DEBIAN_FRONTEND=noninteractive apt-get --print-uris --yes --no-install-recommends install $packages > /tmp/apt-uris.txt \ + && DEBIAN_FRONTEND=noninteractive apt-get -o Acquire::ForceHash=SHA256 --print-uris --yes --no-install-recommends install $packages > /tmp/apt-uris.txt \ && python /usr/local/bin/download_apt_uris.py /tmp/apt-uris.txt /var/cache/apt/archives \ && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-download --no-install-recommends $packages \ && rm -rf /var/lib/apt/lists/* \ @@ -90,7 +113,8 @@ RUN sed -i "s|http://deb.debian.org/debian-security|${DEBIAN_SECURITY_MIRROR}|g; && useradd --create-home --uid 10001 evaluator COPY --from=downloads /usr/local/bin/lightpanda /usr/local/bin/lightpanda COPY --from=downloads /opt/obscura /opt/obscura -COPY --from=downloads /usr/local/bin/moli-1.1.1 /usr/local/bin/moli-1.1.1 +COPY --from=downloads /usr/local/bin/moli-* /usr/local/bin/ +COPY --from=downloads /opt/chrome-for-testing /opt/chrome-for-testing RUN find /opt/obscura -type f -name obscura -exec ln -s '{}' /usr/local/bin/obscura \; WORKDIR /workspace diff --git a/harness/reporting/links.py b/harness/reporting/links.py index 24685d260..a769f8fe0 100644 --- a/harness/reporting/links.py +++ b/harness/reporting/links.py @@ -63,10 +63,10 @@ def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None self.base_href = attrs_by_name.get("href") return if tag=='select': - self.current_filter=attrs_by_name.get('data-wmb-filter') + self.current_filter=(attrs_by_name.get('data-wmb-filter') or attrs_by_name.get('data-filter')) if self.current_filter:self.filter_options.setdefault(self.current_filter,set()) - elif tag=='input' and attrs_by_name.get('data-wmb-filter'): - self.text_filters.add(attrs_by_name['data-wmb-filter']) + elif tag=='input' and (attrs_by_name.get('data-wmb-filter') or attrs_by_name.get('data-filter')): + self.text_filters.add(attrs_by_name.get('data-wmb-filter') or attrs_by_name['data-filter']) elif tag=='option' and self.current_filter: self.filter_options[self.current_filter].add(attrs_by_name.get('value') or '') for key, value in attrs: diff --git a/harness/web_fetch/HEADLESS.md b/harness/web_fetch/HEADLESS.md index f1eb178de..051fc054d 100644 --- a/harness/web_fetch/HEADLESS.md +++ b/harness/web_fetch/HEADLESS.md @@ -8,8 +8,10 @@ The separate frozen-HTML public-data comparison is documented in [WEBMAINBENCH.md](WEBMAINBENCH.md); its fidelity scores are not mixed with the live-URL task acceptance or resource profile below. -The cohort reuses all 150 registered Web Fetch task IDs: 100 web cases and 10 -each for document, image, video, challenge and dynamic delivery. Every method +The original published cohort contained 150 registered Web Fetch task IDs: 100 web +cases and 10 each for document, image, video, challenge and dynamic delivery. +A new cohort takes its participants, task families and counts from its frozen +dataset and current-report profile; it does not inherit the original matrix. Every method receives the same URL, task rule and end-to-end deadline and is judged by the shared `score_attempt` path. Implementations are deliberately not forced through one extractor: the benchmark asks each method to use its fastest documented or @@ -18,13 +20,36 @@ in readable text; direct-file tasks end in verified original bytes. ## Runtime and route -| Method | Deployment | Measured route | +| Method | Deployment | Task-native route | |---|---|---| -| Kitesurf | Cloudflare account Browser Run | `POST /browser-rendering/markdown?browser=kitesurf` | -| Lightpanda | local CLI | `fetch --dump markdown` | -| Obscura | local CLI, standard mode | `fetch --dump markdown` | -| Moli 1.1.1 | local CLI | ordinary pages: `fetch --dump markdown`; direct documents/media: bare `fetch URL`, then timed text extraction only for document-body tasks | -| Chromium | local Playwright/CDP | rendered DOM HTML → `node-html-markdown` 2.0.0 | +| Kitesurf | Cloudflare account Browser Run | page text: `/markdown?browser=kitesurf`; resource URLs: Kitesurf CDP DOM reads; document text: native `/markdown`; original files: the Kitesurf CDP endpoint | +| Lightpanda | local browser | page text: `fetch --dump markdown --json`; original files and shared DOM projections: native CDP | +| Obscura | local browser, standard mode | page text: pinned upstream Markdown expression with the final URL; original files: `fetch --dump original`; synchronous DOM projections: `--eval`; asynchronous embedded documents: native CDP | +| Moli | local CLI | page text: native Markdown; final URL comes only from unambiguous same-call official diagnostics; original inline files: native CDP response bytes; attachment events: the same pinned Moli CLI raw response; resource URLs, media metadata and embedded documents: shared DOM expressions | +| Chromium | local Playwright/CDP | page text: rendered DOM → `node-html-markdown`; original files: navigation response or native download; resource URLs, metadata and embedded documents: shared DOM expressions | + +Choose the route from the input URL and declared deliverable before calling the +browser. Ordinary, dynamic, popular-site and risk-control text remain page tasks. +Image/video cases distinguish resource URLs, original file bytes and page metadata; +a Markdown response cannot substitute for a required original file. + +文档正文必须由所选浏览器的原生内容接口返回。直接 PDF/DOCX 使用各自原生正文 +接口；不支持时保存真实失败，不交给外部 pypdf、DOCX XML 或 OCR 补正文。 +嵌入文档页面可通过该浏览器的 CDP 或脚本接口读取已经加载的 PDF 查看器文字， +或页面可见文字；不得额外下载嵌入 PDF 后解析。Chromium 的例外只覆盖网页 HTML +到 Markdown 的标准转换。 + +图片与视频原始文件保留浏览器响应或下载路线。Moli 附件事件可调用同一固定二进制 +的原始附件接口，仍受同一截止时间约束；原文件字节不会被外部转换成正文成绩。 +资源地址读取已经加载的 DOM 与同源子页面，不额外抓取模块源码挖掘链接。 +最终网址只作辅助信息；正文验收以目标内容、必需字段和拦截页规则为准。 + +Local HTTP downloading and Workers AI conversion are separate methods, not +Kitesurf results. Every Kitesurf Quick Action and CDP connection must explicitly +select `browser=kitesurf`; omitting it selects Cloudflare's Chromium backend. +The current report's native-task route gate rejects these substitutions and stale +one-provider projections while retaining genuine native failures. + Chromium and CDP do not expose a native Markdown result. Chromium Reader Mode produces simplified HTML and does not cover this benchmark's full output @@ -41,7 +66,10 @@ hypothetical local deployment. The routes have different completion controls because those controls belong to the tested interfaces: Kitesurf uses `networkidle2`, Chromium uses Playwright -`domcontentloaded`, and each CLI uses its own documented completion behavior. +`load` by default, and each CLI uses its own documented completion behavior. +Explicit completion contracts retain their selected navigation condition; Chromium +records `navigation_wait_until` in each call rather than inferring it from the +current adapter. Native execution does not select task-specific readiness hints. No method receives task answers, task-specific selectors, clicks, scrolling, personal sessions, stealth fallback or model post-processing. The original published matrix used one provider call per task and did not retry; later @@ -56,8 +84,7 @@ client-observed end-to-end deadline. It does not claim identical internals. Forcing native Markdown methods through HTML conversion would replace their product advantage with benchmark code and answer a less useful question. -Kitesurf is remote while the other six methods are local (covering four local -implementation families). Its latency includes network +Kitesurf is remote while the registered local browser configurations run locally. Its latency includes network round trips, Cloudflare queueing and remote execution; local measurements include local process startup and machine contention. This is a cold-call client latency comparison, not an engine-only CPU/RSS benchmark. The original @@ -165,11 +192,11 @@ filenames rather than claiming the repository's default inputs. ### Capture, freeze and check references -The reference lifecycle in [README.md](README.md) is mandatory. Independently -capture ordinary pages, separately observe browser-dependent complete scopes, -freeze the references, execute, then postcheck: +Use the certified wrapper below. Existing frozen task and answer bytes remain +unchanged. Only cases declaring `reference_mode: live_exact` need the live +reference lifecycle in [README.md](README.md). -Reference acquisition follows `source_verification.observation_route` (`http` +For those live-reference cases, acquisition follows `source_verification.observation_route` (`http` or `browser`). Existing frozen inputs that name independent visible/rendered browser DOM acquisition retain that browser requirement. Task identifiers do not select the acquisition route. A browser-bound reference needs independent @@ -184,41 +211,50 @@ manifest; response artifacts remain in the physical run for assembly. A failed reference check remains failed when the command resumes. ```sh -uv run --frozen python scripts/freeze_headless_references.py --captures CAPTURES --output FROZEN --browser-observations PRE.json -uv run --frozen python scripts/run_headless_benchmark.py --dataset FROZEN/dataset.json --output RUN --workers 5 -uv run --frozen python scripts/postcheck_headless.py --run-dir RUN --captures POST --browser-observations POST.json -uv run --frozen python scripts/certify_headless_cases.py --run-dir RUN --observations POST/observations.json --output CERTIFICATES -uv run --frozen python scripts/review_headless_results.py --evidence EVIDENCE --work REVIEW --batch-cases 5 --workers 4 --model EXACT_AVAILABLE_MODEL --install +uv run --frozen python scripts/run_certified_benchmark.py --dataset DATASET --output RUN --work-root WORK --providers METHODS --case-ids IDS --provider-config PROVIDERS --workers 1 --browser-binary /absolute/path/to/chromium ``` ### Certify a new run -For new `run_benchmark.py` cohorts, use the fail-closed wrapper so an aggregate -run cannot be mistaken for publishable evidence before its worker manifests -and top-level execution file carry verified postchecks: - -```sh -uv run --frozen python scripts/run_certified_benchmark.py --dataset evaluation/dataset.json --output RUN --work-root WORK --providers METHOD --case-ids ID1,ID2 --browser-binary /absolute/path/to/chromium -``` +An unchanged compiled acceptance-contract dataset is the execution authority. +Freeze its task and rule bytes; do not recover historical qualification files or +repeat admission checks. Live exact-reference cases retain pre-call capture, +freeze and post-call verification under [README.md](README.md). Source +qualification belongs to new tasks or changes to references/acceptance rules. +Use bounded cohorts when a complete serial run would exceed the live-reference +window. The wrapper verifies worker manifests before finalizing the aggregate. -Use `--provider-config /absolute/path/providers.json` to select one provider -configuration for the entire requested scope. The certified wrapper forwards -it to the shared runner while retaining all reference checks. Without this -option, the repository's existing task-group configurations apply. Diagnostic -`scripts/run_benchmark.py` runs support the same option and remain uncertified. - -After selecting and freezing the resource cohort, run it separately and keep the -same postcheck and independent all-attempt review gates: +Select the resource tasks through the existing provider-blind selector before +resource calls. Set the dataset comparison kind to +`headless_client_resource_profile`; its provider configuration binds each local +engine, image, executable, 50 ms sample interval, two CPUs and 2 GiB memory. +Run it through the same certified wrapper with one worker, then package: ```sh make resource-image -uv run --frozen python scripts/run_headless_benchmark.py --dataset RESOURCE/dataset.json --output RESOURCE_RUN --workers 1 --resource-sample-interval-ms 50 -uv run --frozen python scripts/postcheck_headless.py --run-dir RESOURCE_RUN --captures RESOURCE_POST --browser-observations RESOURCE_POST.json -uv run --frozen python scripts/certify_headless_cases.py --run-dir RESOURCE_RUN --observations RESOURCE_POST/observations.json --output RESOURCE_CERTIFICATES -uv run --frozen python scripts/review_headless_results.py --evidence RESOURCE_RUN --work RESOURCE_REVIEW --batch-cases 1 --workers 4 --model EXACT_AVAILABLE_MODEL --scope all --install +uv run --frozen python scripts/run_certified_benchmark.py --dataset RESOURCE/dataset.json --output RESOURCE_RUN --work-root RESOURCE_WORK --providers LOCAL_METHODS --case-ids IDS --provider-config RESOURCE/providers.json --workers 1 uv run --frozen python scripts/package_headless_resources.py --run RESOURCE_RUN ``` +The image recipe accepts version-bound build arguments. Set `MOLI_VERSION` +and `MOLI_SHA_ARM64` / `MOLI_SHA_AMD64` together; its executable is installed as +`/usr/local/bin/moli-VERSION`. Lightpanda uses the existing per-architecture +`LIGHTPANDA_URL_*` / `LIGHTPANDA_SHA_*` arguments, and Obscura uses +`OBSCURA_VERSION` with its per-architecture digests. For official Chrome for +Testing, supply `CHROMIUM_URL_ARM64` / `CHROMIUM_SHA_ARM64` or +`CHROMIUM_URL_AMD64` / `CHROMIUM_SHA_AMD64`; a URL without a digest fails the +build. Select the archive's executable under `/opt/chrome-for-testing/` through +`resource_binary`. Omitting these Chrome arguments keeps Debian Chromium. +The recipe defaults preserve the original image configuration; a new cohort +must supply its selected release inputs and verify actual versions and hashes +inside the built image before any measured call. + +Independent result review is opt-in for both matrices. Preserve the declared +review policy; start failure review only after an explicit user request, and +review passes or the complete matrix only after a separate full-review request. +Packaging retains the exact reviewed/total scope. Do not use missing independent +review as an execution blocker when the frozen protocol does not require it. + Build the resource image once per declared image definition or browser-version change. Benchmark runs reuse `lex-web-fetch-resource:local` by image ID and do not rebuild it; repeated `make resource-image` calls reuse Docker layers. A @@ -235,8 +271,8 @@ required metrics, image ID and executable hashes. Do not substitute the Headless resource packaging fails if CPU, memory or network evidence is missing; the report release check requires grouped winners whenever the cohort is present. -The resource-mode default is the four local methods: Lightpanda, Obscura, Moli -1.1.1, and Chromium. Kitesurf remains in the correctness and latency report but +The resource cohort uses the four local browser families: Lightpanda, Obscura, +Moli and Chromium. Each new run pins its actual versions and image identity. Kitesurf remains in the correctness and latency report but is excluded from the resource table because its remote browser CPU, memory, and upstream network traffic are not observable from the client. @@ -328,3 +364,5 @@ The preserved public destination for this comparison is the separate app-domain Sites project `UUAhhQMnrGwm`. The current 10-method report is the separate project declared by `evaluation/current-report.json`; never use a headless receipt or evidence bundle to update it. + +固定参考答案默认保持冻结。`source_verification.reference_mode: live_exact` 是实时参考核验的显式声明；旧来源记录和缺少新版验收契约均不构成该声明。普通重跑不得据此重新抓取或更新答案。 diff --git a/harness/web_fetch/README.md b/harness/web_fetch/README.md index 532ad7c54..92c1ea201 100644 --- a/harness/web_fetch/README.md +++ b/harness/web_fetch/README.md @@ -135,12 +135,21 @@ checks where applicable, and one authoritative task predicate. Live execution an offline rescoring use the same entry point. Case contracts define the differences; provider identity never selects a more lenient evaluator. -Live stable-contract cohorts use `scripts/run_certified_benchmark.py --diagnostic`. -The stable-rule path does not qualify the current site's solvability; the generic -entry therefore rejects such a cohort before external work unless its diagnostic -state is explicit. This does not waive any scoring requirement. For a declared -native-completion experiment, add `--retrieval-mode native`: it preserves all -acceptance rules and rejects existing retrieval hints rather than dropping them. +Reruns of an unchanged compiled stable-contract dataset use +`scripts/run_certified_benchmark.py` normally: freeze existing task and acceptance +rule bytes without recovering historical qualification files or rechecking +admission. Source qualification is required when adding tasks or changing +references or acceptance rules. Live exact-reference cases retain their pre/post +reference checks. Use `--diagnostic` only when a diagnostic run is intended; its +outputs cannot be upgraded to certified evidence. For declared native completion, +add `--retrieval-mode native`: it preserves acceptance rules and rejects existing +selector-guided retrieval hints rather than dropping them. +Dynamic task families stay `dynamic`: the explicit +`retrieval_hints: {"version": 3, "mode": "native"}` contract selects each +provider's ordinary completion without injecting selectors, extraction hints or +extra waits. It accepts no other fields. Selecting native mode again preserves +this marker unchanged; the runner validates it through the same strict parser. +Version 1/2 selector-guided contracts keep their existing meaning. The diagnostic report accepts a `webfetch_diagnostic_inputs/1` manifest naming complete runs, independently captured whole-page observations and hash-bound per-case reviews. The supported builder adapts these inputs, preserves scored @@ -389,7 +398,8 @@ tasks use one `--dump json --trace-network` call and convert its final DOM to Markdown within the same deadline while retaining only a public-safe trace summary. Dynamic, resource-page, and embedded-document tasks use one bounded `--eval` call for -their declared rendered output. Direct file tasks use bare `moli fetch URL`, +their declared rendered output. Resource and document-page expressions are +shared with the other browsers; resource extraction makes no additional requests. Direct file tasks use bare `moli fetch URL`, without `--dump` or page-wait flags, so the evaluator can receive and verify original bytes. No alternate provider or output-format fallback is used after a failed call. The seven-method cohort's direct PDF and DOCX body route is @@ -695,14 +705,15 @@ logs, requests or bare URLs. `uniform_native_parameters: true` selects the fixed-native body/structure policy for web and dynamic tasks. Choose documented native extraction options once per method (for example Tavily advanced, Cloudflare networkidle2, Crawl4AI iframe/overlay options, native freshness and capacity options). The caller deadline is 45 seconds; product connection, navigation and extraction timers retain their native defaults. HTTP-client deadlines and worker cancellation enforce collection time, not product-side budgets. CowAgent globals are never patched. Per-case retrieval hints and auxiliary document/media/challenge routes are rejected before invocation in this mode. Cross-provider fallback, injected extraction algorithms and answer synthesis are prohibited. Caller failure retries follow the shared [retry policy](#retries); historical cohorts retain their frozen retry budgets. Keep the native response envelope and its declared answer; the product may still impose its own limits or internal retries. Prepare a new frozen invocation profile after a policy change; preserve previous measurements and do not infer revised scores without calls. -### Main-document identity on declared navigation tasks -When a body task declares `source_benchmark.risk_control.navigation_contract`, -Moli uses the existing `fetch --dump json` final-DOM route, without network tracing, -except in fixed-native mode, which preserves the same native Markdown route for every case. -The same response binds `final_url`, HTTP status and HTML; the existing local -Markdown conversion stays inside the call deadline. It adds no second fetch. -Ordinary body tasks without that requirement retain native Markdown output. +### 原生内容接口与评测边界 + +给定网址的正文任务验收返回的目标内容，不要求接口返回最终网址或 HTML 语言标签。 +最终网址、HTTP 状态和语言标签保留为辅助诊断；缺失这些字段本身不扣分。 +登录页、验证页、错误页、空正文、缺少题目要求的文字或字段仍然失败。 +验收规则对所有选手一致，禁止从标准答案补正文或推断缺失输出。 + +### Main-document identity on declared navigation tasks Legacy debug GET completion rows do not bind redirect request owners or Location headers. Their order, a challenge query hint, or the first completion after a @@ -815,3 +826,40 @@ physical-call output, uses the final declared response, verifies hashes and exposes the directory only after the entire matrix passes. Unbound review receipts cannot authorize a new assembly. Existing published reports retain their frozen evidence and are restored from their pinned release. +按任务类型预选官方支持的正文、资源或原始文件接口。JSON 包装中的原生 Markdown +可以解包；不能把 JSON DOM、HTML 或下载的 PDF/DOCX 另行转换成选手的正文成绩。 +Chromium 没有原生 Markdown，网页 HTML 使用固定版本的 node-html-markdown；此例外 +不授权外部文档解析器。浏览器内已经加载的 PDF 查看器可通过原生脚本接口读取文字， +不得下载嵌入文件后交给外部解析器。图片、视频的原始文件交付只验收字节，不冒充正文提取。 + +Lightpanda 使用 `--dump markdown --json` 的原生正文；Obscura 使用其固定官方 +`HTML_TO_MARKDOWN_JS`，脚本哈希必须与所测版本绑定；Kitesurf 使用显式指定该浏览器 +的官方 `/markdown` 或 CDP 接口。Moli 正文使用原生 Markdown，资源或嵌入文档可用 +原生脚本/CDP 读取对应输出；禁止失败后切换为 HTML 或 JSON 再替它加工正文。 + +`output_mode_for_case` 负责路线选择，`browser_task_interface_matrix` 使用同一选择器 +生成每类任务与每种浏览器的接口表。`require_native_task_routes` 检查实测路线与外部 +转换标记；当前报告配置必须启用该门禁，检查失败时不得汇总或发布。 +改变调用接口应冻结完整受影响范围、保留旧调用并通过认证入口补跑；只改共享评分时 +离线重算所有选手，不重复请求网站。合成正反测试必须同时覆盖正确正文缺少网址和错误正文。 + +方法名与浏览器引擎分开登记在 `configs/providers.headless.json`。同一浏览器的发布版和分支构建分别计分， +原生路线检查、接口表和官方能力说明共用该登记的引擎；未知方法不能跳过检查。 +报告分别统计浏览器引擎和执行配置，同一引擎的多个版本不算成多种浏览器。 +资源属性中的单个 URL 保留完整地址；只有 `srcset` 按候选及描述符解析，URL 中间的逗号不能拆开。 +分支构建在调用前固定来源提交、基线提交、构建环境和二进制 SHA-256。新增配置只调用完整固定题库， +旧配置的原始结果保持不变。不同日期的批次用于展示各自任务完成情况；代码带来的耗时变化须另按同窗配对协议测量。 + +### 已编译评测集的重跑 + +`scripts/run_certified_benchmark.py` 直接使用冻结的题目与验收规则。已有固定答案或稳定规则的题目 +不重新审核入选资格，不读取历史资格调查原文，也不因旧归档缺失而改成诊断结果。 +新增题目、修改参考或验收规则时，仍须通过相应构建与修订流程核验来源。 +固定答案不会因为有来源核验信息、旧核验日期或缺少新版验收契约而自动变成实时参考。 +只有题目明确声明 `source_verification.reference_mode: live_exact`，才执行调用前冻结和调用后核验。 +没有该声明时，执行器保持题库中的答案和评分规则不变。明确修订参考答案仍走独立的参考修订入口。 + +已授权的错误修正应自动完成：先查明共享原因并添加必要的正反测试，再冻结受影响的完整调用范围，经认证入口补跑。 +接口、版本或执行错误需要新调用；评分或展示错误只重算或重建。不能为提高分数挑选失败题重试， +不能让无关题目重复调用，也不能把已有题库的重跑变成重新收集资格证明。保留旧结果及修订影响。 +评测配置错误或适配器在调用前拦截属于执行问题，不能当作浏览器能力失败发布；原生路线门禁拒绝 `configuration` 结果。供应商实际调用返回的错误仍按原规则保留。 diff --git a/harness/web_fetch/assembly.py b/harness/web_fetch/assembly.py index 6fc1b92a3..a001c2810 100644 --- a/harness/web_fetch/assembly.py +++ b/harness/web_fetch/assembly.py @@ -668,8 +668,7 @@ def _assemble(run, certificates, replacements, output, additions=None, provider_ for root in additions: new_dataset=json.loads((root/'dataset.json').read_text()) rows=read_records(root) - manifest=json.loads((root/'manifest.json').read_text()) - require_verified_reference_checks(manifest,cases=new_dataset,records=rows,source=root) + addition_batches=read_verified_batches(root,new_dataset,rows) incoming={c['id']:c for c in new_dataset['cases']} if set(incoming)&set(cases):raise ValueError('Addition overlaps existing cases') planned={(c['id'],p) for c in incoming.values() for p in new_dataset['coverage_plan'][c.get('track','web')]} @@ -682,9 +681,9 @@ def _assemble(run, certificates, replacements, output, additions=None, provider_ dataset['coverage_plan'].setdefault(case.get('track','web'),new_dataset['coverage_plan'][case.get('track','web')]) expected|=planned selected.extend((r,root) for r in rows) - batches.append(project_execution_manifest(manifest));evidence.append(root) - reference_evidence.append((root, [manifest])) - audit.extend({'case_id':cid,'selection_reason':'missing case completed as a full five-method addition; no score-based selection','excluded_attempts':[]} for cid in incoming) + batches.extend(project_execution_manifest(batch) for batch in addition_batches);evidence.append(root) + reference_evidence.append((root, addition_batches)) + audit.extend({'case_id':cid,'selection_reason':'missing case completed as the full declared provider cohort; no score-based selection','excluded_attempts':[]} for cid in incoming) pair_replacements = {} for root in provider_replacements: new_dataset=json.loads((root/'dataset.json').read_text()) @@ -774,8 +773,6 @@ def _assemble(run, certificates, replacements, output, additions=None, provider_ validate_report_ids(dataset,records) if len(records)!=len(expected) or {(r['case_id'],r['provider']) for r in records}!=expected: raise ValueError('Assembled coverage differs from the complete cohort') - if added_providers: - dataset.setdefault('comparison',{})['require_independent_result_review']=True failed=[cid for batch in batches for cid,row in batch.get('reference_checks',{}).get('postcheck',{}).get('cases',{}).items() if row['status']!='verified'] dataset['comparison']['publication_state']='blocked_reference_checks' if failed else 'verified' dataset['comparison']['unverified_reference_cases']=sorted(set(failed)) diff --git a/harness/web_fetch/certification.py b/harness/web_fetch/certification.py index d8aa38bdb..50f1ad054 100644 --- a/harness/web_fetch/certification.py +++ b/harness/web_fetch/certification.py @@ -188,20 +188,9 @@ def main(argv: list[str] | None = None) -> None: if args.output.exists() or args.work_root.exists(): parser.error("Output and work root must both be new immutable paths") - # Stable rules deliberately bypass exact-reference lifecycle checks. A - # completed wrapper therefore cannot certify that their old labels or - # record requirements are still present on the live site. - from harness.web_fetch.reference_lifecycle import source_unqualified_contract_cases - - unqualified = source_unqualified_contract_cases( - case for case in dataset["cases"] if case["id"] in case_ids - ) - if unqualified and not args.diagnostic: - parser.error( - "Stable live-page contracts have no fresh source qualification; " - "use --diagnostic to retain uncertified evidence, not to waive scoring: " - + ", ".join(unqualified) - ) + # Frozen answers are already evaluation inputs. Historical admission + # evidence does not turn a version rerun into live source qualification. + unqualified = [] from harness.web_fetch.retrieval_hints import native_fetch_dataset, parse_retrieval_hints @@ -227,10 +216,12 @@ def main(argv: list[str] | None = None) -> None: parse_retrieval_hints( case.get("track", "web"), case.get("retrieval_hints"), identifier=case["id"] ) - if args.retrieval_mode == "native" or args.diagnostic: - args.work_root.mkdir(parents=True) - args.dataset = args.work_root / "input-dataset.json" - args.dataset.write_text(json.dumps(dataset, ensure_ascii=False, indent=2) + "\n") + source_ids = ",".join(case["id"] for case in dataset["cases"] + if case["id"] in case_ids and requires_fresh_reference(case)) + args.work_root.mkdir(parents=True) + dataset.setdefault('comparison', {})['source_reference_policy'] = 'frozen_benchmark_inputs_with_explicit_live_exact_checks' + args.dataset = args.work_root / "input-dataset.json" + args.dataset.write_text(json.dumps(dataset, ensure_ascii=False, indent=2) + "\n") selected = ",".join(case_ids) pre_captures = args.work_root / "pre-captures" @@ -239,17 +230,12 @@ def main(argv: list[str] | None = None) -> None: post_captures = args.work_root / "post-captures" post_browser = args.work_root / "post-browser.json" - run( - source_capture.main, - [ - str(args.dataset), - str(pre_captures), - "--workers", - str(args.capture_workers), - "--only", - selected, - ] - ) + if source_ids: + run(source_capture.main, [str(args.dataset), str(pre_captures), + "--workers", str(args.capture_workers), "--only", source_ids]) + else: + pre_captures.mkdir(parents=True) + (pre_captures / 'audit.json').write_text('[]\n') try: pre_browser_cases = browser_observation_cases(dataset, pre_captures, case_ids) except (KeyError, TypeError, ValueError) as error: @@ -325,17 +311,12 @@ def main(argv: list[str] | None = None) -> None: # choose browser fallbacks from that post-call evidence. Reusing the # pre-call browser inventory is unsafe: an endpoint can become unavailable # during a long cohort even though its frozen pre-capture was usable. - run( - source_capture.main, - [ - str(args.output / "dataset.json"), - str(post_captures), - "--workers", - str(args.capture_workers), - "--only", - selected, - ] - ) + if source_ids: + run(source_capture.main, [str(args.output / "dataset.json"), str(post_captures), + "--workers", str(args.capture_workers), "--only", source_ids]) + else: + post_captures.mkdir(parents=True) + (post_captures / 'audit.json').write_text('[]\n') post_dataset = json.loads((args.output / "dataset.json").read_text(encoding="utf-8")) try: post_browser_cases = browser_observation_cases(post_dataset, post_captures, case_ids) @@ -368,17 +349,9 @@ def main(argv: list[str] | None = None) -> None: except ValueError as error: parser.error(str(error)) post_browser_args = ["--browser-observations", str(post_browser)] - run( - reference_freeze.postcheck_main, - [ - "--run-dir", - str(args.output), - "--captures", - str(post_captures), - "--resume", - *post_browser_args, - ] - ) + if source_ids: + run(reference_freeze.postcheck_main, ["--run-dir", str(args.output), + "--captures", str(post_captures), "--resume", *post_browser_args]) observations = post_captures / "observations.json" if not observations.exists(): observations.parent.mkdir(parents=True, exist_ok=True) diff --git a/harness/web_fetch/evidence.py b/harness/web_fetch/evidence.py index 589b53dbe..2c83e7215 100644 --- a/harness/web_fetch/evidence.py +++ b/harness/web_fetch/evidence.py @@ -35,6 +35,92 @@ validate_report_ids, ) +def recover_completed_batch(source: Path, manifest: dict, records: list[dict]) -> dict: + """Bind all flushed calls from an interrupted fixed-answer physical batch. + + The source manifest stays unchanged. The returned boundary describes only + saved completed calls, and explicitly retains the original unfinished state. + It never supplies missing live-reference certification or chooses scores. + """ + from datetime import datetime + from .artifacts import public_call_log_binding, verify_public_call_log + from .reference_lifecycle import requires_fresh_reference + + if (manifest.get("schema_version") != 1 or manifest.get("tool") != "web_fetch" + or manifest.get("completed_at") or manifest.get("attempt_ids") is not None + or not records): + raise ValueError("Recovery requires an interrupted versioned physical batch") + parameters = manifest.get("parameters", {}) + if parameters.get("runs") != 1 or parameters.get("max_failure_retries") != 0: + raise ValueError("Interrupted recovery requires one call per logical result") + + def verified_input(path, digest): + path = Path(path) + if path.is_symlink() or not path.is_file(): + raise ValueError("Recovery input is missing or unsafe") + raw = path.read_bytes() + if not digest or hashlib.sha256(raw).hexdigest() != digest: + raise ValueError("Recovery input hash changed") + return raw + + input_raw = verified_input(manifest["dataset"]["source"], manifest.get("dataset_sha256")) + verified_input(manifest["provider_config_source"], manifest.get("provider_config_sha256")) + dataset = json.loads((source / "dataset.json").read_bytes()) + original_cases = {case["id"]: case for case in json.loads(input_raw)["cases"]} + cases = {case["id"]: case for case in dataset["cases"]} + if (len(cases) != len(dataset["cases"]) + or any(original_cases.get(identifier) != case for identifier, case in cases.items())): + raise ValueError("Recovery task differs from frozen pre-call input") + providers = {provider["name"]: provider for provider in manifest["providers"]} + lower = datetime.fromisoformat(manifest["started_at"]) + ends, identities = [], set() + for row in records: + key = (row["provider"], row["case_id"]) + if (key in identities or key[0] not in providers or key[1] not in cases + or row["run"] != 1 or row["provider_type"] != providers[key[0]]["type"]): + raise ValueError("Interrupted result has duplicate or undeclared identity") + identities.add(key) + if requires_fresh_reference(cases[key[1]]): + raise ValueError("Interrupted live-reference calls require their full certificate") + started, ended = (datetime.fromisoformat(row[field]) + for field in ("started_at", "completed_at")) + binding = public_call_log_binding(row.get("provider_metadata", {}).get("call_log")) + if not binding: + raise ValueError("Interrupted result has no completed physical call log") + log = json.loads(verify_public_call_log(source, binding).read_bytes()) + if any(log["identity"].get(field) != row[field] + for field in ("attempt_id", "provider", "provider_type", "case_id", "run")): + raise ValueError("Interrupted result does not match its physical call identity") + timing = log["timing"] + if (log["identity"]["physical_attempt"] != 1 + or not lower <= started <= datetime.fromisoformat(timing["started_at"]) + <= datetime.fromisoformat(timing["completed_at"]) <= ended + or timing["elapsed_ms"] != row["elapsed_ms"]): + raise ValueError("Interrupted result does not match its completed call timing") + artifact = contained_path(source, row["artifact_path"]) + digest = row["metrics"]["content_sha256"] + if (artifact.is_symlink() or not artifact.is_file() + or hashlib.sha256(artifact.read_bytes()).hexdigest() != digest + or log["output"]["path"] != row["artifact_path"] + or log["output"]["sha256"] != digest): + raise ValueError("Interrupted result output differs from completed physical call") + verify_record_artifacts(source, row) + ends.append(ended) + batch = deepcopy(manifest) + batch.update(completed_at=max(ends).isoformat(), attempt_count=len(records), + attempt_ids=[row["attempt_id"] for row in records]) + batch["recovery"] = { + "kind": "all-saved-completed-calls-from-interrupted-batch", + "original_batch_completed": False, + "source_manifest_sha256": hashlib.sha256((source / "manifest.json").read_bytes()).hexdigest(), + "source_results_sha256": hashlib.sha256((source / "results.jsonl").read_bytes()).hexdigest(), + "source_dataset_sha256": hashlib.sha256((source / "dataset.json").read_bytes()).hexdigest(), + "pending_logical_results": len(cases) * len(providers) - len(records), + "selection": "every persisted completed call, regardless of pass or fail", + "boundary": "last saved call completion; original full batch did not complete", + } + return batch + def add_provider_run(canonical: Path, incoming: Path, provider: str, implementation=None, *, track=None): for root in (canonical, incoming): @@ -258,19 +344,23 @@ def _project_batches( def _project_provider_run(source: Path, output: Path, provider: str, reference_root: Path | None = None, - case_ids: set[str] | None = None) -> None: + case_ids: set[str] | None = None, *, retain_completed: bool = False) -> None: source, output = source.resolve(), output.resolve() reference_root = (reference_root or source).resolve() if output.exists(): raise ValueError("Projection destination exists: " + str(output)) dataset = json.loads((source / "dataset.json").read_text()) + all_records = _records(source) + if retain_completed: + if case_ids is not None: + raise ValueError("Interrupted recovery cannot select a case subset") + case_ids = {row["case_id"] for row in all_records if row["provider"] == provider} if case_ids is not None: known = {case["id"] for case in dataset["cases"]} if not case_ids or case_ids - known: raise ValueError("Projection case IDs are empty or unknown") dataset["cases"] = [case for case in dataset["cases"] if case["id"] in case_ids] cases = {case["id"]: case for case in dataset["cases"]} - all_records = _records(source) selected = [row for row in all_records if row["provider"] == provider and row["case_id"] in cases] expected = set(cases) @@ -282,6 +372,10 @@ def _project_provider_run(source: Path, output: Path, provider: str, reference_r source_batches, source_kind, source_document_bytes = _source_batches(source) source_document_sha256 = hashlib.sha256(source_document_bytes).hexdigest() + if retain_completed: + if source_kind != "manifest" or len(source_batches) != 1: + raise ValueError("Interrupted recovery requires one physical source manifest") + source_batches = [recover_completed_batch(source, source_batches[0], all_records)] dataset["coverage_plan"] = { track: [provider] for track, providers in dataset["coverage_plan"].items() if provider in providers @@ -340,7 +434,7 @@ def _project_provider_run(source: Path, output: Path, provider: str, reference_r def project_provider_run(source: Path, output: Path, provider: str, reference_root: Path | None = None, - case_ids: set[str] | None = None) -> None: + case_ids: set[str] | None = None, *, retain_completed: bool = False) -> None: """Expose a new projection only after its complete evidence passes validation.""" for root in (source, reference_root or source): reject_evidence_symlinks(root) @@ -353,7 +447,8 @@ def project_provider_run(source: Path, output: Path, provider: str, output.parent.mkdir(parents=True, exist_ok=True) with tempfile.TemporaryDirectory(prefix='.' + output.name + '-', dir=output.parent) as work: staging = Path(work) / 'evidence' - _project_provider_run(source, staging, provider, reference_root, case_ids) + _project_provider_run(source, staging, provider, reference_root, case_ids, + retain_completed=retain_completed) staging.replace(output) @@ -362,6 +457,8 @@ def projection_main(argv=None) -> None: parser.add_argument("--source", type=Path, required=True) parser.add_argument("--output", type=Path, required=True) parser.add_argument("--provider", required=True) + parser.add_argument("--retain-completed", action="store_true", + help="Retain every saved completed call from an interrupted fixed-answer batch") parser.add_argument("--case-ids", help="Optional comma-separated explicit subset") parser.add_argument("--reference-root", type=Path, help="Root containing frozen sources/list-references when the live run omits them") @@ -372,7 +469,8 @@ def projection_main(argv=None) -> None: if any(not value for value in values) or len(set(values)) != len(values): parser.error('Case IDs must be nonempty and unique') case_ids = set(values) - project_provider_run(args.source, args.output, args.provider, args.reference_root, case_ids) + project_provider_run(args.source, args.output, args.provider, args.reference_root, case_ids, + retain_completed=args.retain_completed) print(f"Projected {args.provider}: {args.output}") diff --git a/harness/web_fetch/methods.py b/harness/web_fetch/methods.py index 7c48b4149..53fa0f6c3 100644 --- a/harness/web_fetch/methods.py +++ b/harness/web_fetch/methods.py @@ -1,9 +1,38 @@ from __future__ import annotations +import json +import importlib.metadata import re +from functools import lru_cache +from pathlib import Path from typing import Any +@lru_cache(maxsize=1) +def browser_method_registry(): + """Use the execution registry for release and source-build participants.""" + path = Path(__file__).resolve().parents[2] / "configs/providers.headless.json" + if not path.is_file(): + from harness.distribution import installed_data_path + try: + path = installed_data_path("share/browser-harness-eval/providers.headless.json") + except importlib.metadata.PackageNotFoundError: + return {} + except ValueError as error: + if str(error).startswith('Distribution must declare one data file:'): + return {} + raise + return json.loads(path.read_text())["providers"] + + +def native_browser_engine(name): + return browser_method_registry().get(name, {}).get("engine") + + +def browser_method_label(name): + return browser_method_registry().get(name, {}).get("label", name) + + METHOD_REGISTRY: dict[str, dict[str, Any]] = { "http": {"endpoint":"HTTP GET","contract":"已知 URL → HTML 文本或文件任务的原始字节","output":"HTML 提取文本；文件任务保留响应字节","freshness":"每次独立请求，不执行页面 JavaScript","leaderboard":"独立 HTTP 对照"}, "headless_html": {"endpoint":"browser HTML → node-html-markdown 2.0.0","contract":"已知 URL → DOM HTML → Markdown","output":"Markdown；独立保存 HTML 和诊断","freshness":"每次新进程/会话；domcontentloaded；45 秒外层预算","leaderboard":"六类任务分别验收","report":{"observed_routes":True}}, @@ -25,6 +54,13 @@ } +def is_browser_comparison(records: list[dict[str, Any]]) -> bool: + """Recognize browser cohorts independently of their task-specific adapters.""" + types = {row.get("provider_type") for row in records} + return bool(types & {"headless_html", "headless_markdown"}) and types <= { + "headless_html", "headless_markdown", "moli_webfetch"} + + def method_info(provider_type: str) -> dict[str, Any]: return METHOD_REGISTRY.get(provider_type, {"endpoint":"未记录","contract":"未知","output":"未知","freshness":"未知","leaderboard":"未分类"}) @@ -34,7 +70,7 @@ def method_info(provider_type: str) -> dict[str, Any]: "lightpanda_native_markdown": "Lightpanda CLI 直接返回 Markdown", "obscura_native_markdown": "Obscura CLI 直接返回 Markdown", "moli_native_markdown": "Moli CLI 直接返回 Markdown", - "node-html-markdown": "Chromium 渲染 DOM → node-html-markdown 2.0.0", + "node-html-markdown": "浏览器渲染 DOM → node-html-markdown 2.0.0", } diff --git a/harness/web_fetch/metrics.py b/harness/web_fetch/metrics.py index 2b5ddaa30..40c790173 100644 --- a/harness/web_fetch/metrics.py +++ b/harness/web_fetch/metrics.py @@ -325,9 +325,6 @@ def task_passed(ok: bool, metrics: dict, task_kind: str | None = None) -> bool: # Scored records carry the case kind so dataset-free summaries use the same # predicate as the report. Explicit case context still takes precedence. task_kind = task_kind if task_kind is not None else metrics.get('task_kind', 'body') - if (metrics.get('page_identity_required') - and metrics.get('page_identity_status') != 'verified'): - return False if (not task_measured(metrics, task_kind) or metrics.get('deadline_overrun') or metrics.get('blocked_page')): return False @@ -347,10 +344,8 @@ def task_passed(ok: bool, metrics: dict, task_kind: str | None = None) -> bool: def task_measured(metrics: dict, task_kind: str | None = None) -> bool: if metrics.get('rule_review_status') == 'unresolved': return False - if (metrics.get('page_identity_required') - and metrics.get('page_identity_status') == 'unavailable' - and metrics.get('request_ok') is not False): - return False + # A completed call missing required page observations is a measured failure. + # Missing benchmark rules remain unmeasured through rule_review_status above. kind = task_kind or metrics.get('task_kind', 'body') return kind != 'file' or isinstance(metrics.get('raw_file_pass'), bool) diff --git a/harness/web_fetch/models.py b/harness/web_fetch/models.py index c37d20128..61bd9559b 100644 --- a/harness/web_fetch/models.py +++ b/harness/web_fetch/models.py @@ -9,8 +9,8 @@ TASK_KINDS = ('body', 'structure', 'resource', 'file') TRACKS = { 'web': '静态网站正文', 'dynamic': '动态网站正文', 'challenge': '热门网站正文', + 'risk_control': '风控网站正文', 'document': '文档正文', 'image': '图片获取', 'video': '视频获取', - 'risk_control': '历史访问受限网址正文', } diff --git a/harness/web_fetch/providers/base.py b/harness/web_fetch/providers/base.py index 1a7c231f0..cf8c0c02e 100644 --- a/harness/web_fetch/providers/base.py +++ b/harness/web_fetch/providers/base.py @@ -1,11 +1,80 @@ from __future__ import annotations from abc import ABC, abstractmethod +import os +import signal +import subprocess +import time from typing import Any from ..models import EvalCase, ProviderResult +def terminate_owned_process_tree(process): + """Stop an owned POSIX session, including detached descendant groups. + + If the process tree cannot be observed, leave the worker alive so the + caller can quarantine unresolved work instead of claiming it stopped. + """ + if process is None or process.poll() is not None: + return True + if os.name != 'posix': + return False + try: + observed = subprocess.run( + ['ps', '-axo', 'pid=,ppid=,pgid='], capture_output=True, + text=True, check=True, timeout=.5, + ) + rows = [tuple(map(int, line.split())) for line in observed.stdout.splitlines()] + if any(len(row) != 3 for row in rows): + return False + except (OSError, ValueError, subprocess.SubprocessError): + return False + parents = {pid: parent for pid, parent, _ in rows} + groups = {pid: group for pid, _, group in rows} + if groups.get(process.pid) != process.pid: + return process.poll() is not None + owned = {process.pid} + while True: + descendants = {pid for pid, parent in parents.items() if parent in owned} + expanded = owned | descendants + if expanded == owned: + break + owned = expanded + owned_groups = {groups[pid] for pid in owned} + if os.getpgrp() in owned_groups: + return False + # Descendants may call setsid (Playwright does this for Chromium). Stop + # their groups first, then the original worker, preserving its diagnostics. + for group in sorted(owned_groups, key=lambda value: value == process.pid): + try: + os.killpg(group, signal.SIGKILL) + except ProcessLookupError: + pass + except OSError: + return False + # SIGKILL delivery is asynchronous. Confirm the owned groups are quiescent + # before releasing the next call; zombies cannot perform physical work. + deadline = time.monotonic() + .5 + while True: + try: + observed = subprocess.run( + ['ps', '-axo', 'pgid=,stat='], capture_output=True, + text=True, check=True, timeout=.5, + ) + survivors = [parts for line in observed.stdout.splitlines() + if len(parts := line.split()) == 2 + and int(parts[0]) in owned_groups + and not parts[1].startswith('Z')] + except (OSError, ValueError, subprocess.SubprocessError): + return False + if not survivors: + return True + if time.monotonic() >= deadline: + return False + time.sleep(.01) + + class Provider(ABC): def __init__(self, name: str, config: dict[str, Any]): self.name = name @@ -17,7 +86,13 @@ def check_native_call(self, case: EvalCase) -> None: if not self.config.get("uniform_native_parameters"): return if case.retrieval_hints: - raise ValueError("fixed native calls prohibit per-case retrieval hints") + from ..retrieval_hints import parse_retrieval_hints + try: + plan = parse_retrieval_hints(case.track, case.retrieval_hints, identifier=case.id) + except ValueError as error: + raise ValueError("fixed native calls prohibit per-case retrieval hints") from error + if plan is not None: + raise ValueError("fixed native calls prohibit per-case retrieval hints") if case.track not in {"web", "dynamic"} or case.task_kind not in {"body", "structure"}: raise ValueError("fixed native calls support webpage body/structure only; auxiliary routes are prohibited") diff --git a/harness/web_fetch/providers/cloudflare.py b/harness/web_fetch/providers/cloudflare.py index 7be3b0861..672a1db3a 100644 --- a/harness/web_fetch/providers/cloudflare.py +++ b/harness/web_fetch/providers/cloudflare.py @@ -325,7 +325,7 @@ def _fetch_payload(self, case: EvalCase, timeout: float, request_body: dict) -> account_id=account_id, http_timeout=http_timeout, ) - if _uses_direct_document_route(case): + if _uses_direct_document_route(case) and self.config.get("browser") != "kitesurf": return self._fetch_direct_document( case, account_id=account_id, @@ -348,12 +348,12 @@ def _fetch_payload(self, case: EvalCase, timeout: float, request_body: dict) -> request_body.get("gotoOptions", {}).pop("timeout", None) request_body.pop("actionTimeout", None) wait_metadata = None + plan = None if case.track == "dynamic" and not native_default_case(case) and "url" in request_body: try: if not math.isfinite(timeout) or timeout <= 0: raise ValueError plan = plan_for_case(case) - assert plan is not None except (KeyError, TypeError, ValueError): return ProviderResult( False, @@ -362,6 +362,7 @@ def _fetch_payload(self, case: EvalCase, timeout: float, request_body: dict) -> error_message="invalid dynamic retrieval_hints v1", ) + if plan is not None: # The Markdown endpoint exposes waitForSelector, but no selector-count # or arbitrary-function wait. Use only independently declared, # answer-free readiness signals and disclose unsupported semantics. @@ -457,7 +458,7 @@ def _fetch_payload(self, case: EvalCase, timeout: float, request_body: dict) -> # Task scoring, not request classification, decides its usefulness. content=content if envelope_success else "", status_code=response.status, - final_url=meta.get("finalUrl") or case.url, + final_url=meta.get("finalUrl"), title=meta.get("title"), error_type=error_type, error_message=error_message, @@ -504,7 +505,8 @@ def _fetch_media_resources( ["video", "video source", "audio", "audio source", "iframe", "a[href]"]) ) response = request( - f"https://api.cloudflare.com/client/v4/accounts/{account_id}/browser-rendering/scrape", + f"https://api.cloudflare.com/client/v4/accounts/{account_id}/browser-rendering/scrape" + + ("?" + urlencode({"browser": self.config["browser"]}) if self.config.get("browser") else ""), method="POST", timeout=http_timeout, max_bytes=int(self.config.get("max_bytes", 20_000_000)), @@ -555,6 +557,9 @@ def _fetch_media_resources( "raw_response_sha256": hashlib.sha256(response.body).hexdigest(), "raw_response_bytes": len(response.body), "route": "browser_rendering_scrape_resources", + "browser": self.config.get("browser"), + "endpoint_route": "/browser-rendering/scrape" + ("?" + urlencode({"browser": self.config["browser"]}) if self.config.get("browser") else ""), + "adapter_sha256": hashlib.sha256(Path(__file__).read_bytes()).hexdigest(), "request_method": "POST", "request_summary": { "selectors": selectors, diff --git a/harness/web_fetch/providers/headless.py b/harness/web_fetch/providers/headless.py index 4b52bb3c2..f303f5f45 100644 --- a/harness/web_fetch/providers/headless.py +++ b/harness/web_fetch/providers/headless.py @@ -2,24 +2,27 @@ from __future__ import annotations import base64 +from contextlib import contextmanager +import socket import hashlib import importlib.util import json import math import os import shutil -import signal import subprocess import sys import tempfile +import threading import time from pathlib import Path +from urllib.parse import urlsplit from ..markdown import converter_package, require_converter from ..models import EvalCase, ProviderResult from ..retrieval_hints import plan_for_case from ..safety import validate_http_url -from .base import Provider +from .base import Provider, terminate_owned_process_tree from .runtime_identity import require_expected_provider_versions ENGINES = ('kitesurf', 'lightpanda', 'obscura', 'moli', 'chromium') @@ -30,9 +33,9 @@ '(allow network-outbound (remote ip "localhost:*"))' ) HEADLESS_CONVERSION = ( - 'Native Markdown for Kitesurf, Lightpanda, Obscura and Moli; direct ' - 'Moli documents/media use automatic raw output; Chromium ' - 'whole-DOM HTML via node-html-markdown 2.0.0' + 'Native Markdown for ordinary pages; task-native browser bytes for files and ' + 'documents with shared text extraction; shared DOM projections for resources, ' + 'embedded documents and media metadata; Chromium HTML uses node-html-markdown 2.0.0' ) @@ -148,6 +151,26 @@ def completion_metadata(engine, plan, output_mode): } +def _native_script_bytes(metadata): + if 'script_source' in metadata: + if 'script_path' in metadata or not isinstance(metadata['script_source'], str): + raise ValueError('native Markdown script source is ambiguous or invalid') + source = metadata['script_source'].encode('utf-8') + else: + source = Path(metadata['script_path']).read_bytes() + if hashlib.sha256(source).hexdigest() != metadata['script_sha256']: + raise ValueError('native Markdown script hash mismatch') + return source + + +def _worker_native_metadata(metadata): + if not metadata or metadata.get('format') != 'obscura_eval': + return metadata + source = _native_script_bytes(metadata) + return {**{key: value for key, value in metadata.items() if key != 'script_path'}, + 'script_source': source.decode('utf-8')} + + def cli_command( engine, binary, @@ -159,11 +182,49 @@ def cli_command( allow_private_network=False, ca_cert=None, http_proxy=None, + native_metadata=None, + track=None, + detail_links=False, ): + if native_metadata: + if output_mode != 'markdown' or retrieval_plan: + raise ValueError('native metadata requires native Markdown completion') + if engine == 'lightpanda' and native_metadata.get('format') == 'lightpanda_json': + return cli_command(engine, binary, url, timeout, output_mode, + wait_until, retrieval_plan, allow_private_network, + ca_cert, http_proxy) + ['--json'] + if engine == 'obscura' and native_metadata.get('format') == 'obscura_eval': + source = _native_script_bytes(native_metadata) + expression = ('JSON.stringify({content:(' + source.decode('utf-8') + + '),final_url:location.href,title:document.title})') + prefix = [binary, '--allow-private-network'] if allow_private_network else [binary] + if http_proxy: + prefix += ['--proxy', http_proxy] + return prefix + ['fetch', '--eval', expression, '--timeout', + str(math.ceil(timeout)), '--quiet', url] + raise ValueError('unsupported native metadata route') ms = str(max(1, math.ceil(timeout * 1000))) + if output_mode in {'resource', 'media_metadata', 'document_page'}: + from ..task_outputs import MEDIA_METADATA_EXPRESSION, DOCUMENT_PAGE_EXPRESSION, resource_expression + expression = resource_expression(track, detail_links) if output_mode == 'resource' else {'media_metadata': MEDIA_METADATA_EXPRESSION, 'document_page': DOCUMENT_PAGE_EXPRESSION}[output_mode] + if engine == 'obscura': + command = [binary, 'fetch', '--eval', expression, '--timeout', str(math.ceil(timeout)), '--quiet'] + if allow_private_network: + command.append('--allow-private-network') + if http_proxy: + command += ['--proxy', http_proxy] + return command + [url] + raise ValueError('Rendered projection requires a native CDP or eval interface') if output_mode not in {'html', 'markdown', 'document', 'file'}: raise ValueError('Unsupported browser output mode: ' + output_mode) if output_mode in {'document', 'file'}: + if engine == 'obscura': + command = [binary, 'fetch', '--dump', 'original'] + if allow_private_network: + command.append('--allow-private-network') + if http_proxy: + command += ['--proxy', http_proxy] + return command + ['--timeout', str(math.ceil(timeout)), '--quiet', url] if engine != 'moli': raise ValueError('Raw document auto-output is only supported by Moli') # All three measured Moli releases write direct raw responses such as @@ -210,23 +271,62 @@ def cli_command( def output_mode_for_case(engine, provider_type, case): """Choose the fastest version-supported route for the requested deliverable.""" - document_url = case.url.casefold().split('?', 1)[0] - direct_document = (engine == 'moli' and case.track == 'document' - and (document_url.endswith(('.pdf', '.docx')) - or '/pdf/' in document_url)) - if engine == 'moli' and case.task_kind == 'file': + if engine in ENGINES and case.task_kind == 'file': return 'file' - if direct_document: - return 'document' + # Document text must come from the selected browser content interface. + # Downloaded bytes must not be turned into readable text by the harness. + if case.track == 'document' and (urlsplit(case.url).path.casefold().endswith('.html') or urlsplit(case.url).path.endswith('/')): + return 'document_page' + if case.track == 'document': + return 'markdown' if provider_type == 'headless_markdown' else 'html' + if case.task_kind == 'resource' and case.track in {'image', 'video'}: + return 'resource' + if case.task_kind == 'body' and case.track == 'video': + return 'media_metadata' return 'markdown' if provider_type == 'headless_markdown' else 'html' class HeadlessProvider(Provider): + def __init__(self, name, config): + super().__init__(name, config) + self._process_lock = threading.Lock() + self._active_process = None + self._cancel_requested = False + + def begin_call(self): + with self._process_lock: + self._active_process = None + self._cancel_requested = False + + def cancel(self): + with self._process_lock: + self._cancel_requested = True + terminate_owned_process_tree(self._active_process) + + @property + def cancellation_cleanup_seconds(self): + # Remote and resource-container calls retain their own boundaries. + local = (self.config.get('engine') != 'kitesurf' + and self.config.get('resource_sample_interval_ms') is None) + return 1.0 if local else 0.0 + def validate(self): errors = [] engine = self.config.get('engine') if engine not in ENGINES: errors.append('engine') + native_metadata = self.config.get('native_metadata') + if native_metadata is not None: + try: + if not isinstance(native_metadata, dict): + raise ValueError('native metadata must be an object') + cli_command(engine, self.config.get('binary', 'browser'), + 'https://example.com/', 1, 'markdown', + native_metadata=native_metadata) + if self.type != 'headless_markdown': + raise ValueError('native metadata requires headless_markdown') + except (KeyError, OSError, ValueError) as error: + errors.append('native_metadata:' + str(error)) resource_interval = self.config.get('resource_sample_interval_ms') if resource_interval is not None: if engine == 'kitesurf' and self.type == 'headless_markdown': @@ -365,20 +465,36 @@ def fetch(self, case: EvalCase, timeout: float) -> ProviderResult: getattr(self, '_resolved_binary', None) or _resolve_binary( engine, self.config.get('binary'))) cli_wait = retrieval_plan.wait_until if retrieval_plan else 'done' - wait_until = ('automatic raw response' if output_mode == 'file' + wait_until = ('browser response bytes or completed native download' if output_mode in {'file','document'} and engine == 'chromium' + else 'navigation response bytes' if output_mode in {'file','document'} and engine in {'lightpanda','kitesurf'} + else 'automatic raw response' if output_mode == 'file' else 'automatic raw response or serialized HTML' if output_mode == 'document' else 'networkidle2' if engine == 'kitesurf' and output_mode == 'markdown' else cli_wait if output_mode == 'markdown' and engine in {'moli','lightpanda'} else 'load + adaptive settle' if output_mode == 'markdown' and engine == 'obscura' - else 'domcontentloaded') - extractor = ('moli_auto_raw' if output_mode == 'file' - else 'moli_auto_raw+document_text' if output_mode == 'document' + else 'load + adaptive settle' if engine == 'obscura' and output_mode in {'resource','media_metadata'} + else self.config.get('cdp_wait_until', 'load') if engine == 'kitesurf' + else 'load') + extractor = ('browser_native_response_or_download' if output_mode == 'file' and engine == 'chromium' + else 'kitesurf_cdp_response' if output_mode == 'file' and engine == 'kitesurf' + else 'lightpanda_cdp_response' if output_mode == 'file' and engine == 'lightpanda' + else 'obscura_original_response' if output_mode == 'file' and engine == 'obscura' + else 'moli_cdp_response' if output_mode == 'file' + else 'browser_native_response_or_download+document_text' if output_mode == 'document' and engine == 'chromium' + else 'kitesurf_cdp_response+document_text' if output_mode == 'document' and engine == 'kitesurf' + else 'lightpanda_cdp_response+document_text' if output_mode == 'document' and engine == 'lightpanda' + else 'obscura_original_response+document_text' if output_mode == 'document' and engine == 'obscura' + else 'moli_cdp_response+document_text' if output_mode == 'document' else 'cloudflare_native_markdown' if engine == 'kitesurf' and output_mode == 'markdown' else f'{engine}_native_markdown' if output_mode == 'markdown' and engine != 'chromium' + else 'rendered_dom_resources' if output_mode == 'resource' + else 'rendered_body_and_standard_media_metadata' if output_mode == 'media_metadata' + else 'browser_runtime_document_text' if output_mode == 'document_page' else 'node-html-markdown') output_contract = ('original_file_bytes' if output_mode == 'file' else - 'readable_document_text' if output_mode == 'document' else - 'markdown') + 'readable_document_text' if output_mode in {'document','document_page'} else + 'structured_resource_urls' if output_mode == 'resource' else + 'readable_media_metadata' if output_mode == 'media_metadata' else 'markdown') metadata = {'engine': engine, 'wait_until': wait_until, 'extractor': extractor, 'output_contract': output_contract, 'session': 'fresh per call', @@ -389,6 +505,10 @@ def fetch(self, case: EvalCase, timeout: float) -> ProviderResult: if resource_interval is not None else 'public cloud CDP' if engine == 'kitesurf' else 'local process')} + if output_mode in {'resource', 'media_metadata', 'document_page'}: + from ..task_outputs import MEDIA_METADATA_EXPRESSION, DOCUMENT_PAGE_EXPRESSION, resource_expression, resource_detail_links + projection = resource_expression(case.track, resource_detail_links(case)) if output_mode == 'resource' else {'media_metadata': MEDIA_METADATA_EXPRESSION, 'document_page': DOCUMENT_PAGE_EXPRESSION}[output_mode] + metadata['output_projection_sha256'] = hashlib.sha256(projection.encode('utf-8')).hexdigest() normalized_plan = retrieval_plan.worker_dict() if retrieval_plan else None if normalized_plan and engine != 'kitesurf': metadata['completion'] = completion_metadata(engine, normalized_plan, output_mode) @@ -401,6 +521,7 @@ def fetch(self, case: EvalCase, timeout: float) -> ProviderResult: 'account_id': self.config['account_id'], 'api_token': self.config['api_token'], 'browser': 'kitesurf', 'wait_until': self.config.get('wait_until', 'networkidle2'), 'max_bytes': self.config.get('max_bytes', 20_000_000), + 'endpoint': 'wss://api.cloudflare.com/client/v4/accounts/' + self.config['account_id'] + '/browser-run/devtools/browser?browser=kitesurf', }) result = cloudflare.fetch(case, timeout) result.metadata = {**metadata, **result.metadata, @@ -410,31 +531,45 @@ def fetch(self, case: EvalCase, timeout: float) -> ProviderResult: identity = self.implementation_identity() metadata['binary_sha256'] = identity['binary_sha256'] if identity['reported_version']: + metadata['reported_version'] = identity['reported_version'] metadata['browser_version'] = identity['reported_version'] if identity['version_probe_error']: metadata['browser_version_probe_error'] = identity['version_probe_error'] request = {'engine': engine, 'binary': binary, 'url': case.url, 'timeout': timeout, 'output_mode': output_mode, 'task_kind': case.task_kind, + 'resource_detail_links': resource_detail_links(case) if output_mode == 'resource' else False, 'browser_args': ['--no-sandbox'] if resource_interval is not None and engine == 'chromium' else [], 'wait_until': retrieval_plan.wait_until if retrieval_plan else None, 'track': case.track, 'retrieval_plan': normalized_plan, + 'retrieval_hints': case.retrieval_hints, 'allow_private_network': bool(self.config.get('allow_private_network')), + 'block_private_networks': bool(self.config.get('block_private_networks', False)), + 'log_level': self.config.get('log_level'), + 'max_diagnostic_bytes': self.config.get('max_diagnostic_bytes', 100_000_000), + 'max_original_file_bytes': self.config.get('max_original_file_bytes'), + 'native_metadata': _worker_native_metadata(self.config.get('native_metadata')) + if output_mode == 'markdown' else None, 'ca_cert': self.config.get('ca_cert'), 'http_proxy': self.config.get('http_proxy'), 'network_policy': self.config.get('network_policy', 'unrestricted'), 'endpoint': self.config.get('endpoint', 'wss://kitesurf.cloudflare.app/devtools/browser')} if engine == 'kitesurf': + cdp_wait = self.config.get('cdp_wait_until', 'load') + if cdp_wait not in {'load', 'domcontentloaded', 'networkidle', 'commit'}: + raise ValueError('Kitesurf cdp_wait_until must be a supported CDP completion event') + metadata.update(browser='kitesurf', endpoint_route='/browser-run/devtools/browser?browser=kitesurf') request.update({ - 'remote_markdown_api': True, + 'remote_markdown_api': output_mode == 'markdown', 'account_id': self.config['account_id'], 'api_token': self.config['api_token'], 'browser': 'kitesurf', - 'wait_until': self.config.get('wait_until', 'networkidle2'), + 'wait_until': self.config.get('wait_until', 'networkidle2') if output_mode == 'markdown' else cdp_wait, 'max_bytes': self.config.get('max_bytes', 20_000_000), + 'endpoint': 'wss://api.cloudflare.com/client/v4/accounts/' + self.config['account_id'] + '/browser-run/devtools/browser?browser=kitesurf', }) if engine == 'chromium' or self.type == 'headless_html': configured_package = self.config.get('node_html_markdown_path') @@ -442,6 +577,7 @@ def fetch(self, case: EvalCase, timeout: float) -> ProviderResult: if resource_interval is None else configured_package) worker = Path(__file__).with_name('headless_worker.py') metadata['worker_sha256'] = hashlib.sha256(worker.read_bytes()).hexdigest() + metadata['execution_sha256'] = metadata['worker_sha256'] if request['browser_args']: metadata['browser_launch_args'] = request['browser_args'] limit = 40_000_000 @@ -491,42 +627,132 @@ def fetch(self, case: EvalCase, timeout: float) -> ProviderResult: metadata['rendered_html_sha256'] = hashlib.sha256(html.encode()).hexdigest() return ProviderResult(**result, metadata=metadata) - @staticmethod - def _run_local_worker(worker, request, timeout, limit): + def _run_local_worker(self, worker, request, timeout, limit): with tempfile.TemporaryFile() as stdout, tempfile.TemporaryFile() as stderr: command = [sys.executable, str(worker)] if request.get('network_policy') == 'loopback_only': command = ['/usr/bin/sandbox-exec', '-p', LOOPBACK_SANDBOX_PROFILE, *command] + with self._process_lock: + if self._cancel_requested: + raise TimeoutError('Isolated fetch cancelled before launch') + # Popen must not hold the lock needed by the deadline thread. process = subprocess.Popen(command, stdin=subprocess.PIPE, stdout=stdout, stderr=stderr, start_new_session=True) - process.stdin.write(json.dumps(request).encode()) - process.stdin.close() - deadline = time.monotonic() + timeout + with self._process_lock: + self._active_process = process + if self._cancel_requested: + terminate_owned_process_tree(process) stopped = None - while process.poll() is None: - if stdout.tell() > limit or stderr.tell() > limit: - stopped = 'output_too_large' - break - if time.monotonic() >= deadline: - stopped = 'timeout' - break - try: - process.wait(timeout=min(.05, max(.001, deadline-time.monotonic()))) - except subprocess.TimeoutExpired: - pass - if stopped: + try: try: - os.killpg(process.pid, signal.SIGKILL) - except ProcessLookupError: - pass - except PermissionError: - # Some managed runners forbid process-group signals while - # permitting signals to a child created by this process. - process.kill() + process.stdin.write(json.dumps(request).encode()) + process.stdin.close() + except BrokenPipeError: + with self._process_lock: + if not self._cancel_requested: + raise + deadline = time.monotonic() + timeout + while process.poll() is None: + with self._process_lock: + cancelled = self._cancel_requested + if cancelled: + stopped = 'timeout' + break + if stdout.tell() > limit or stderr.tell() > limit: + stopped = 'output_too_large' + break + if time.monotonic() >= deadline: + stopped = 'timeout' + break + try: + process.wait(timeout=min(.05, max(.001, deadline-time.monotonic()))) + except subprocess.TimeoutExpired: + pass + with self._process_lock: + if self._cancel_requested: + stopped = 'timeout' + if stopped: + terminate_owned_process_tree(process) + process.wait() + stdout_bytes = os.fstat(stdout.fileno()).st_size + stderr_bytes = os.fstat(stderr.fileno()).st_size + stdout.seek(0); raw = stdout.read(limit+1) + stderr.seek(0); diagnostic_raw = stderr.read(limit+1) + diagnostic = diagnostic_raw[:limit].decode('utf-8', 'replace') + finally: + terminate_owned_process_tree(process) process.wait() - stdout_bytes = os.fstat(stdout.fileno()).st_size - stderr_bytes = os.fstat(stderr.fileno()).st_size - stdout.seek(0); raw = stdout.read(limit+1) - stderr.seek(0); diagnostic_raw = stderr.read(limit+1) - diagnostic = diagnostic_raw[:limit].decode('utf-8', 'replace') + if not process.stdin.closed: + try: + process.stdin.close() + except BrokenPipeError: + pass + with self._process_lock: + if self._active_process is process: + self._active_process = None return raw, diagnostic, stopped, process.returncode, stdout_bytes, stderr_bytes + + +@contextmanager +def native_cdp_browser(playwright, request, deadline, metadata=None): + """Own a fresh native CDP server within the caller's complete deadline.""" + with socket.socket() as reservation: + reservation.bind(('127.0.0.1', 0)) + port = reservation.getsockname()[1] + remaining = lambda: max(.001, deadline - time.monotonic()) + command = [request['binary'], 'serve', '--host', '127.0.0.1', '--port', str(port)] + engine = request['engine'] + if engine == 'lightpanda': + command += ['--http-timeout', str(max(1, math.ceil(remaining() * 1000)))] + elif engine == 'moli': + if request.get('block_private_networks'): command.append('--block-private-networks') + if request.get('log_level'): command += ['--log-level', request['log_level']] + elif engine == 'obscura': + command.append('--quiet') + if request.get('allow_private_network'): + command.append('--allow-private-network') + else: + raise ValueError('Unsupported native CDP server engine') + if engine == 'lightpanda' and request.get('output_mode') in {'resource', 'document_page'}: + command += ['--load-resources', 'iframe'] + if request.get('http_proxy'): + command += ['--http-proxy' if engine in {'lightpanda', 'moli'} else '--proxy', request['http_proxy']] + if request.get('ca_cert') and engine in {'lightpanda', 'moli'}: + command += ['--ca-cert', str(request['ca_cert'])] + browser = None + with tempfile.TemporaryFile() as log: + environment = {**os.environ, 'SSL_CERT_FILE': str(request['ca_cert'])} if engine == 'obscura' and request.get('ca_cert') else None + process = subprocess.Popen(command, stdout=log, stderr=log, env=environment) + try: + while True: + if process.poll() is not None: + log.seek(0) + raise RuntimeError(log.read().decode('utf-8', 'replace')[:4000]) + if time.monotonic() >= deadline: + raise TimeoutError('Native CDP server startup exceeded the call deadline') + try: + with socket.create_connection(('127.0.0.1', port), timeout=min(.1, remaining())): + break + except OSError: + time.sleep(min(.02, remaining())) + browser = playwright.chromium.connect_over_cdp( + 'http://127.0.0.1:' + str(port), timeout=remaining() * 1000) + yield browser + finally: + try: + if browser is not None: + browser.close() + finally: + if process.poll() is None: + process.terminate() + try: + process.wait(timeout=1) + except subprocess.TimeoutExpired: + process.kill() + process.wait(timeout=1) + if engine == 'moli' and metadata is not None: + log.seek(0) + raw_log = log.read(int(request.get('max_diagnostic_bytes', 100_000_000))+1) + metadata.update(_diagnostic_log_b64=base64.b64encode(raw_log).decode('ascii'), + diagnostic_log_sha256=hashlib.sha256(raw_log).hexdigest(), diagnostic_log_bytes=len(raw_log), + diagnostic_log_process_completed=True) diff --git a/harness/web_fetch/providers/headless_worker.py b/harness/web_fetch/providers/headless_worker.py index 0ac874ded..dcc733e1d 100644 --- a/harness/web_fetch/providers/headless_worker.py +++ b/harness/web_fetch/providers/headless_worker.py @@ -7,26 +7,131 @@ import subprocess import sys import time +import tempfile from importlib.metadata import version from pathlib import Path sys.path.insert(0, str(Path(__file__).resolve().parents[3])) -from harness.web_fetch.document_text import extract_document_text from harness.web_fetch.markdown import CONVERTER, converter_command, require_converter -from harness.web_fetch.providers.headless import cli_command, completion_metadata +from harness.web_fetch.document_text import extract_document_text +from harness.web_fetch.providers.headless import cli_command, completion_metadata, native_cdp_browser from harness.web_fetch.safety import validate_http_url def require_normalized_retrieval_plan(request): - """Enforce presence at this process boundary; the runner owns validation.""" + """Keep the declared family while validating explicit native completion.""" + from harness.web_fetch.retrieval_hints import parse_retrieval_hints plan = request.get('retrieval_plan') - if request.get('track') == 'dynamic' and not plan: - raise ValueError('dynamic tasks require a normalized retrieval_plan') - if request.get('track') not in {None, 'dynamic'} and plan: + track = request.get('track') + hints = request.get('retrieval_hints') + declared = parse_retrieval_hints(track, hints) if hints else None + if track == 'dynamic' and not plan: + if not hints or declared is not None: + raise ValueError('dynamic tasks require a normalized retrieval_plan or explicit native contract') + if declared is None and hints and plan: + raise ValueError('native completion cannot carry a retrieval_plan') + if track not in {None, 'dynamic'} and plan: raise ValueError('retrieval_plan is only valid for dynamic tasks') return plan +def _native_browser_file(page, request, route_timeout, start, meta): + """Deliver bytes from this browser navigation or its native download.""" + from playwright.sync_api import Error + downloads = [] + page.on('download', lambda download: downloads.append(download)) + response = None + remaining_ms = lambda: max(1, (route_timeout - (time.monotonic() - start)) * 1000) + meta['navigation_wait_until'] = 'commit' + try: + response = page.goto(request['url'], wait_until='commit', timeout=remaining_ms()) + except Error as error: + if 'ERR_ABORTED' not in str(error) and 'Download is starting' not in str(error): + raise + if not downloads: + downloads.append(page.wait_for_event('download', timeout=remaining_ms())) + if downloads: + download = downloads[0] + if request.get('engine') == 'moli': + # This is a native attachment event, not an HTTP/scoring retry. + # Moli's CLI exposes raw attachments while its current CDP download + # completion event is incompatible with Playwright's file helper. + command = cli_command('moli', request['binary'], download.url, + remaining_ms()/1000, output_mode='file', ca_cert=request.get('ca_cert'), + http_proxy=request.get('http_proxy')) + if request.get('block_private_networks'): command.append('--block-private-networks') + result = subprocess.run(command, capture_output=True, timeout=remaining_ms()/1000) + if result.returncode: raise ValueError(result.stderr.decode('utf-8','replace')) + raw = result.stdout + meta.update(native_attachment_interface='moli_cli_raw_response', + native_navigation_count=1, native_attachment_fetch_count=1, + native_attachment_stderr=result.stderr.decode('utf-8','replace')) + else: + failure = download.failure() + if failure: + raise ValueError('Browser download failed: ' + failure) + raw = Path(download.path()).read_bytes() + final_url, status, mime = download.url, None, None + meta['native_file_route'] = 'moli_cli_attachment_response' if request.get('engine') == 'moli' else 'browser_download' + elif response is not None: + final_url, status = response.url, response.status + mime = response.headers.get('content-type') + meta.update(file_response_status=status, file_response_url=final_url) + raw = response.body() + meta['native_file_route'] = 'navigation_response_body' + else: + raise ValueError('Browser navigation exposed neither a response nor a download') + limit = request.get('max_original_file_bytes') + if limit is not None and len(raw) > limit: + meta.update(original_file_bytes=len(raw), original_file_byte_limit=limit) + return dict(ok=False, content='', html='', status_code=status, final_url=final_url, + metadata=meta, error_type='output_too_large', + error_message='Original file exceeds the configured byte budget') + meta.update(raw_stdout_sha256=hashlib.sha256(raw).hexdigest(), + raw_fetch_ms=round((time.monotonic() - start) * 1000, 2), + file_response_mime=mime) + ok = bool(raw) and (status is None or 200 <= status < 300) + if not ok: + meta['_raw_stdout_b64'] = base64.b64encode(raw).decode('ascii') + return dict(ok=ok, content='', html='', status_code=status, final_url=final_url, + metadata=meta, files_b64=[base64.b64encode(raw).decode('ascii')] if ok else [], + error_type=None if ok else 'http_error' if status is not None and status >= 400 else 'empty_response') + + +def _projection_expression(output_mode, track, detail_links=False): + from harness.web_fetch.task_outputs import MEDIA_METADATA_EXPRESSION, DOCUMENT_PAGE_EXPRESSION, resource_expression + return resource_expression(track, detail_links) if output_mode == 'resource' else {'media_metadata': MEDIA_METADATA_EXPRESSION, + 'document_page': DOCUMENT_PAGE_EXPRESSION}[output_mode] + + +def _projection_text(content, output_mode, meta, deadline): + if output_mode != 'document_page': + return content + extract_start = time.monotonic() + envelope = json.loads(content) + if not isinstance(envelope, dict): + raise ValueError('Invalid document projection') + if envelope.get('source'): + validate_http_url(envelope['source']) + meta['embedded_document_url'] = envelope['source'] + if envelope.get('kind') in {'text', 'page_text'}: + text = str(envelope.get('text') or '') + meta['document_extractor'] = 'pdfjs_runtime' if envelope['kind'] == 'text' else 'rendered_page_text' + else: + raise ValueError('Unknown document projection kind') + meta['document_extract_ms'] = round((time.monotonic()-extract_start)*1000, 2) + if time.monotonic() >= deadline: + raise TimeoutError('Document projection exceeded the call deadline') + return text + + +def _page_projection(page, request, output_mode, meta, deadline): + if output_mode == 'document_page': + from harness.web_fetch.task_outputs import DOCUMENT_PAGE_READY + page.wait_for_function(DOCUMENT_PAGE_READY, timeout=max(1, (deadline-time.monotonic())*1000)) + return _projection_text(page.evaluate(_projection_expression(output_mode, request.get('track'), request.get('resource_detail_links', False))), output_mode, meta, deadline) + + def fetch(request): # Match the provider boundary; this validates URL syntax, not network isolation. validate_http_url(request['url']) @@ -41,9 +146,7 @@ def fetch(request): html = '' try: if engine == 'kitesurf' and request.get('remote_markdown_api'): - from harness.web_fetch.providers.cloudflare import ( - CloudflareMarkdownProvider, - ) + from harness.web_fetch.providers.cloudflare import CloudflareMarkdownProvider provider = CloudflareMarkdownProvider('kitesurf', { 'account_id': request['account_id'], @@ -67,6 +170,29 @@ def fetch(request): 'error_message': result.error_message, 'metadata': result.metadata, } + if (engine == 'moli' and output_mode in {'document', 'file', 'document_page'}) or (engine == 'lightpanda' and output_mode in {'document', 'file', 'resource', 'media_metadata', 'document_page'}) or (engine == 'obscura' and output_mode == 'document_page'): + from playwright.sync_api import sync_playwright + with sync_playwright() as playwright: + with native_cdp_browser(playwright, request, start + route_timeout, metadata=meta) as browser: + meta['browser_version'] = browser.version + page = browser.new_context().new_page() + if output_mode in {'resource', 'media_metadata', 'document_page'}: + meta['navigation_wait_until'] = 'load' + response = page.goto(request['url'], wait_until=meta['navigation_wait_until'], timeout=max(1, (route_timeout-(time.monotonic()-start))*1000)) + content = _page_projection(page, request, output_mode, meta, start + route_timeout) + return dict(ok=response is not None and 200 <= response.status < 300, content=content, html='', status_code=response.status if response else None, final_url=page.url, metadata=meta) + native = _native_browser_file(page, request, route_timeout, start, meta) + if output_mode == 'file' or not native['ok']: + return native + raw = base64.b64decode(native.pop('files_b64')[0], validate=True) + meta['_raw_stdout_b64'] = base64.b64encode(raw).decode('ascii') + extract_start = time.monotonic() + text, extractor = extract_document_text(raw) + meta.update(document_extractor=extractor, + document_extract_ms=round((time.monotonic()-extract_start)*1000, 2)) + native.update(ok=bool(text.strip()), content=text, + error_type=None if text.strip() else 'empty_response') + return native if engine in ('moli', 'obscura', 'lightpanda'): command = cli_command( engine, @@ -79,6 +205,9 @@ def fetch(request): allow_private_network=bool(request.get('allow_private_network')), ca_cert=request.get('ca_cert'), http_proxy=request.get('http_proxy'), + native_metadata=request.get('native_metadata'), + track=request.get('track'), + detail_links=request.get('resource_detail_links', False), ) environment = None if engine == 'obscura' and request.get('ca_cert'): @@ -87,7 +216,7 @@ def fetch(request): **({'env': environment} if environment else {})) meta.update(browser_exit_code=result.returncode, browser_stderr=result.stderr.decode('utf-8', 'replace')) - if result.returncode: + if result.returncode and not request.get('native_metadata'): return dict(ok=False, content='', html='', error_type='browser_exit', error_message=meta['browser_stderr'] or str(result.returncode), metadata=meta) if output_mode == 'file': @@ -98,12 +227,12 @@ def fetch(request): files_b64=[base64.b64encode(result.stdout).decode('ascii')] if result.stdout else [], error_type=None if result.stdout else 'empty_response') - if output_mode == 'document' and result.stdout.startswith((b'%PDF', b'PK')): + if output_mode == 'document': raw_fetch_ms = round((time.monotonic()-start)*1000, 2) - text, extractor = extract_document_text(result.stdout) meta['_raw_stdout_b64'] = base64.b64encode(result.stdout).decode('ascii') meta['raw_stdout_sha256'] = hashlib.sha256(result.stdout).hexdigest() meta['raw_fetch_ms'] = raw_fetch_ms + text, extractor = extract_document_text(result.stdout) meta['document_extractor'] = extractor meta['document_extract_ms'] = round((time.monotonic()-start)*1000-raw_fetch_ms, 2) return dict(ok=bool(text.strip()), content=text, html='', status_code=None, @@ -116,39 +245,101 @@ def fetch(request): meta['raw_stdout_sha256'] = hashlib.sha256(result.stdout).hexdigest() return dict(ok=False, content='', html='', error_type='non_utf8_response', error_message='HTML adapter received non-UTF-8 bytes; preserved without conversion', metadata=meta) + if output_mode == 'markdown' and request.get('native_metadata'): + meta['_raw_stdout_b64'] = base64.b64encode(result.stdout).decode('ascii') + meta['raw_stdout_sha256'] = hashlib.sha256(result.stdout).hexdigest() + contract = request['native_metadata'] + meta['native_metadata'] = {key: value for key, value in contract.items() + if key not in {'script_path', 'script_source'}} + envelope = json.loads(browser_output) + if not isinstance(envelope, dict) or not isinstance(envelope.get('content'), str): + raise ValueError('native Markdown envelope requires string content') + content = envelope['content'] + final_url = envelope.get('url') if engine == 'lightpanda' else envelope.get('final_url') + final_url = final_url or None + if final_url is not None: + validate_http_url(final_url) + status = envelope.get('http_status') if engine == 'lightpanda' else None + status = status if isinstance(status, int) and 100 <= status <= 599 else None + error = envelope.get('error') + ok = not result.returncode and not error and bool(content.strip()) + meta['native_markdown_ms'] = round((time.monotonic()-start)*1000, 2) + return dict(ok=ok, content=content, html='', status_code=status, + final_url=final_url, title=envelope.get('title'), metadata=meta, + error_type='browser_exit' if result.returncode or error else + None if content.strip() else 'empty_response', + error_message=str(error or meta['browser_stderr']) if result.returncode or error else None) + if output_mode in {'resource', 'media_metadata', 'document_page'}: + meta['projection_ms'] = round((time.monotonic()-start)*1000, 2) + content = _projection_text(browser_output, output_mode, meta, start + route_timeout) + return dict(ok=bool(content.strip()), content=content, html='', status_code=None, final_url=None, metadata=meta) if output_mode == 'markdown': meta['native_markdown_ms'] = round((time.monotonic()-start)*1000, 2) return dict(ok=bool(browser_output.strip()), content=browser_output, html='', status_code=None, final_url=None, metadata=meta, error_type=None if browser_output.strip() else 'empty_response') html = browser_output - if output_mode == 'document': - meta['document_response'] = 'serialized_html_not_raw_document' status = None final_url = None else: from playwright.sync_api import sync_playwright meta['playwright_version'] = version('playwright') with sync_playwright() as p: + fixture_https = bool(request.get('ca_cert')) and request.get('network_policy') == 'loopback_only' + profile = None + persistent_context = None if engine == 'kitesurf': - browser = p.chromium.connect_over_cdp(request['endpoint'], timeout=timeout*1000) + browser = p.chromium.connect_over_cdp(request['endpoint'], timeout=max(1, (route_timeout-(time.monotonic()-start))*1000), headers={'Authorization': 'Bearer ' + request['api_token']}) else: launch = dict(executable_path=request['binary'], headless=True, args=request.get('browser_args') or [], timeout=timeout*1000) if request.get('http_proxy'): launch['proxy'] = {'server': request['http_proxy']} - browser = p.chromium.launch(**launch) + if output_mode in {'file', 'document'}: + # Use Chrome's standard PDF download preference. Its + # built-in viewer replaces response bytes with an HTML + # wrapper, which cannot satisfy a binary/text contract. + profile = tempfile.TemporaryDirectory(prefix='browser-download-') + preferences = Path(profile.name) / 'Default' / 'Preferences' + preferences.parent.mkdir() + preferences.write_text(json.dumps({'plugins': {'always_open_pdf_externally': True}})) + persistent_context = p.chromium.launch_persistent_context( + profile.name, accept_downloads=True, ignore_https_errors=fixture_https, **launch) + browser = persistent_context.browser + meta['pdf_handling'] = 'native download; viewer disabled in fresh temporary profile' + else: + browser = p.chromium.launch(**launch) try: - meta['browser_version'] = browser.version + if browser is not None: + if engine == 'kitesurf': + meta['cdp_reported_browser_version'] = browser.version + meta['engine_version_exposed'] = False + else: + meta['browser_version'] = browser.version # The benchmark fixture has a per-run CA, not a system-trusted # certificate. Only isolated loopback requests may bypass the # browser's public trust store. - fixture_https = bool(request.get('ca_cert')) and request.get('network_policy') == 'loopback_only' - context = browser.new_context(ignore_https_errors=fixture_https) + context = persistent_context or browser.new_context(ignore_https_errors=fixture_https) page = context.new_page() + if output_mode in {'file', 'document'}: + native = _native_browser_file(page, request, route_timeout, start, meta) + if output_mode == 'file' or not native['ok']: + return native + raw = base64.b64decode(native.pop('files_b64')[0], validate=True) + meta['_raw_stdout_b64'] = base64.b64encode(raw).decode('ascii') + if raw.startswith((b'%PDF', b'PK')): + extract_start = time.monotonic() + text, extractor = extract_document_text(raw) + meta.update(document_extractor=extractor, + document_extract_ms=round((time.monotonic()-extract_start)*1000, 2)) + native.update(ok=bool(text.strip()), content=text, + error_type=None if text.strip() else 'empty_response') + return native + raise ValueError('Browser document response is not a supported PDF or DOCX') + meta['navigation_wait_until'] = request.get('wait_until') or 'load' response = page.goto( request['url'], - wait_until=request.get('wait_until') or 'domcontentloaded', + wait_until=meta['navigation_wait_until'], timeout=max(1, (route_timeout-(time.monotonic()-start))*1000), ) if plan: @@ -162,17 +353,26 @@ def fetch(request): page.wait_for_function(plan['wait_value'], timeout=remaining_ms) if plan['delay_ms']: page.wait_for_timeout(plan['delay_ms']) + if output_mode in {'resource', 'media_metadata', 'document_page'}: + content = _page_projection(page, request, output_mode, meta, start + route_timeout) + return dict(ok=response is not None and 200 <= response.status < 300, content=content, html='', status_code=response.status if response else None, final_url=page.url, metadata=meta) # Avoid Playwright's XMLSerializer helper, which is not # implemented by every CDP engine. Same expression for both. html = page.evaluate('document.documentElement.outerHTML') final_url = page.url status = response.status if response else None finally: - browser.close() + if persistent_context is not None: + persistent_context.close() + elif browser is not None: + browser.close() + if profile is not None: + profile.cleanup() meta['fetch_html_ms'] = round((time.monotonic()-start)*1000, 2) + converter = CONVERTER package = require_converter(request.get('node_html_markdown_path')) meta['extractor_version'] = json.loads((package / 'package.json').read_text())['version'] - meta['converter_sha256'] = hashlib.sha256(CONVERTER.read_bytes()).hexdigest() + meta['converter_sha256'] = hashlib.sha256(converter.read_bytes()).hexdigest() meta['node_version'] = subprocess.run( [shutil.which('node') or 'node', '--version'], capture_output=True, text=True, check=True, timeout=5).stdout.strip() @@ -190,6 +390,7 @@ def fetch(request): error_type=None if ok else 'empty_response') except Exception as error: return dict(ok=False, content='', html=html, metadata=meta, + status_code=meta.get('file_response_status'), final_url=meta.get('file_response_url'), error_type='timeout' if 'Timeout' in type(error).__name__ else 'browser_error', error_message=str(error)[:4000]) diff --git a/harness/web_fetch/providers/moli.py b/harness/web_fetch/providers/moli.py index 7c23f6d67..f0f392d6b 100644 --- a/harness/web_fetch/providers/moli.py +++ b/harness/web_fetch/providers/moli.py @@ -10,7 +10,6 @@ import os import re import shutil -import signal import subprocess import tempfile import threading @@ -18,12 +17,10 @@ from pathlib import Path from urllib.parse import parse_qsl, quote, urlsplit -from ..document_text import extract_document_text -from ..markdown import CONVERTER, converter_command, require_converter from ..models import EvalCase, ProviderResult from ..retrieval_hints import native_default_case, plan_for_case from ..safety import validate_http_url -from .base import Provider +from .base import Provider, terminate_owned_process_tree _WAIT_UNTIL_VALUES = { "done", @@ -96,10 +93,11 @@ def identity(value: str) -> tuple | None: navigations = list(navigation_pattern.finditer(diagnostic)) for index, navigation in enumerate(navigations): boundary = navigations[index + 1].start() if index + 1 < len(navigations) else len(diagnostic) - committed = diagnostic.find( + commits = [diagnostic.find(marker, navigation.end(), boundary) for marker in ( b"committed main navigation before replacement realm bootstrap", - navigation.end(), boundary, - ) + b"detached committed main WindowProxy for replacement context", + )] + committed = next((position for position in commits if position >= 0), -1) if committed < 0: continue navigation_identity = identity(field(navigation, "url")) @@ -122,175 +120,11 @@ def identity(value: str) -> tuple | None: -_RESOURCE_PAGE_EXPRESSION = r""" -(async () => { - const urls = new Set(); - const add = (value, base = location.href) => { - if (!value) return; - for (const part of String(value).split(',')) { - const raw = part.trim().split(/\s+/)[0]; - if (!raw || /^(?:data|blob|javascript|mailto):/i.test(raw)) continue; - try { urls.add(new URL(raw, base).href); } catch {} - } - }; - - // Enumerate the top document and bounded, already-loaded same-origin frame - // documents. This is answer-free: it follows the page's own document - // hierarchy and never receives an expected URL or case-specific selector. - const roots = [document]; - for (let cursor = 0; cursor < roots.length && roots.length < 16; cursor++) { - for (const frame of roots[cursor].querySelectorAll('iframe,frame')) { - try { - if (frame.contentDocument && !roots.includes(frame.contentDocument)) { - roots.push(frame.contentDocument); - if (roots.length >= 16) break; - } - } catch {} - } - } - for (const root of roots) { - for (const node of root.querySelectorAll('[src],[href],[data],[poster],[srcset]')) { - for (const name of ['src', 'href', 'data', 'poster', 'srcset']) { - add(node.getAttribute(name), node.baseURI || root.baseURI || location.href); - } - } - } - const queue = roots.flatMap(root => [...root.querySelectorAll( - 'astro-island[component-url],script[type="module"][src]' - )].map(node => ({ - value: node.getAttribute('component-url') || node.getAttribute('src'), - base: node.baseURI || root.baseURI || location.href - }))) - .filter(item => item.value).slice(0, 16) - .map(item => ({url: new URL(item.value, item.base).href, depth: 0})); - const seen = new Set(); - while (queue.length && seen.size < 48) { - const item = queue.shift(), moduleUrl = item.url; - if (seen.has(moduleUrl) || new URL(moduleUrl).origin !== location.origin) continue; - seen.add(moduleUrl); - try { - const source = await (await fetch(moduleUrl)).text(); - for (const match of source.matchAll(/https?:\/\/[^\s`"'<>]+/g)) add(match[0]); - if (item.depth < 2) { - for (const match of source.matchAll(/(?:from\s*|import\s*)[('"`]([^'"`]+)[)'"`]/g)) { - try { - const imported = new URL(match[1], moduleUrl).href; - if (new URL(imported).origin === location.origin) { - queue.push({url: imported, depth: item.depth + 1}); - } - } catch {} - } - } - } catch {} - } - return JSON.stringify({links: [...urls]}); -})() -""" - -_DOCUMENT_PAGE_EXPRESSION = r""" -(async () => { - const app = globalThis.PDFViewerApplication; - if (app) { - await app.initializedPromise; - const pdf = app.pdfDocument; - if (pdf) { - const pages = []; - for (let number = 1; number <= pdf.numPages; number++) { - const page = await pdf.getPage(number), text = await page.getTextContent(); - pages.push(text.items.map(item => item.str).join(' ')); - } - return JSON.stringify({ - kind: 'text', source: new URL(app.url, location.href).href, - text: pages.join('\n\n') - }); - } - } - const nodes = [...document.querySelectorAll('embed[src],iframe[src],object[data],a[href]')]; - const source = nodes.map(node => node.src || node.data || node.href) - .find(value => value && /\.pdf(?:$|[?#])/i.test(value)); - if (!source) return JSON.stringify({kind: 'page_text', text: document.body?.innerText || ''}); - const response = await fetch(new URL(source, location.href)); - if (!response.ok) throw new Error(`embedded PDF HTTP ${response.status}`); - const bytes = new Uint8Array(await response.arrayBuffer()); - let binary = ''; - for (let offset = 0; offset < bytes.length; offset += 32768) { - binary += String.fromCharCode(...bytes.subarray(offset, offset + 32768)); - } - return JSON.stringify({kind: 'pdf_base64', source: response.url, data: btoa(binary)}); -})() -""" - -_DOCUMENT_PAGE_READY = r""" -() => { - const viewer = document.querySelector('#viewer'); - if (viewer) return !!globalThis.PDFViewerApplication?.pdfDocument; - return !!document.querySelector('embed[src],iframe[src],object[data],a[href*=".pdf"]') - || !!document.body?.innerText?.trim(); -} -""" -_DOCUMENT_RUNTIME_FETCH_EXPRESSION = r""" -(async () => { - const response = await fetch(location.href, {credentials: 'include'}); - if (!response.ok) throw new Error(`document HTTP ${response.status}`); - const bytes = new Uint8Array(await response.arrayBuffer()); - let binary = ''; - for (let offset = 0; offset < bytes.length; offset += 32768) { - binary += String.fromCharCode(...bytes.subarray(offset, offset + 32768)); - } - return JSON.stringify({ - kind: 'document_base64', source: response.url, status: response.status, - content_type: response.headers.get('content-type'), data: btoa(binary) - }); -})() -""" +from ..task_outputs import resource_expression, resource_detail_links +from ..task_outputs import MEDIA_METADATA_EXPRESSION as _MEDIA_METADATA_EXPRESSION -_MEDIA_METADATA_EXPRESSION = r""" -(() => { - const values = []; - const add = value => { - const text = String(value || '').trim(); - if (text && !values.includes(text)) values.push(text); - }; - add(document.title); - add(document.body?.innerText); - for (const node of document.querySelectorAll( - 'meta[name="author"],meta[property$=":author"],meta[itemprop="author"],' - + '[itemprop="author"],[itemprop="creator"],[itemprop="publisher"]' - )) { - add(node.content); - add(node.getAttribute?.('content')); - add(node.innerText); - for (const name of node.querySelectorAll?.('[itemprop="name"]') || []) { - add(name.content); - add(name.getAttribute?.('content')); - add(name.innerText); - } - } - let visited = 0; - const visit = (value, depth = 0) => { - if (++visited > 2048 || depth > 6 || value == null) return; - if (Array.isArray(value)) { - for (const item of value.slice(0, 64)) visit(item, depth + 1); - return; - } - if (typeof value !== 'object') return; - for (const [key, item] of Object.entries(value).slice(0, 128)) { - if (/^(?:author|creator|publisher)$/i.test(key)) { - if (typeof item === 'string') add(item); - else if (item && typeof item === 'object') add(item.name); - } - visit(item, depth + 1); - } - }; - for (const script of [...document.querySelectorAll('script[type="application/ld+json"]')] - .slice(0, 16)) { - try { visit(JSON.parse(script.textContent)); } catch {} - } - return values.join('\n\n'); -})() -""" def _dynamic_text_expression(plan) -> str: @@ -333,56 +167,13 @@ def _dynamic_text_expression(plan) -> str: ) -_ROUTE_OUTPUTS = { - "raw_file": "raw_bytes", - "document_runtime_fetch": "runtime_document_base64+local_text", - "raw_document": "raw_document_bytes+local_text", - "document_page_runtime": "document_text_or_embedded_document", - "resource_page_runtime": "structured_resource_urls", - "challenge_json_trace": "final_dom_html+local_markdown+network_diagnostics", - "page_identity_json": "final_dom_html+local_markdown", - "dynamic_selected_text": "selected_records_or_rendered_body_fallback", - "dynamic_native_markdown": "markdown", - "media_metadata_runtime": "visible_text_and_standard_media_metadata", - "ordinary_markdown": "markdown", -} - - -def _route_for_case(case: EvalCase, *, native_default: bool = False) -> str: - """Select one output contract before applying page-family extraction rules.""" - if case.task_kind == "file": - return "raw_file" - if case.task_kind == "resource": - return "resource_page_runtime" - if case.track == "document": - path = urlsplit(case.url).path.casefold() - if path.endswith(".docx"): - return "document_runtime_fetch" - if path.endswith(".pdf") or "/pdf/" in path: - return "raw_document" - if path.endswith((".html", "/")): - return "document_page_runtime" - navigation = ((case.source_benchmark or {}).get("risk_control") or {}).get("navigation_contract") - if case.track == "challenge": - return "challenge_json_trace" - if (not native_default and case.task_kind == "body" and isinstance(navigation, dict) - and navigation.get("requested_url") and navigation.get("observed_final_url")): - return "page_identity_json" - if case.track == "dynamic" and not native_default_case(case): - return "dynamic_selected_text" - if case.track == "dynamic": - return "dynamic_native_markdown" - if case.track == "video" and case.task_kind == "body": - return "media_metadata_runtime" - return "ordinary_markdown" - - class MoliWebFetchProvider(Provider): def __init__(self, name, config): super().__init__(name, config) self._process_lock = threading.Lock() self._cancel_requested = False self._active_process = None + self._active_native_provider = None def begin_call(self): with self._process_lock: @@ -529,15 +320,16 @@ def fetch(self, case: EvalCase, timeout: float) -> ProviderResult: error_type="configuration", error_message=str(error), ) - route = _route_for_case(case, native_default=bool(self.config.get("uniform_native_parameters"))) - if retrieval_plan and (route == "raw_file" or ( - retrieval_plan.version == 2 and route != "dynamic_selected_text" + declared_text_route = (case.track == 'dynamic' and case.task_kind not in {'file', 'resource'} + and not native_default_case(case)) + if retrieval_plan and (case.task_kind == "file" or ( + retrieval_plan.version == 2 and not declared_text_route )): return ProviderResult( False, "", error_type="configuration", error_message=( "DOM completion cannot be applied to raw file delivery" - if route == "raw_file" else + if case.task_kind == "file" else "Declared text extraction requires the dynamic text route" ), ) @@ -550,140 +342,55 @@ def fetch(self, case: EvalCase, timeout: float) -> ProviderResult: return ProviderResult( False, "", error_type="configuration", error_message="; ".join(errors) ) - wait = self.config.get("wait_until", "done") - file_task = route == "raw_file" - document_task = route in {"raw_document", "document_runtime_fetch"} - document_runtime_fetch_task = route == "document_runtime_fetch" - document_page_task = route == "document_page_runtime" - media_metadata_task = route == "media_metadata_runtime" - challenge_task = route == "challenge_json_trace" - json_page_task = route in {"challenge_json_trace", "page_identity_json"} - if json_page_task: + from .headless import HeadlessProvider, output_mode_for_case + if output_mode_for_case('moli', 'headless_markdown', case) in {'document', 'file', 'document_page'}: + native = HeadlessProvider(self.name, {**self.config, 'type':'headless_markdown', + 'engine':'moli', 'binary':binary, + 'max_original_file_bytes':int(self.config.get('max_bytes',20_000_000))}) + with self._process_lock: + self._active_native_provider = native + if self._cancel_requested: native.cancel() try: - markdown_package = require_converter(self.config.get('node_html_markdown_path')) - except ValueError as error: - return ProviderResult(False, '', error_type='configuration', error_message=str(error)) - raw_task = file_task or (document_task and not document_runtime_fetch_task) + result = native.fetch(case, max(.001, call_deadline-time.monotonic())) + result.metadata.update(route='raw_file' if case.task_kind == 'file' else 'document_page_runtime' if output_mode_for_case('moli', 'headless_markdown', case) == 'document_page' else 'raw_document', + reported_version=self._reported_version, binary_sha256=self._binary_sha256) + return result + finally: + with self._process_lock: self._active_native_provider = None + wait = self.config.get("wait_until", "done") + media_metadata_task = case.track == "video" and case.task_kind == "body" timeout_ms = str(max(1, int(timeout * 1000))) - if raw_task: - # Direct file delivery uses Moli's automatic response mode. Forcing - # --dump html/markdown can reject or serialize a non-HTML response - # before the evaluator receives the original bytes. - dump = None - command = [ - binary, - "fetch", - "--timeout", - timeout_ms, - "--http-timeout", - timeout_ms, - ] - wait_metadata = None - else: - dump = "html" if case.task_kind == "resource" else "markdown" - try: - if document_runtime_fetch_task: - wait_arguments = [ - "--eval", - _DOCUMENT_RUNTIME_FETCH_EXPRESSION, - "--wait-until", - wait, - ] - wait_metadata = { - "wait_until": wait, - "document_delivery": "page_runtime_second_http_fetch", - "completion_support": "exact", - } - dump = None - elif document_page_task: - wait_arguments = [ - "--eval", - _DOCUMENT_PAGE_EXPRESSION, - "--wait-script", - _DOCUMENT_PAGE_READY, - "--wait-until", - wait, - ] - wait_metadata = { - "wait_until": wait, - "embedded_document": "page_runtime", - "completion_support": "exact", - } - dump = None - elif json_page_task: - wait_arguments = (["--trace-network"] if challenge_task else []) + ["--wait-until", wait] - wait_metadata = { - "wait_until": wait, - "extraction": "json_final_dom_then_local_markdown", - "network_trace": "metadata_without_response_bodies" if challenge_task else "not_requested", - } - dump = "json" - elif route == "resource_page_runtime": - wait_arguments = ["--eval", _RESOURCE_PAGE_EXPRESSION, "--wait-until", wait] - wait_metadata = { - "wait_until": wait, - "resource_discovery": "rendered_dom_and_same_origin_component_modules", - } - dump = None - elif route == "dynamic_selected_text": - plan = retrieval_plan - assert plan is not None - wait_arguments = ["--eval", _dynamic_text_expression(plan)] - dump = None - wait_metadata = { - "extraction": ( - "declared_scope_or_rendered_body_fallback" - if plan.version == 2 - else "selected_records_or_rendered_body_fallback" - ), - } - if plan.version == 2: - wait_metadata["extraction_scope"] = { - "selector": plan.extraction_selector, - "maximum_count": plan.extraction_maximum_count, - "frame_selector": plan.extraction_frame_selector, - "exclude_selectors": list(plan.extraction_exclude_selectors), - "fallback": plan.extraction_fallback, - } - elif media_metadata_task: - wait_arguments = [ - "--eval", - _MEDIA_METADATA_EXPRESSION, + dump = "html" if case.task_kind == "resource" else "markdown" + try: + if case.task_kind == "resource": + wait_arguments = ["--eval", resource_expression(case.track, resource_detail_links(case)), "--wait-until", wait] + wait_metadata = { + "wait_until": wait, + "resource_discovery": "rendered_dom_and_loaded_same_origin_frames", + } + dump = None + elif retrieval_plan is not None: + plan = retrieval_plan + assert plan is not None + wait_arguments = ["--eval", _dynamic_text_expression(plan)] + dump = None + wait_arguments.extend( + [ + "--wait-selector" if plan.wait_kind == "selector" else "--wait-script", + plan.wait_value, "--wait-until", - wait, + plan.wait_until, ] - wait_metadata = { - "wait_until": wait, - "extraction": "visible_text_and_standard_media_metadata", - } - dump = None - else: - wait_arguments = ["--wait-until", wait] - wait_metadata = {"wait_until": wait, **({"strategy": "native_default_completion"} if case.track == "dynamic" else {})} - except ValueError as error: - return ProviderResult( - False, - "", - error_type="configuration", - error_message=str(error), ) - if retrieval_plan: - plan = retrieval_plan - # Completion is independent of the selected output representation. - # Replace the default navigation wait without losing route flags. - if "--wait-until" in wait_arguments: - index = wait_arguments.index("--wait-until") - del wait_arguments[index:index + 2] - wait_arguments.extend([ - "--wait-selector" if plan.wait_kind == "selector" else "--wait-script", - plan.wait_value, "--wait-until", plan.wait_until, - ]) if plan.delay_ms: wait_arguments.extend(["--delay-ms", str(plan.delay_ms)]) - wait_metadata.update({ + wait_metadata = { "strategy": ( - "frame_selector_count_script" if plan.frame_selector else - "selector" if plan.wait_kind == "selector" else "selector_count_script" + "frame_selector_count_script" + if plan.frame_selector + else "selector" + if plan.wait_kind == "selector" + else "selector_count_script" ), "selector": plan.selector, "minimum_count": plan.minimum_count, @@ -692,27 +399,61 @@ def fetch(self, case: EvalCase, timeout: float) -> ProviderResult: "timeout_ms": plan.timeout_ms, "wait_until": plan.wait_until, "completion_support": "exact", - }) + "extraction": ( + "declared_scope_or_rendered_body_fallback" + if plan.version == 2 + else "selected_records_or_rendered_body_fallback" + ), + } + if plan.version == 2: + wait_metadata["extraction_scope"] = { + "selector": plan.extraction_selector, + "maximum_count": plan.extraction_maximum_count, + "frame_selector": plan.extraction_frame_selector, + "exclude_selectors": list(plan.extraction_exclude_selectors), + "fallback": plan.extraction_fallback, + } if plan.minimum_text_fragments: wait_metadata["minimum_text_fragments"] = plan.minimum_text_fragments timeout_ms = str(min(int(timeout_ms), plan.timeout_ms)) - command = [binary, "fetch"] - if dump is not None: - command.extend(["--dump", dump]) - command.extend( - [ - *wait_arguments, - "--timeout", - timeout_ms, - "--http-timeout", - timeout_ms, + elif media_metadata_task: + wait_arguments = [ + "--eval", + _MEDIA_METADATA_EXPRESSION, + "--wait-until", + wait, ] + wait_metadata = { + "wait_until": wait, + "extraction": "visible_text_and_standard_media_metadata", + } + dump = None + else: + wait_arguments = ["--wait-until", wait] + wait_metadata = {"wait_until": wait} + except ValueError as error: + return ProviderResult( + False, + "", + error_type="configuration", + error_message=str(error), ) - # Forward the common limits to the native transport as well as stdout. + command = [binary, "fetch"] + if dump is not None: + command.extend(["--dump", dump]) + command.extend( + [ + *wait_arguments, + "--timeout", + timeout_ms, + "--http-timeout", + timeout_ms, + ] + ) + # Forward common capacity limits while preserving native internal timers when declared. command.extend(["--http-connect-timeout", timeout_ms, "--http-max-response-size", str(int(self.config.get("max_bytes", 20_000_000)))]) if self.config.get("uniform_native_parameters"): - # These are product timers, not the runner's wall-clock deadline. for flag in ("--timeout", "--http-timeout", "--http-connect-timeout"): if flag in command: index = command.index(flag) @@ -737,10 +478,28 @@ def fetch(self, case: EvalCase, timeout: float) -> ProviderResult: "http_connect_timeout_ms": None if self.config.get("uniform_native_parameters") else int(timeout_ms), "internal_timeout_policy": "native_default" if self.config.get("uniform_native_parameters") else "explicit", "http_max_response_bytes": int(self.config.get("max_bytes", 20_000_000)), - "route": route, - "output": _ROUTE_OUTPUTS[route], + "route": ( + "resource_page_runtime" + if case.task_kind == "resource" + else "dynamic_selected_text" + if retrieval_plan is not None + else "dynamic_native_markdown" + if case.track == "dynamic" + else "media_metadata_runtime" + if media_metadata_task + else "ordinary_markdown" + ), + "output": ( + "structured_resource_urls" + if case.task_kind == "resource" + else "selected_records_or_rendered_body_fallback" + if retrieval_plan is not None + else "visible_text_and_standard_media_metadata" + if media_metadata_task + else "markdown" + ), "wait": wait_metadata, - "wait_until": None if raw_task else wait_metadata.get("wait_until"), + "wait_until": wait_metadata.get("wait_until"), "binary": Path(binary).name, "block_private_networks": block_private_networks, "private_network_guard_exception": private_network_guard_exception, @@ -748,11 +507,15 @@ def fetch(self, case: EvalCase, timeout: float) -> ProviderResult: requested_host if private_network_guard_exception else None ), "log_level": log_level, - "network_trace_requested": challenge_task, + "network_trace_requested": False, "local_report_complete_diagnostic_links": bool( self.config.get("local_report_complete_diagnostic_links", False) ), } + projection = (resource_expression(case.track, resource_detail_links(case)) if case.task_kind == "resource" else + _MEDIA_METADATA_EXPRESSION if media_metadata_task else None) + if projection is not None: + metadata["output_projection_sha256"] = hashlib.sha256(projection.encode("utf-8")).hexdigest() try: if hasattr(self, "_binary_identity"): stat = Path(binary).stat() @@ -817,7 +580,7 @@ def fetch(self, case: EvalCase, timeout: float) -> ProviderResult: with self._process_lock: self._active_process = process if self._cancel_requested: - self._kill(process) + terminate_owned_process_tree(process) except OSError as error: return ProviderResult( False, @@ -849,7 +612,7 @@ def fetch(self, case: EvalCase, timeout: float) -> ProviderResult: except subprocess.TimeoutExpired: pass if stopped: - self._kill(process) + terminate_owned_process_tree(process) process.wait() if dns_broker is not None: metadata["dns_queries"] = list(dns_broker.traces) @@ -912,342 +675,6 @@ def fetch(self, case: EvalCase, timeout: float) -> ProviderResult: error_message="Moli fetch exceeded time budget", metadata=metadata, ) - if file_task: - ok = process.returncode == 0 - return ProviderResult( - ok, - "", - files=[{"data": raw}] if raw else [], - error_type=None if ok else "tool_error", - error_message=None if ok else stderr or "Moli file request failed", - metadata=metadata, - ) - if document_runtime_fetch_task: - if process.returncode != 0: - return ProviderResult( - False, - "", - error_type="tool_error", - error_message=stderr or "Moli document runtime fetch failed", - metadata=metadata, - ) - try: - envelope = json.loads(raw.decode("utf-8")) - if not isinstance(envelope, dict) or envelope.get("kind") != "document_base64": - raise ValueError("unknown document envelope kind") - document_bytes = base64.b64decode(envelope["data"], validate=True) - if not document_bytes.startswith(b"PK"): - raise ValueError("runtime response is not an OOXML ZIP document") - except ( - UnicodeDecodeError, - json.JSONDecodeError, - KeyError, - TypeError, - ValueError, - ) as error: - metadata.update( - raw_stdout_sha256=hashlib.sha256(raw).hexdigest(), - _raw_stdout_b64=base64.b64encode(raw).decode("ascii"), - ) - return ProviderResult( - False, - "", - error_type="invalid_output", - error_message=f"Moli document runtime output is invalid: {error}", - metadata=metadata, - ) - metadata.update( - document_runtime_source=envelope.get("source"), - document_runtime_status=envelope.get("status"), - document_runtime_content_type=envelope.get("content_type"), - document_runtime_second_http_fetch=True, - raw_stdout_sha256=hashlib.sha256(document_bytes).hexdigest(), - _raw_stdout_b64=base64.b64encode(document_bytes).decode("ascii"), - ) - try: - content, extractor = extract_document_text(document_bytes) - except Exception as error: - return ProviderResult( - False, - "", - error_type="document_parse_error", - error_message=str(error), - metadata=metadata, - ) - metadata["document_extractor"] = extractor - if time.monotonic() >= call_deadline: - return ProviderResult( - False, - "", - error_type="timeout", - error_message="Moli document runtime extraction exceeded time budget", - metadata=metadata, - ) - ok = True - status = envelope.get("status") - return ProviderResult( - ok, - content, - status_code=status if isinstance(status, int) else None, - final_url=envelope.get("source"), - error_type=None, - error_message=None, - metadata=metadata, - ) - if document_task and raw.startswith((b"%PDF", b"PK")): - try: - content, extractor = extract_document_text(raw) - except Exception as error: - return ProviderResult( - False, - "", - error_type="document_parse_error", - error_message=str(error), - metadata=metadata, - ) - metadata.update( - document_extractor=extractor, - raw_stdout_sha256=hashlib.sha256(raw).hexdigest(), - _raw_stdout_b64=base64.b64encode(raw).decode("ascii"), - ) - if time.monotonic() >= call_deadline: - return ProviderResult( - False, - "", - error_type="timeout", - error_message="Moli document extraction exceeded time budget", - metadata=metadata, - ) - ok = process.returncode == 0 - return ProviderResult( - ok, - content, - error_type=None if ok else "tool_error", - error_message=None if ok else stderr or "Moli document request failed", - metadata=metadata, - ) - if document_task: - metadata.update( - document_response="serialized_text_not_raw_document", - raw_stdout_sha256=hashlib.sha256(raw).hexdigest(), - _raw_stdout_b64=base64.b64encode(raw).decode("ascii"), - ) - return ProviderResult( - False, - "", - error_type="invalid_output" if process.returncode == 0 else "tool_error", - error_message=( - "Moli serialized a direct document instead of returning raw bytes" - if process.returncode == 0 - else stderr or "Moli did not return raw document bytes" - ), - metadata=metadata, - ) - if document_page_task: - if process.returncode != 0: - return ProviderResult( - False, - "", - error_type="tool_error", - error_message=stderr or "Moli document-page extraction failed", - metadata=metadata, - ) - try: - envelope = json.loads(raw.decode("utf-8")) - except (UnicodeDecodeError, json.JSONDecodeError) as error: - return ProviderResult( - False, - "", - error_type="invalid_output", - error_message=f"Moli document-page output is invalid JSON: {error}", - metadata=metadata, - ) - metadata["embedded_document_url"] = envelope.get("source") - if envelope.get("kind") == "pdf_base64": - try: - embedded = base64.b64decode(envelope["data"], validate=True) - content, extractor = extract_document_text(embedded) - except Exception as error: - return ProviderResult( - False, - "", - error_type="document_parse_error", - error_message=str(error), - metadata=metadata, - ) - metadata.update( - document_extractor=extractor, - raw_stdout_sha256=hashlib.sha256(embedded).hexdigest(), - _raw_stdout_b64=base64.b64encode(embedded).decode("ascii"), - ) - elif envelope.get("kind") in {"text", "page_text"}: - content = str(envelope.get("text") or "") - metadata["document_extractor"] = ( - "pdfjs_runtime" if envelope.get("kind") == "text" else "rendered_page_text" - ) - else: - return ProviderResult( - False, - "", - error_type="invalid_output", - error_message="Moli document-page output has an unknown kind", - metadata=metadata, - ) - if time.monotonic() >= call_deadline: - return ProviderResult( - False, - "", - error_type="timeout", - error_message="Moli document-page extraction exceeded time budget", - metadata=metadata, - ) - ok = True - return ProviderResult( - ok, - content, - error_type=None, - error_message=None, - metadata=metadata, - ) - if json_page_task: - metadata["final_url_source"] = "same_call_json" - metadata.update( - raw_stdout_sha256=hashlib.sha256(raw).hexdigest(), - _raw_stdout_b64=base64.b64encode(raw).decode("ascii"), - ) - try: - envelope = json.loads(raw.decode("utf-8")) - except (UnicodeDecodeError, json.JSONDecodeError) as error: - return ProviderResult( - False, - "", - error_type="tool_error" if process.returncode else "invalid_output", - error_message=(stderr or "Moli page request failed") if process.returncode - else f"Moli page JSON is invalid: {error}", - metadata=metadata, - ) - if not isinstance(envelope, dict) or not isinstance(envelope.get("html"), str): - return ProviderResult( - False, - "", - status_code=envelope.get("status") if isinstance(envelope, dict) else None, - final_url=envelope.get("final_url") if isinstance(envelope, dict) else None, - title=envelope.get("title") if isinstance(envelope, dict) else None, - error_type="invalid_output", - error_message="Moli page JSON omitted final DOM HTML", - metadata=metadata, - ) - trace = envelope.get("network") if isinstance(envelope.get("network"), dict) else {} - main = ( - trace.get("main_document") if isinstance(trace.get("main_document"), dict) else {} - ) - subresources = ( - trace.get("subresources") if isinstance(trace.get("subresources"), list) else [] - ) - metadata["network_trace"] = { - "main_document": { - key: main.get(key) - for key in ( - "url", - "requested_url", - "status", - "content_type", - "redirected", - "redirect_count", - ) - if key in main - }, - "subresource_count": len(subresources), - "failed_subresource_count": sum( - isinstance(item, dict) and item.get("ok") is False for item in subresources - ), - "response_header_names": sorted( - { - str(item.get("name", "")).casefold() - for item in envelope.get("headers", []) - if isinstance(item, dict) and item.get("name") - } - ), - "redirect_chain": [ - {key: item.get(key) for key in ("from_url", "to_url", "status")} - for item in envelope.get("redirect_chain", [])[:20] - if isinstance(item, dict) - ], - } - remaining = call_deadline - time.monotonic() - if remaining <= 0: - return ProviderResult( - False, - "", - status_code=envelope.get("status"), - final_url=envelope.get("final_url"), - title=envelope.get("title"), - error_type="timeout", - error_message="Moli page conversion exceeded time budget", - metadata=metadata, - ) - try: - metadata['converter_sha256'] = hashlib.sha256(CONVERTER.read_bytes()).hexdigest() - metadata['extractor_version'] = '2.0.0' - converted = subprocess.run( - converter_command(markdown_package), - input=envelope["html"].encode("utf-8"), - capture_output=True, - timeout=remaining, - check=False, - ) - except subprocess.TimeoutExpired: - return ProviderResult( - False, - "", - status_code=envelope.get("status"), - final_url=envelope.get("final_url"), - title=envelope.get("title"), - error_type="timeout", - error_message="Moli page conversion exceeded time budget", - metadata=metadata, - ) - if converted.returncode != 0 or len(converted.stdout) > limit: - return ProviderResult( - False, - "", - status_code=envelope.get("status"), - final_url=envelope.get("final_url"), - title=envelope.get("title"), - error_type="conversion_error" if converted.returncode else "output_too_large", - error_message=( - converted.stderr.decode("utf-8", "replace")[:2000] - if converted.returncode - else "Converted page output exceeds max_bytes" - ), - metadata=metadata, - ) - content = converted.stdout.decode("utf-8", "replace") - status = envelope.get("status") - http_ok = isinstance(status, int) and 200 <= status < 400 - ok = process.returncode == 0 and http_ok - return ProviderResult( - ok, - content, - status_code=status if isinstance(status, int) else None, - final_url=envelope.get("final_url"), - title=envelope.get("title"), - error_type=( - None - if ok - else "api_error" - if isinstance(status, int) and status >= 400 - else "tool_error" - ), - error_message=( - None - if ok - else stderr or f"Moli returned HTTP {status}" - if isinstance(status, int) and status >= 400 - else stderr or "Moli challenge request failed" - ), - metadata=metadata, - ) try: content = raw.decode("utf-8") except UnicodeDecodeError: @@ -1289,27 +716,9 @@ def cancel(self) -> None: # thread on process creation. The launching thread kills a late child. with self._process_lock: self._cancel_requested = True - self._kill(self._active_process) + terminate_owned_process_tree(self._active_process) + if self._active_native_provider is not None: self._active_native_provider.cancel() @property def cancellation_cleanup_seconds(self) -> float: return 1.0 - - @staticmethod - def _kill(process): - if process is not None and process.poll() is None: - if os.name == "posix": - try: - os.killpg(process.pid, signal.SIGKILL) - return - except ProcessLookupError: - return - except OSError: - # Sandboxes can forbid process-group signalling while - # still allowing the direct child to be terminated. - pass - try: - if process.poll() is None: - process.kill() - except OSError: - pass diff --git a/harness/web_fetch/publication_privacy.py b/harness/web_fetch/publication_privacy.py index 20e1dcfa3..72fc4b992 100644 --- a/harness/web_fetch/publication_privacy.py +++ b/harness/web_fetch/publication_privacy.py @@ -104,10 +104,17 @@ def _redact_plain_url_queries(text): def redact_url(match): nonlocal replacements - url = match.group() + original = match.group() + decoded = html.unescape(original) + bounded = EMBEDDED_URL.match(decoded) + if bounded is None: + return original + url = bounded.group() + suffix = decoded[len(url):] query = url.partition('?')[2].partition('#')[0] if not query: - return url + return original + prior_replacements = replacements parts = [] for item in query.split('&'): key, separator, value = item.partition('=') @@ -122,7 +129,10 @@ def redact_url(match): item = key + '=[REDACTED]' replacements += 1 parts.append(item) - return url.replace('?' + query, '?' + '&'.join(parts), 1) + if replacements == prior_replacements: + return original + projected = url.replace('?' + query, '?' + '&'.join(parts), 1) + suffix + return html.escape(projected, quote=True) if decoded != original else projected return EMBEDDED_URL.sub(redact_url, text), replacements @@ -188,12 +198,18 @@ def publication_risk(data): def project_file(name, data): - """Redact only auxiliary HTML. Any match elsewhere requires human review.""" + """Project public copies and bind their hashes to retained original evidence.""" path = PurePosixPath(name) if 'private_artifacts' in path.parts: raise ValueError('Private evidence is not publishable: ' + name) if any(part.startswith('.') for part in path.parts): raise ValueError('Private/hidden file is not publishable: ' + name) + if OTHER_SECRET.search(data): + raise ValueError('Potential credential-shaped value in publication file: ' + name) + if name.endswith(('.json', '.jsonl')): + structured = _project_json(name, data) + if structured is not None: + return structured original = data inline_replacements = 0 inline_kinds = set() @@ -242,3 +258,57 @@ def project_pre(match): 'original_bytes': len(original), 'published_bytes': len(projected), } + + +def _project_json(name, data): + """Project decoded values so header text cannot consume JSON delimiters.""" + lines = data.splitlines(keepends=True) if name.endswith('.jsonl') else [data] + try: + documents = [json.loads(line) if line.strip() else None for line in lines] + except (ValueError, UnicodeError): + return None + replacements = 0 + kinds = set() + + def project(value): + nonlocal replacements + if isinstance(value, dict): + result = {} + for key, child in value.items(): + if (isinstance(child, str) and child + and re.fullmatch(SENSITIVE_KEY, key.encode(), re.I) + and not child.startswith(('<', '[')) + and not PUBLIC_AUTH_PLACEHOLDER.fullmatch((key+': '+child).encode())): + result[key] = '[REDACTED]' + replacements += 1 + kinds.add('credential') + else: + result[key] = project(child) + return result + if isinstance(value, list): + return [project(child) for child in value] + if isinstance(value, str): + raw, receipt = project_file(name+'.value.txt', value.encode()) + if receipt: + replacements += receipt['replacements'] + kinds.update(receipt['kinds']) + return raw.decode() + return value + + output = [] + for raw, document in zip(lines, documents): + revised = project(document) + if revised == document: + output.append(raw) + else: + ending = b'\r\n' if raw.endswith(b'\r\n') else b'\n' if raw.endswith(b'\n') else b'' + output.append(json.dumps(revised, ensure_ascii=True).encode()+ending) + projected = b''.join(output) + if projected == data: + return data, None + return projected, {'path': name, 'kinds': sorted(kinds), + 'reason': 'sensitive values in decoded JSON; JSON structure and non-sensitive fields retained', + 'replacements': replacements, + 'original_sha256': hashlib.sha256(data).hexdigest(), + 'published_sha256': hashlib.sha256(projected).hexdigest(), + 'original_bytes': len(data), 'published_bytes': len(projected)} diff --git a/harness/web_fetch/reference_freeze.py b/harness/web_fetch/reference_freeze.py index 8c0ea7715..8696b0669 100644 --- a/harness/web_fetch/reference_freeze.py +++ b/harness/web_fetch/reference_freeze.py @@ -118,7 +118,7 @@ def _require_promotion(path, *, dataset_sha256, capture_audit_sha256, def select_freeze_cases(dataset, case_ids=None, exclude_case_ids=None): """Return the exact requested subset; shared by prepare and freeze.""" selected=deepcopy(dataset) - selected.setdefault('comparison',{})['require_independent_result_review']=True + selected.setdefault('comparison',{}) if case_ids: wanted=set(case_ids.split(',')) selected['cases']=[case for case in selected['cases'] if case['id'] in wanted] @@ -161,7 +161,7 @@ def load_browser_observations(dataset, path=None, directory=None): return observations -def compute_reference_update(dataset, captures, observations=None): +def compute_reference_update(dataset, captures, observations=None, *, refresh_case_ids=()): """Compute the one authoritative frozen dataset/change set from captures.""" from harness.web_fetch.reference_lifecycle import _completion @@ -171,7 +171,13 @@ def compute_reference_update(dataset, captures, observations=None): changes=[];contexts={} for case in frozen['cases']: cid=case['id'] - refresh=requires_fresh_reference(case) + refresh=(requires_fresh_reference(case) or ( + cid in refresh_case_ids + and case.get('track','web') in {'web','dynamic','challenge'} + and case.get('task_kind','body')=='body' + and not case.get('acceptance_contract') + and bool(case.get('source_verification')) + and bool((case.get('ground_truth_main_content') or '').strip()))) if not refresh: # A previously unavailable body reference must be allowed to # recover when a later independent HTTP capture is complete. Do @@ -297,8 +303,11 @@ def main(argv=None): args.exclude_case_ids) observations=load_browser_observations( dataset,args.browser_observations,args.browser_observation_directory) + refresh_case_ids = ({change['case_id'] for change in + json.loads(args.reference_promotion.read_text()).get('changes', [])} + if args.reference_promotion else set()) dataset,changes,contexts=compute_reference_update( - dataset,args.captures,observations) + dataset,args.captures,observations, refresh_case_ids=refresh_case_ids) except (OSError,KeyError,ValueError,json.JSONDecodeError) as error: p.error(str(error)) audit_path=args.captures/'audit.json' @@ -318,7 +327,7 @@ def main(argv=None): args.output.mkdir(parents=True) materialize_task_references(args.output, input_trees, [case['id'] for case in dataset['cases']]) for case in dataset['cases']: - if not requires_fresh_reference(case): + if case['id'] not in contexts: continue cid = case['id']; target = args.output/'sources'/cid target.mkdir(parents=True, exist_ok=True) @@ -457,7 +466,8 @@ def prepare_promotion(*,dataset_path,captures,output,approved_by,approved_at, dataset=select_freeze_cases(json.loads(dataset_bytes),case_ids,exclude_case_ids) observations=load_browser_observations( dataset,browser_observations,browser_observation_directory) - candidate,changes,_=compute_reference_update(dataset,captures,observations) + candidate,changes,_=compute_reference_update(dataset,captures,observations, + refresh_case_ids=[case['id'] for case in dataset['cases']]) if not changes: raise ValueError('Captured reference scope is unchanged; no promotion is needed') audit_bytes=(captures/'audit.json').read_bytes() @@ -548,7 +558,7 @@ def promote(*,canonical_path,candidate_path,captures,receipt_path,receipt_target observations=load_browser_observations( selected,browser_observations,browser_observation_directory) expected,changes,contexts=compute_reference_update( - selected,captures,observations) + selected,captures,observations, refresh_case_ids=case_ids) expected_bytes=(json.dumps(expected,ensure_ascii=False,indent=2)+'\n').encode() candidate_bytes=candidate_path.read_bytes() if candidate_bytes!=expected_bytes: diff --git a/harness/web_fetch/reference_lifecycle.py b/harness/web_fetch/reference_lifecycle.py index 578e01864..13affaa39 100644 --- a/harness/web_fetch/reference_lifecycle.py +++ b/harness/web_fetch/reference_lifecycle.py @@ -81,29 +81,22 @@ def _not_future(value, now: datetime) -> datetime: return checked -def source_unqualified_contract_cases(cases) -> list[str]: - """Invariant rule validation is not current source-solvability evidence. - - The generic lifecycle does not currently qualify live acceptance contracts. - Specialized frozen-input tracks have their own certificates and boundaries. - """ - return [_get(case, 'id') for case in cases if _get(case, 'acceptance_contract', {})] - - def requires_fresh_reference(case) -> bool: + """Existing answers stay frozen unless the task explicitly selects live checks.""" contract = _get(case, "acceptance_contract", {}) if contract: - # This mode evaluates invariant rules, not equality with live values. - # Validate before choosing it; snapshot-bound runs retain all checks. from .stable_tasks import validate_acceptance_contract validate_acceptance_contract(contract) return False - # Tiny synthetic fixtures intentionally have no independent verification. - # Production body cases carry this metadata, including invalid attestations. - return (_get(case, "track", "web") in {"web", "dynamic", "challenge"} + verification = _get(case, "source_verification", {}) or {} + mode = verification.get("reference_mode", "frozen") + if mode not in {"frozen", "live_exact"}: + raise ReferenceLifecycleError("Unknown reference mode; use frozen or live_exact") + return (mode == "live_exact" + and _get(case, "track", "web") in {"web", "dynamic", "challenge"} and _get(case, "task_kind", "body") == "body" and bool((_get(case, "ground_truth_main_content", "") or "").strip()) - and bool(_get(case, "source_verification", {}))) + and bool(verification)) def expected_providers(dataset, case): @@ -881,6 +874,34 @@ def revision_edge(value): return before, after reference_edges = set() + presentation_path = root/'report-binding-revisions.json' + if presentation_path.is_file(): + try: + presentation_rows = json.loads(presentation_path.read_text()) + if not isinstance(presentation_rows, list): + return False + for receipt in presentation_rows: + edge = revision_edge(receipt) + if (edge is None or receipt.get('kind') != 'report_interpretation_binding_revision' + or receipt.get('provider_calls') != 0 or edge[0][1] != edge[1][1]): + return False + snapshots = [] + for side, binding in zip(('input', 'output'), edge): + raw = contained_path(root, receipt[f'{side}_dataset_path']).read_bytes() + if sha256(raw) != binding[0]: + return False + document = json.loads(raw) + if not isinstance(document, dict): + return False + document.get('comparison', {}).pop('report_interpretation', None) + if not document.get('comparison'): + document.pop('comparison', None) + snapshots.append(document) + if snapshots[0] != snapshots[1]: + return False + reference_edges.add(edge) + except (OSError, ValueError, TypeError, KeyError): + return False reference_path = root/'reference-revisions.json' if reference_path.is_file(): try: diff --git a/harness/web_fetch/reference_quality.py b/harness/web_fetch/reference_quality.py index 1e1a456ce..a490e32ff 100644 --- a/harness/web_fetch/reference_quality.py +++ b/harness/web_fetch/reference_quality.py @@ -1,7 +1,7 @@ """Deterministic reference preparation; original source bytes remain evidence.""" import re -from dataclasses import dataclass import unicodedata +from dataclasses import dataclass def clean_reference_text(text: str) -> str: @@ -89,26 +89,41 @@ class MarkdownLink: _MAX_MARKDOWN_LINK_SPAN = 8192 -def _closing(text: str, start: int, opening: str, closing: str) -> int | None: - depth = 1 - index = start + 1 - limit = min(len(text), start + _MAX_MARKDOWN_LINK_SPAN) - while index < limit: - if text[index] == "\\": - index += 2 +def markdown_delimiter_pairs(text, *, maximum_span=None): + """Index escaped, nested Markdown delimiters once, including malformed prose.""" + pairs = {} + stacks = {'[': [], '(': []} + escaped_starts = {'[': {}, '(': {}} + openings = {']': '[', ')': '('} + i = 0 + while i < len(text): + char = text[i] + if char == "\\": + # Existing callers may begin at an escaped opening delimiter. + # Such an opening does not nest inside another candidate, but it + # shares that depth's next close when parsed as the link start. + if i + 1 < len(text) and text[i + 1] in stacks: + opening = text[i + 1] + escaped_starts[opening].setdefault(len(stacks[opening]), []).append(i + 1) + i += 2 continue - if text[index] == opening: - depth += 1 - elif text[index] == closing: - depth -= 1 - if not depth: - return index - index += 1 - return None + if char in stacks: + stacks[char].append(i) + elif char in openings: + opening = openings[char] + starts = escaped_starts[opening].pop(len(stacks[opening]), []) + if stacks[opening]: + starts.append(stacks[opening].pop()) + for start in starts: + if maximum_span is None or i - start < maximum_span: + pairs[start] = i + i += 1 + return pairs def markdown_links(text: str) -> list[MarkdownLink]: """Return explicit balanced Markdown links without guessing from bare URLs.""" + pairs = markdown_delimiter_pairs(text, maximum_span=_MAX_MARKDOWN_LINK_SPAN) links: list[MarkdownLink] = [] index = 0 while index < len(text): @@ -117,7 +132,7 @@ def markdown_links(text: str) -> list[MarkdownLink]: if text[start:start + 1] != "[": index += 1 continue - label_end = _closing(text, start, "[", "]") + label_end = pairs.get(start) if label_end is None: index += 1 continue @@ -127,7 +142,7 @@ def markdown_links(text: str) -> list[MarkdownLink]: if text[destination_start:destination_start + 1] != "(": index += 1 continue - destination_end = _closing(text, destination_start, "(", ")") + destination_end = pairs.get(destination_start) if destination_end is None: index += 1 continue diff --git a/harness/web_fetch/report.py b/harness/web_fetch/report.py index 99eb60df0..96a94651a 100644 --- a/harness/web_fetch/report.py +++ b/harness/web_fetch/report.py @@ -15,7 +15,7 @@ from .accounts import report_access_registry from .failure_diagnosis import diagnose -from .methods import ROUTE_LABELS, method_page_model +from .methods import ROUTE_LABELS, method_page_model, is_browser_comparison, browser_method_label from .metrics import percentile, task_measured, task_passed from .models import TRACKS, execution_id from .report_components import ( @@ -31,7 +31,7 @@ tabs, task_instruction, ) -from .report_interfaces import HEADLESS_ACCESS +from .report_interfaces import HEADLESS_ACCESS, method_access, method_version, method_provenance from .report_methods import PROFILES, adoption_counts from .report_resources import render_resource_comparison from .report_style import STYLE @@ -298,7 +298,7 @@ def score_digits(values, direction='max'): page(lang,f'cases/{case}.html',case,body) from harness.report_languages import write_language_contract write_language_contract(out, include_english=include_english) -def provider_label(name):return PROVIDER_LABELS.get(name,name) +def provider_label(name):return PROVIDER_LABELS.get(name,browser_method_label(name)) def case_track_label(case): if case.get('track','web')=='web': return {'file':'文件交付','resource':'资源发现','structure':'结构化内容'}.get(case.get('task_kind','body'),TRACKS['web']) @@ -317,7 +317,7 @@ def active_tracks(cases): def track_count_label(count):return str(count) def is_headless_cohort(records): - return bool(records) and all(r.get('provider_type') in {'headless_html','headless_markdown'} for r in records) + return is_browser_comparison(records) def provider_configuration(name,records,info): """Compatibility accessor for the generic method-page model.""" @@ -356,7 +356,7 @@ def provider_page_body(name,records,cases,headers,tracks,*,native_default=False) function closeCallLog(button){let dialog=button.closest('dialog');if(!dialog)return;if(typeof dialog.close==='function')dialog.close();else{dialog.removeAttribute('open');let trigger=callLogTrigger;callLogTrigger=null;trigger?.focus()}} function applyReportAnchor(anchor){if(!anchor)return;if(wmbInspect(anchor))return;if(anchor.startsWith('track-')){chooseTab('dataset-hub','cases');let s=document.querySelector('select[data-filter="track"]');if(s){s.value=anchor.slice(6);document.querySelectorAll('select[data-filter]').forEach(x=>{if(x!==s)x.value=''});cascadeFilters(s)}}else if(['references','sources'].includes(anchor)){chooseTab('dataset-hub',anchor)}else{let target=document.getElementById(anchor);for(let p=target;p;p=p.parentElement){if(p.dataset?.panel)chooseTab(p.parentElement.id,p.dataset.panel);if(p.tagName==='DETAILS')p.open=true}}} function wmbFilters(){let table=document.querySelector('[data-wmb-table]');if(!table)return;let rows=table.tBodies[0].rows,controls=[...document.querySelectorAll('[data-wmb-filter]')],values=Object.fromEntries(controls.map(c=>[c.dataset.wmbFilter,c.value])),axis=values.axis||'',query=(values.query||'').trim().toLowerCase();let category=controls.find(c=>c.dataset.wmbFilter==='category');if(category){let allowed=new Set();for(let row of rows)if(!values.method||row.dataset.method===values.method)allowed.add(row.dataset[axis]);for(let option of category.options)option.disabled=!!option.value&&!allowed.has(option.value);if(category.value&&!allowed.has(category.value)){category.value='';values.category=''}}let failure=controls.find(c=>c.dataset.wmbFilter==='failure');if(failure){let available=new Set();for(let row of rows)if((!values.method||row.dataset.method===values.method)&&(!values.request||row.dataset.request===values.request)&&row.dataset.failure)available.add(row.dataset.failure);for(let option of failure.options)option.disabled=!!option.value&&!available.has(option.value);failure.disabled=values.request==='success'||available.size===0;if(failure.disabled||failure.value&&!available.has(failure.value)){failure.value='';values.failure=''}}let band=controls.find(c=>c.dataset.wmbFilter==='band');if(band){band.disabled=!values.metric;if(band.disabled){band.value='';values.band=''}}let count=0;for(let row of rows){let search=row.dataset.searchNormalized||(row.dataset.searchNormalized=(row.dataset.search||'').toLowerCase());let show=(!values.method||row.dataset.method===values.method)&&(!axis||!values.category||row.dataset[axis]===values.category)&&(!values.request||row.dataset.request===values.request)&&(!values.failure||row.dataset.failure===values.failure)&&(!query||search.includes(query));if(show&&values.metric){let raw=row.dataset[values.metric],number=raw===''||raw==null?null:Number(raw);if(values.band==='unavailable')show=number===null;else if(values.band){let [low,high]=values.band.split(':').map(Number);show=number!==null&&number>=low&&number<(high===1?1.0000001:high)}}row.hidden=!show;if(show){count++;row.cells[0].textContent=count}}let indicator=document.querySelector('[data-wmb-count]');if(indicator)indicator.textContent=tr(`显示 ${count} / ${rows.length} 条`,`Showing ${count} / ${rows.length}`)} -function wmbInspect(anchor){if(!anchor.startsWith('inspect?'))return false;let params=new URLSearchParams(anchor.slice(8).replace(/\+/g,'%2B'));document.querySelectorAll('[data-wmb-filter]').forEach(c=>{let value=params.get(c.dataset.wmbFilter);if(value!==null)c.value=value});wmbFilters();(document.querySelector('.wmb-controls')||document.querySelector('[data-wmb-table]')?.closest('section'))?.scrollIntoView({block:'start'});return true} +function wmbInspect(anchor){if(!anchor.startsWith('inspect?'))return false;let params=new URLSearchParams(anchor.slice(8).replace(/\+/g,'%2B'));let regular=[...document.querySelectorAll('[data-filter]')];if(regular.length){regular.forEach(c=>c.value=params.get(c.dataset.filter)||'');cascadeFilters();document.querySelector('.filters')?.scrollIntoView({block:'start'});return true}document.querySelectorAll('[data-wmb-filter]').forEach(c=>{let value=params.get(c.dataset.wmbFilter);if(value!==null)c.value=value});wmbFilters();(document.querySelector('.wmb-controls')||document.querySelector('[data-wmb-table]')?.closest('section'))?.scrollIntoView({block:'start'});return true} function wmbReset(){document.querySelectorAll('[data-wmb-filter]').forEach(c=>{c.value=c.dataset.wmbDefault||''});wmbFilters()} function wmbSliceFilters(){let table=document.getElementById('wmb-slices'),axis=document.getElementById('wmb-slice-axis'),category=document.getElementById('wmb-slice-category'),method=document.getElementById('wmb-slice-method'),count=document.getElementById('wmb-slice-count');if(!table||!axis||!category||!method||!count)return;let rows=[...table.tBodies[0].rows],available=new Set(rows.filter(r=>r.dataset.axis===axis.value&&(!method.value||r.dataset.method===method.value)).map(r=>r.dataset.category));for(let option of category.options)option.disabled=!!option.value&&!available.has(option.value);if(category.value&&!available.has(category.value))category.value='';let shown=0;for(let row of rows){let visible=row.dataset.axis===axis.value&&(!category.value||row.dataset.category===category.value)&&(!method.value||row.dataset.method===method.value);row.hidden=!visible;if(visible)row.cells[0].textContent=++shown}count.textContent=tr(`显示 ${shown} / ${rows.length} 条分类结果`,`Showing ${shown} / ${rows.length} category results`)} function wmbDistribution(){let table=document.getElementById('wmb-distribution'),axis=document.getElementById('wmb-distribution-axis'),count=document.getElementById('wmb-distribution-count');if(!table||!axis||!count)return;let rows=[...table.tBodies[0].rows],shown=0,parent=0,sample=0;for(let row of rows){let visible=row.dataset.axis===axis.value;row.hidden=!visible;if(visible){row.cells[0].textContent=++shown;parent+=Number(row.dataset.parent||0);sample+=Number(row.dataset.sample||0)}}count.textContent=tr(`${shown} 个分类 · 母集合计 ${parent} · 样本合计 ${sample}`,`${shown} categories · parent total ${parent} · sample total ${sample}`)} @@ -635,8 +635,7 @@ def run_log_section(out,record,output): native_text,caller_text=_separate_observed_logs(debug_bytes.decode("utf-8", "replace")) native_view=('
'+esc(native_text)+'
' if native_text else '

本次没有取得产品或依赖库的原生日志。已保存的调用方记录见下方。

') diagnostics=[f'

实际诊断日志 · 产品与依赖库

{esc(metadata.get("public_log_note") or "仅展示真实观测记录；未开放的内部过程不作推断。")}

返回 JSON 在“原始返回”中单独展示；接入程序记录在下方“调用方事件”中。

{native_view}
调用方事件（测试程序记录）
{esc(caller_text)}

下载运行时日志（已分离返回内容） · 公开 SHA-256：{esc(metadata["public_debug_log_sha256"])} · 原始采集记录 {int(metadata.get("diagnostic_log_bytes") or 0):,} 字节（可含协议返回） · 原始 SHA-256：{esc(metadata.get("diagnostic_log_sha256") or "—")}

'] - version=(metadata.get('reported_version') or metadata.get('binary_version') or metadata.get('browser_version') or - metadata.get('moli_version') or metadata.get('version') or '—') + version=method_version(record,method_provenance(out)) or '—' command=metadata.get('command_args') or metadata.get('command') or '—（调用记录未保存可展示的完整命令）' if isinstance(command,list):command=' '.join(str(item) for item in command) status='成功' if record.get('ok') else '失败' @@ -646,7 +645,7 @@ def run_log_section(out,record,output): if any_stderr else '0 字节') fields=(f'
调用状态
{status}
耗时
{ms(record.get("elapsed_ms"))}
' f'
退出码
{esc(process.get("exit_code",metadata.get("exit_code","—")))}
' - f'
Markdown 输出
{output_bytes:,} 字节
运行日志
{top_log_size}
' + f'
原始输出
{output_bytes:,} 字节
运行日志
{top_log_size}
' f'
日志是否完整
{complete_label}
运行 ID
{esc(record.get("run_id") or execution_id(record) or "—")}
' f'
工具版本
{esc(version)}
') full_fields=(f'
开始时间
{esc(record.get("started_at") or timing.get("started_at") or "—")}
' @@ -655,6 +654,9 @@ def run_log_section(out,record,output): f'
公开日志
{top_log_size}
公开日志脱敏
{"已隐藏凭据与本机路径" if public_debug else "否"}
' f'
方法 SHA-256
{esc(metadata.get("binary_sha256") or "—")}
' f'
正式命令
{esc(command)}
') + runtime_error='' + if not record.get('ok') or record.get('error_type'): + runtime_error='

程序错误

分类：'+esc(record.get('error_type') or '未记录')+'

'+esc(record.get('error_message') or '调用记录未提供错误文字。')+'
' diagnostic=''.join(diagnostics) if diagnostics else '

这次历史调用没有保存可展示的 stderr / debug 日志；不会补造不存在的日志。

' raw_record=' · '.join(f'下载{esc(log["label"])}记录' for log in logs) if artifact_relative: @@ -681,7 +683,7 @@ def run_log_section(out,record,output): f'（{diagnostic_size:,} 字节 · SHA-256：{esc(diagnostic_sha)} · 截断：{diagnostic_truncated}）') if not raw_record:raw_record='这次历史调用没有可下载的原始记录' return (f'

运行日志与证据

' - f'
{fields}
' + f'
{fields}
{runtime_error}' f'
查看完整运行字段
{full_fields}
' f'

结论说明：{result_note}

' f'
查看本次真实日志
{diagnostic}
' @@ -693,7 +695,37 @@ def link(url,label):return f'{esc(label)}' def cell(value,sort=None):return '{value}' def rate(a,b):return pct(a/b) if b else '—' def section(title,body,id=''):return '

{title}

{body}
' -def hero(title,desc):return f'
Web Fetch · 评测报告

{title}

{desc}

' +def hero(title,desc): + label='合成示例' if '合成示例 · 非正式评测' in _RELIABILITY_BANNER else '诊断运行' if '诊断运行 · 非正式评测' in _RELIABILITY_BANNER else '实测报告' + return f'
Web Fetch · {label}

{title}

{desc}

' +def comparative_rows(rows, directions): + """Mark raw-value column extrema without ranking unavailable values or ties.""" + matches=[list(re.finditer(r']*>.*?',values,re.S)) for _,values in rows] + numbers={} + for index,(metric,direction) in directions.items(): + values=[] + for cells in matches: + match=re.search(r'data-sort="([^"]+)"',cells[index].group()) + value=float(match.group(1)) if match else None + values.append(value if value is not None and value>=0 else None) + valid=[v for v in values if v is not None] + extreme=(max(valid) if direction=='max' else min(valid)) if valid else None + numbers[index]=(values,extreme,valid.count(extreme)==1) + result=[] + for row_number,((attrs,values),cells) in enumerate(zip(rows,matches)): + for index in sorted(directions,reverse=True): + metric,direction=directions[index];column,extreme,unique=numbers[index] + fragment=cells[index].group();opening,body=fragment.split('>',1) + if column[row_number] is None:opening=re.sub(r' data-sort="[^"]+"','',opening) + opening+=' data-metric="'+metric+'" data-metric-direction="'+direction+'"' + content=body[:-5] + if unique and column[row_number] is not None and column[row_number]==extreme: + content=''+content+'' + values=values[:cells[index].start()]+opening+'>'+content+''+values[cells[index].end():] + result.append((attrs,values)) + return result + + def table(headers,rows,id='',sortable=False,cls='',metrics=None,index_header='序号',default_sort=None): head=f'{esc(index_header)}'+''.join(f'' if sortable else f'{h}' for i,h in enumerate(headers)) body=''.join(f'{i+1}{values}' for i,(attrs,values) in enumerate(rows)) @@ -1579,6 +1611,9 @@ def avg(key): missing=sorted({r['provider'] for r in records}-set(groups)) note+=' 各类型按各自任务范围比较，不合并为跨类型总分。' if missing:note+=' 未测方法（不计为失败）：'+esc('、'.join(missing))+'。' + directions={i:('quality_'+str(i),'max') for i in range(1,len(heads)) if heads[i] not in {'均值','P50','P95','判分完整性','审查结果'}} + directions.update({i:('latency_'+str(i),'min') for i,h in enumerate(heads) if h in {'均值','P50','P95'}}) + rows=comparative_rows(rows,directions) result=table(heads,rows,id or 'ranking-'+track,True,'ranking',metrics=[] if compact else None) if text_scopes and len(set(text_scopes.values()))>1: result='
文字核对覆盖范围
    '+''.join('
  • '+esc(name)+f'：{len(scope)}/{len(selected)} 例；'+esc('、'.join(selected[k]['name'] for k in sorted(scope)) or '无文字核对记录')+'
    • ' for name,scope in sorted(text_scopes.items()))+'
'+result @@ -1687,7 +1722,7 @@ def overall_ranking(records,cases,audit=None): for name,rs in groups.items(): n=sum(measured(r,cases[r['case_id']]) for r in rs) passed=sum(succeeded(r,cases[r['case_id']]) for r in rs) - values=cell(link(f'providers/{name}.html',provider_label(name)))+cell(rate(sum(bool(r['ok']) for r in rs),len(rs)))+cell(rate(passed,n),passed/n if n else -1) + values=cell(link(f'providers/{name}.html',provider_label(name)))+cell(rate(sum(bool(r['ok']) for r in rs),len(rs)),sum(bool(r['ok']) for r in rs)/len(rs))+cell(rate(passed,n),passed/n if n else -1) for track,_ in active_tracks(cases): subset=[r for r in rs if cases[r['case_id']].get('track','web')==track and measured(r,cases[r['case_id']])] values+=cell(f'{sum(succeeded(r,cases[r["case_id"]]) for r in subset)}/{len(subset)}',sum(succeeded(r,cases[r['case_id']]) for r in subset)/len(subset) if subset else -1) @@ -1712,12 +1747,15 @@ def overall_ranking(records,cases,audit=None): f'
全调用均值 / P50 / P95
每种方法在同一组 {case_count} 个计划任务上的端到端耗时，包含通过与失败；三项不参与通过率排名。主矩阵调用跨日期、跨批次，不等同于同窗受控速度比较。
' '
审查结果
独立盲审覆盖数、审查方法和未决数。
' '
') + directions={i:('acceptance_'+str(i),'max') for i in range(1,3+len(tracks))} + directions.update({i:('all_latency_'+str(i),'min') for i in range(3+len(tracks),6+len(tracks))}) + rows=comparative_rows(rows,directions) return intro+help+table(heads,rows,'ranking-overall',True,'ranking',metrics=[]).replace('{label} ' return '
'+controls+'
' @@ -1768,7 +1806,7 @@ def case_attrs(r,c): stage='' if succeeded(r,c) or not measured(r,c) else 'task' if r['ok'] else 'request' category='' if not stage else failure_category(r,c) result='unmeasured' if not measured(r,c) else 'pass' if succeeded(r,c) else 'fail' - return dataset_attrs(c)+f' data-provider="{esc(r["provider"])}" data-transport="{"pass" if r["ok"] else "fail"}" data-result="{result}" data-content="{content}" data-resources="{resources}" data-failure-stage="{stage}" data-failure-category="{esc(category)}" data-search="{esc(reason(r,c))}"' + return dataset_attrs(c)+f' data-provider="{esc(r["provider"])}" data-transport="{"pass" if r["ok"] else "fail"}" data-result="{result}" data-content="{content}" data-resources="{resources}" data-failure-stage="{stage}" data-failure-category="{esc(category)}" data-search="{esc(str(c.get("id",r.get("case_id","")))+" "+str(c.get("name",""))+" "+str(c.get("url",""))+" "+reason(r,c))}"' def attempt_row(r,c,base='../',*,label=None): state=result_label(r,c) @@ -1832,6 +1870,9 @@ def generate(out:Path,dataset:dict,records:list,manifests:list,*,include_english home+=f'

{len(cases)} 个案例 · {len(records)} 次调用，机器判分。'+source_note+link('guide/index.html#call-chain-audit','查看调用链问题和日志留存范围')+'

' from .report_explainer import article_title,render_explainer,supports_authored_explainer TITLE='本轮调用链检查与日志范围' if dataset.get('comparison',{}).get('call_chain_audit') else article_title(records) if authored_interpretation and supports_authored_explainer(cases) and not all(row.get('provider_type') == 'fixture' for row in records) else '本报告的任务范围与验收规则' + if authored_interpretation and dataset.get('comparison',{}).get('report_interpretation'): + binding=dataset['comparison']['report_interpretation'] + TITLE=json.loads(contained_path(out,binding['path']).read_text())['title'] reading_note='从分数差异、失败案例和实现机制，理解如何选择抓取方法。' if authored_interpretation else '查看本报告的任务范围、结果数量与验收规则。' home+='' if dataset.get('comparison',{}).get('call_chain_audit'): @@ -1845,10 +1886,10 @@ def generate(out:Path,dataset:dict,records:list,manifests:list,*,include_english selected=[c for c in cases.values() if c.get('track','web')==track] coverage=f'{len(selected)} 个案例 · {len({r["provider"] for r in records if cases[r["case_id"]].get("track","web")==track})} 种已测方法' panels.append((track,label,f'

{label}

{coverage}
'+ranking(records,cases,track,compact=True,audit=audit)+f'

{link("datasets/index.html#track-"+track,"案例与参考 →")}

')) - home+=section(f'{track_label} 类任务概览',review_summary(out)+'

调用完整：'+str(reliability['expected_calls'])+'/'+str(reliability['expected_calls'])+' 次均有结果。

只在同一任务类别内比较。

'+cards+'
','overview') home+=section('综合通过率总榜',overall_ranking(records,cases,audit),'overall-ranking') if dataset.get('comparison',{}).get('kind')=='uniform_native_local_measurement': home+=section('同一任务集合的全部调用耗时',overview_all_call_latency(records,cases),'all-call-latency') + home+=section(f'{track_label} 类任务概览',review_summary(out)+'

调用完整：'+str(reliability['expected_calls'])+'/'+str(reliability['expected_calls'])+' 次均有结果。

只在同一任务类别内比较。

'+cards+'
','overview') home+=section('全部方法共同成功任务的耗时',overview_matched_latency(records,cases),'overall-matched-latency') if (out/'resource-cohort/resource-comparison.json').is_file(): home+=section('CPU、内存与网络',overview_resources(out),'overall-resources') @@ -1858,9 +1899,9 @@ def generate(out:Path,dataset:dict,records:list,manifests:list,*,include_english rows=[] for c in cases.values(): verified=c.get('source_verification',{});state='受限' if verified.get('state')=='restricted' else '有效' - rows.append((dataset_attrs(c),cell(link(c['id']+'.html',c['name']))+cell(esc(c['category']))+cell(link(c['url'],'源页 ↗'))+cell(link(c['id']+'.html#facts',basis_label(c)))+cell(reference_label(c))+cell(state)+cell(case_rate(bycase[c['id']],c)))) + rows.append((dataset_attrs(c),cell(link(c['id']+'.html',c['name']))+cell(esc(c['category']))+cell(link(c['url'],'源页 ↗'))+cell(link(c['id']+'.html#facts',basis_label(c)))+cell(reference_label(c))+cell(state))) anchors=''.join(f'' for t,_ in tracks) - dataset_panels=[('cases','案例',anchors+controls+'

核对依据可点击展开。访问受限不等于网站失效；稳定内容与条目结构按登记契约核验，不固定频繁变化的榜单。动态网站正文按各案例声明的完整范围核验。

'+table(['案例','类别','URL','核对依据','参考状态','原访问','达标率'],rows,'filtered-table')),('references','参考状态',''+reference_status(list(cases.values()))),('sources','来源',''+sources_panel())] + dataset_panels=[('cases','案例',anchors+controls+'

核对依据可点击展开。访问受限不等于网站失效；稳定内容与条目结构按登记契约核验，不固定频繁变化的榜单。动态网站正文按各案例声明的完整范围核验。

'+table(['案例','类别','URL','核对依据','参考状态','原访问'],rows,'filtered-table')),('references','参考状态',''+reference_status(list(cases.values()))),('sources','来源',''+sources_panel())] write(out,'datasets/index.html','评测集',hero('评测集',f'{len(cases)} 个案例，按任务类型查看。')+section('案例与依据',tabs('dataset-hub',dataset_panels))) headers=['案例','类型','结果','失败分类','耗时','运行 ID'] for c in ([] if presentation_only else cases.values()): @@ -1882,7 +1923,7 @@ def generate(out:Path,dataset:dict,records:list,manifests:list,*,include_english rules='

登记任务规则

'+esc(list_rule_description(c))+'

稳定内容与变化字段

稳定标题、栏目、说明按登记内容核对；频繁变化的名单、条目标题、排序、价格或 ID 不固定为旧快照取值，按声明的字段结构、类型、数量及唯一性核验。登录页、重复入口、空链接不计入。

' if c.get('acceptance_contract'): facts=[] - rules=stable_contract_details(c) + rules=stable_contract_details(c,evidence_root=out) if c.get('composite_acceptance'): rules+=composite_contract_details(c['composite_acceptance']) source_record=link('../sources/'+c['id']+'/evidence.json','获取记录') if (out/'sources'/c['id']/'evidence.json').exists() else '未保存源页快照' @@ -1916,7 +1957,7 @@ def generate(out:Path,dataset:dict,records:list,manifests:list,*,include_english f'{pct(rules["minimum_coverage"])}。') else:policy=f'每段至少达到 {pct(rules["minimum_coverage"])}。' body+=section('如何检查是否抓全',f'

这是一份完整答案，不是 {total} 个任务。 只检查标题和几个关键词，可能让“只抓到开头”的残缺结果通过。因此系统按前后位置检查同一份标准答案。{policy} 比对忽略空白、排版符号和 Markdown 链接地址；每次执行页会并列显示各位置的标准原文和实际证据。

'+''.join(region_cards)) - body+=section('全部方法执行结果',table(['方法']+headers[1:],[attempt_row(r,c,label=provider_label(r['provider'])) for r in bycase[c['id']]])) + body+='

'+link('../analysis/index.html#inspect?query='+c['id'],'查看各方法执行结果 →')+'

' write(out,'datasets/'+c['id']+'.html',c['name'],body) for name,rs in byprovider.items(): write(out,'providers/'+name+'.html',name,provider_page_body(name,rs,cases,headers,tracks,native_default=dataset.get("comparison",{}).get("kind")=="uniform_native_local_measurement")) @@ -1997,6 +2038,12 @@ def generate(out:Path,dataset:dict,records:list,manifests:list,*,include_english route_label=f'处理路线：方法直接取得原始文档 → {provider_meta["document_extractor"]} → 可读正文' elif c.get('task_kind')=='file': route_label='处理路线：方法直接取得原始文件 → 格式、长度与 SHA-256 验收' + elif provider_meta.get('output_contract')=='readable_document_text': + route_label='处理路线：浏览器原生接口获取文档 → 共同 PDF / DOCX 正文解析；实际完成步骤见运行证据' + elif provider_meta.get('output_contract')=='structured_resource_urls': + route_label='处理路线：浏览器原生 DOM 接口 → 共同资源读取脚本 → 图片或视频资源地址' + elif provider_meta.get('output_contract')=='readable_media_metadata': + route_label='处理路线：浏览器原生页面接口 → 共同脚本读取正文、作者和媒体元数据' else: extractor=str(provider_meta.get('extractor') or '') route_label='处理路线：'+ROUTE_LABELS.get(extractor,extractor or '参评方法直接返回正文') @@ -2007,7 +2054,7 @@ def generate(out:Path,dataset:dict,records:list,manifests:list,*,include_english f'方法：{esc(provider_label(r["provider"]))}' f'运行 ID：{esc(r.get("run_id") or execution_id(r))}') summary=(f'
' - f'{esc(state)}

{esc(reason(r,c))}

' + f'{esc(state)}

{esc(reason(r,c) if passed else (r.get("diagnosis") or {}).get("category") or "任务规则未满足")}

' f'

方法：{link("../providers/"+r["provider"]+".html",provider_label(r["provider"]))} · ' f'请求：{"成功" if r["ok"] else "失败"} · 耗时：{ms(r["elapsed_ms"])} · {actual_summary}

') summary+=f'

{esc(route_label)}

' @@ -2136,7 +2183,7 @@ def generate(out:Path,dataset:dict,records:list,manifests:list,*,include_english body+=section('原始文件字节证据',table(['文件','格式特征','字节数','SHA-256','目标校验'],file_rows,metrics=[])) write(out,'attempts/'+r['attempt_id']+'.html',c['name']+' · '+r['provider'],body) controls=filter_controls([('provider','方法',[(n,provider_label(n)) for n in names]),('track','类型',tracks),('result','结果',[('pass','任务通过'),('fail','任务失败')])]+failure_filters(records,cases)) - comparison=section('方法组合对比','

按同一任务逐项比较，不用各方法不同的成功样本拼接结论。成功与失败均沿用主榜已核验判定；点击方法可查看标准答案、实际返回和原因。

'+method_comparisons(cases,bycase,names),'method-comparison') + comparison=section('方法组合对比','

按同一任务逐项比较，不用各方法不同的成功样本拼接结论。成功与失败均沿用主榜的程序判定；点击方法可查看标准答案、实际返回和原因。

'+method_comparisons(cases,bycase,names),'method-comparison') write(out,'analysis/index.html','结果分析',hero('结果分析','先比较方法在同一任务上的成败组合，再筛选全部执行记录。')+comparison+section('全部执行记录',controls+table(['方法']+headers,[(case_attrs(r,cases[r['case_id']]),cell(link('../providers/'+r['provider']+'.html',provider_label(r['provider'])))+attempt_row(r,cases[r['case_id']])[1]) for r in records],'filtered-table'))) write_support(out,records,manifests,authored_interpretation=authored_interpretation) from .report_changes import render_changes @@ -2191,12 +2238,13 @@ def coverage(label,item): write(out,'review/index.html','核验记录',page) def write_support(out,records,manifests,*,authored_interpretation=True): + dataset=json.loads((out/'dataset.json').read_text()) if (out/'dataset.json').is_file() else {} local_matrix=(out/'dataset.json').is_file() and json.loads((out/'dataset.json').read_text()).get('comparison',{}).get('kind')=='uniform_native_local_measurement' headless=is_headless_cohort(records) from .report_components import task_contract_guide methodology=hero('评测方法','按任务定义验收规则。同组已测方法使用相同的冻结任务与参考，覆盖数量在各榜单列明。') if headless: - methodology+=section('从参考到发布','

1 · 冻结参考

独立采集源页；需要渲染的页面另做浏览器观测，通过完整范围校验后才冻结。

2 · 执行调用

同一 URL、任务和截止时间；各方法使用其最快的正式任务原生产物路线，保留版本、输出与哈希。

3 · 测后核验

独立重取精确参考页，确认调用期间标准答案未改变；不把参考变化算成浏览器失败。

4 · 结果独立审查

强代理逐次读取完整输出、标准答案与任务规则；分歧必须裁定，未裁定项会阻止构建。

'+review_summary(out,'../'),'评测流程') + methodology+=section('从参考到发布','

1 · 冻结参考

锁定既定题目、答案和验收规则；只有明确声明需要实时参考的任务才重新采集源页。

2 · 执行调用

同一 URL、任务和截止时间；各方法使用其最快的正式任务原生产物路线，保留版本、输出与哈希。

3 · 测后核验

核对冻结题目、答案、规则和调用记录；需要实时参考的任务另做前后采集，确认参考未变。

4 · 结果独立审查

独立审查按明确指定的范围开展，检查输出、参考答案和验收规则；本轮审查覆盖见下方，不把程序评分当成独立审查。

'+review_summary(out,'../'),'评测流程') examples_html=re.sub(r'([^<]*)',lambda m:m[0] if (out/'datasets'/(m[1]+'.html')).exists() else m[2],examples()) methodology+=section('每类怎么测',examples_html)+section('指标说明',metric_guide()) methodology+=section('共同输入、计时边界与调用预算','

所有方法进入同一个评分入口，但按任务交付物验收，而不是强制使用同一种中间格式。网页与文档正文最终验收可读文本；原始文件任务验收 PDF、DOCX、图片或视频字节；结构任务验收字段和稳定文字。HTML/Markdown 链接与明确配对的结构化名称—链接字段按同一规则核验；不把请求回显或诊断日志当成答案。原始输出与原始适配器状态保留，报告层不按方法名称改写任务结果。

计入耗时

从准备发起该次 provider 调用开始，到形成可送入统一评分器的最终答案为止：线程调度、SDK/CLI/HTTP 调用、远端处理与网络下载、浏览器渲染与正文提取、本地解析、统一输入转换，以及文件字节读取、格式识别和 SHA-256 计算均在计时内。

不计入耗时

数据集与参考加载、provider 配置校验、调用间限速等待、最终答案形成后的证据落盘、统一评分计算、报告汇总与页面生成不计入。供应商上报的服务端耗时只作附加证据，排名使用客户端观测的端到端耗时。

这个边界比较的是“拿到本任务最终交付物”所需时间：服务直接返回成品时计完整往返；本地方法若还需下载、渲染或解析，也必须完成后才停表。正文任务中，方法只返回原始 PDF 或 DOCX 时，本地正文解析也计时；原始文件任务则到字节校验完成即停止，不做无关的 Markdown 转换。所有方法使用同一外层计时器和截止预算。超时答案不接收；无法取消的调用会被隔离，整批结果因此不完整并触发不可靠状态，构建产物不得发布。

完整正文任务除原有核对点外，按参考正文最多五个连续位置检查字符六元组覆盖：每个位置至少达到 60%，通常要求至少四处达到 80%。所有位置一次性做全局容量分配；同一返回片段实例只能计入一个位置，位置排列顺序不影响结论。若页面存在经独立证据确认的合法重排，案例可显式登记不同位置门槛。比较忽略空白和排版符号但保留文字顺序；全文 F1 独立保留实际值。文件任务必须有字节验收，资源 URL 不等于文件。

') @@ -2225,15 +2273,26 @@ def write_support(out,records,manifests,*,authored_interpretation=True): methodology+=section('变更后重跑范围','

不把“代码有更新”直接等同于全部重跑。只改展示时重建报告；只改裁判且原始输出足够时离线重算；某个参考变化时重跑该题的全部方法；新增或升级单个实现时重跑该方法的全部题；计时边界变化时废弃并重跑同窗测速。只有全局任务输入、调用边界或所有方法共同依赖发生变化，才需要全量重跑。

') methodology+=section('计算与边界','''
全文评分公式

完整度＝取回的参考词数 ÷ 参考词数；纯净度＝匹配的正文词数 ÷ 返回词数；F1＝2 × 完整度 × 纯净度 ÷ 两者之和。英文按词、中文按字，统一大小写与 Unicode 字符形式；Markdown 链接只计文字。对有完整参考的全部案例取平均；请求失败计 0，请求成功但任务未通过仍按实际正文评分，没有参考不计算。

参考范围与公平性

网页、文档使用独立保存的参考正文；仓库以 README 为准，列表只覆盖指定的已加载范围，不要求无限滚动或跨页翻页。登录页和验证码不能代替目标正文。参考正文自身包含的“404 Not Found”等术语不作为错误页判据。

不同方法并行，同一方法串行；正文结束后再测其他任务。Firecrawl 每次完成后冷却 6.1 秒。本报告不是 WCXB 官方榜单。

媒体网址只证明找到入口，不代表取得文件。文件任务没有另接下载器代替参评方法完成。

''') if headless: - methodology=methodology.replace('Firecrawl 每次完成后冷却 6.1 秒。','五种方法每次使用新进程或新上下文。静态网页正文：Kitesurf 走 Cloudflare 账户级 /markdown，Lightpanda、Obscura 与 Moli 1.1.1 使用各自 CLI 正文能力，Chromium 取完整渲染 DOM 后由 node-html-markdown 2.0.0 转换。Moli 遇到直接 PDF、DOCX、图片或视频 URL 时使用不带 --dump 和页面等待参数的自动响应；正文任务在预算内解析可读文字，文件任务直接校验字节。原始主矩阵未启用失败重试，每次调用有 45 秒端到端预算。运行器可在测量前声明至多一次失败重试：保留全部物理证据和总耗时，并固定采用最后一次结果，不能择优。已确认的命令、版本、裁判、参考或测量错误可按完整声明范围重测并替换权威逻辑结果；旧证据不得删除。历史报告中的 Moli 1.0.6 与 1.1.3 证据继续归档，但不进入当前报告矩阵。方法之间并行、同一方法串行。') + methods_count=len({row['provider'] for row in records}) + methodology=methodology.replace('Firecrawl 每次完成后冷却 6.1 秒。',f'本轮 {methods_count} 种配置分别使用独立进程或新上下文。具体接口、等待条件与输出处理见报告解读的任务接口表，实际版本和配置见执行批次。每次调用使用 45 秒端到端预算，调用次数和重试规则以事前声明的协议为准。命令、版本、参考、评分或计时错误按完整声明范围纠正，保留旧证据；不按分数选择重试结果。') + if dataset.get('comparison',{}).get('retrieval_mode')=='native_default_completion': + methodology=methodology.replace('正文任务中，方法只返回原始 PDF 或 DOCX 时，本地正文解析也计时；原始文件任务则到字节校验完成即停止，不做无关的 Markdown 转换。','本轮文档正文只验收浏览器原生接口交付的文字；取得文件字节后不另接外部解析器补正文。原始文件任务到字节校验完成即停止。') if local_matrix: methodology+=section('本轮固定原生配置','

每种方法内部串行执行，每题只调用一次，45秒为外层交卷期限。内部加载、连接和处理时间保留产品默认值。允许测试前固定的原生选项，不传入逐题等待、选择器、数量提示或脚本；不补抓、不额外重试。产品返回的正文和原始响应分别保存。

每种方法内部串行执行；方法之间的调度以执行批次和保存的调用时间为准。分段执行保留每个原始分段，完整调用时间见执行批次。

') methodology=methodology.replace('方法之间并行、同一方法串行。','每种方法内部串行；方法之间的调度见执行批次。') write(out,'methodology/index.html','评测方法',methodology) - downloads=' · '.join(link('../'+path,label) for path,label in [('dataset.json','评测集与参考'),('results.jsonl','全部调用记录'),('summary.json','分类汇总'),('execution.json','执行批次')]+([('implementation-provenance.json','版本回溯证据')] if (out/'implementation-provenance.json').is_file() else [])+([('reference-revisions.json','参考修订收据')] if (out/'reference-revisions.json').is_file() else [])+([('scoring-revisions.json','评分修订收据')] if (out/'scoring-revisions.json').is_file() else [])+([('latency-corrections.json','时延纠正收据')] if (out/'latency-corrections.json').is_file() else [])+([('request-status-corrections.json','请求状态纠正收据')] if (out/'request-status-corrections.json').is_file() else [])) + downloads=' · '.join(link('../'+path,label) for path,label in [('dataset.json','评测集与参考'),('results.jsonl','全部调用记录'),('summary.json','分类汇总'),('execution.json','执行批次')]+([('implementation-provenance.json','浏览器版本与二进制来源')] if (out/'implementation-provenance.json').is_file() else [])+([('reference-revisions.json','参考修订收据')] if (out/'reference-revisions.json').is_file() else [])+([('scoring-revisions.json','评分修订收据')] if (out/'scoring-revisions.json').is_file() else [])+([('latency-corrections.json','时延纠正收据')] if (out/'latency-corrections.json').is_file() else [])+([('request-status-corrections.json','请求状态纠正收据')] if (out/'request-status-corrections.json').is_file() else [])) reproduce='

在评测仓库中执行；读取 evaluation/ 的当前实测数据，不调用供应商接口。

make setup\nmake report EVIDENCE=/absolute/path/to/evidence\nmake report-check

输出目录：reports/webfetch/。固定的 Python 3.11 与 uv.lock 依赖不满足时会立即失败。凭据仅从本地环境加载，不包含在报告中。

' if headless: - reproduce='

在评测仓库中执行；从明确指定的持久化冻结证据构建，不重新调用浏览器。

make setup\nmake report EVIDENCE=/absolute/path/to/evidence\nmake report-check

唯一输出目录为 reports/webfetch/。固定的 Python 3.11 与 uv.lock 依赖不满足时会立即失败。重新测量使用 uv run --frozen python scripts/run_headless_benchmark.py 和 configs/providers.headless.json；先独立采集并冻结参考，再调用，最后做测后核验。完整命令与版本见仓库 harness/web_fetch/HEADLESS.md。

' + reproduce='

在评测仓库中执行；从明确指定的持久化冻结证据构建，不重新调用浏览器。

make setup\nmake report EVIDENCE=/absolute/path/to/evidence\nmake report-check

唯一输出目录为 reports/webfetch/。固定的 Python 3.11 与 uv.lock 依赖不满足时会立即失败。重新测量使用 uv run --frozen python scripts/run_certified_benchmark.py 和本轮方法配置；先锁定题目、答案、规则和执行范围，再调用；明确声明需要实时参考的任务才做前后采集。完整命令与版本见仓库 harness/web_fetch/HEADLESS.md。

' + profile_path=dataset.get('comparison',{}).get('reproduction_profile') + if profile_path: + candidate=Path(profile_path) + if candidate.is_absolute() or not candidate.parts or '..' in candidate.parts or candidate.parts[0]!='evaluation' or candidate.name!='current-report.json': + raise ValueError('Report reproduction profile must stay under evaluation/') + destination=dataset['comparison'].get('report_output','reports/webfetch') + command=f'uv run --frozen python scripts/build_report.py --profile {profile_path} --evidence /absolute/path/to/evidence --output {destination}\nuv run --frozen python scripts/check_report.py {destination} --profile {profile_path}' + reproduce='

从本轮独立配置和冻结证据构建，不重新调用浏览器。

'+esc(command)+'

重新测量使用仓库的认证入口；既定题目和验收规则在调用前冻结，只有明确声明需要实时参考的任务才做前后采集。

' if local_matrix: reproduce='

以下命令只从本轮证据重建本地中文报告，不重新调用产品。

uv run --frozen python scripts/build_report.py --local-audit --evidence /absolute/path/to/report-evidence --output /absolute/path/to/report\nuv run --frozen python scripts/check_report.py /absolute/path/to/report

调用时配置、实现版本与评分器哈希见 '+link('../local-run-contract.json','调用前冻结协议')+'；各次实际请求参数见详情中的真实调用日志。源码中的当前配置与调用时配置可能不同，以冻结协议和原始批次清单为准。凭据只从仓库 .env 读取。

' common=[] @@ -2246,10 +2305,7 @@ def write_support(out,records,manifests,*,authored_interpretation=True): provider_rows=[] for name in sorted({r['provider'] for r in records}): subset=[r for r in records if r['provider']==name] - versions=sorted({str(value) for r in subset for value in - [r.get('provider_metadata',{}).get('browser_version') or - r.get('provider_metadata',{}).get('library_version')] - if value}) + versions=sorted({str(value) for r in subset if (value:=method_version(r,provenance))}) identities=[p.get('implementation_identity',{}) for m in manifests for p in m.get('providers',[]) if p.get('name')==name] versions=versions or sorted({str(identity['repository']['head']) for identity in identities if identity.get('repository',{}).get('head')}) hashes=sorted({str(r.get('provider_metadata',{}).get('binary_sha256')) for r in subset if r.get('provider_metadata',{}).get('binary_sha256')}) @@ -2301,11 +2357,11 @@ def write_support(out,records,manifests,*,authored_interpretation=True): elif headless: access_rows=[] for name in sorted({r['provider'] for r in records},key=lambda value:(provider_label(value).lower(),value)): - access=HEADLESS_ACCESS[name] + access=method_access(name,records) official='
'.join(link(url,label) for label,url in access['links']) variables='
'.join(''+esc(value)+'' for value in access['env_vars']) access_rows.append(('',cell(provider_label(name))+cell(esc(access['account']))+cell(official)+cell(variables)+cell(esc(access['configuration'])))) - account_intro='只有 Kitesurf 的账户级 Cloudflare Browser Run 路线需要供应商账号与凭据；其余六种方法是本地二进制，不需要账号，但仍需安装并明确指定可执行文件。' + account_intro='只有 Kitesurf 的账户级 Cloudflare Browser Run 路线需要供应商账号与凭据；其余方法是本地二进制，不需要账号，但仍需安装并明确指定可执行文件。' account_body=hero('账号与安装',account_intro) account_body+=section('本轮方法清单','

这里列的是完成本报告所需的全部外部准备：账号、官方获取入口和变量名称。变量值、本机路径、登录会话、Cookie 和 Token 均不进入报告。

'+table(['方法','账号要求','官方获取入口','变量名称','本轮配置'],access_rows,cls='accounts-table',metrics=[])) account_body+=section('公开内容边界','

报告只公开复现所需的名称和入口，不公开凭据值。 发布门禁会扫描 HTML、JSON、下载文件和逐次输出中的常见 Token、私钥、Authorization 值与用户目录路径；第三方原始 HTML 中命中密钥形态的片段单独脱敏，计分输出保持逐字节不变。扫描结果和脱敏文件哈希随发布包保存。

') @@ -2796,7 +2852,7 @@ def generate_webmainbench(out:Path,dataset:dict,records:list,summaries:dict,mani failure_types=sorted({r['_output_error'] for r in records if not r['_body_present'] and r['_output_error']}) failed_request_count=sum(not r['_body_present'] for r in records) case_count=len(cases);result_count=case_count*len(names) - labels={'kitesurf':'Kitesurf','lightpanda':'Lightpanda','obscura':'Obscura','moli':'Moli 1.1.5','moli-1.1.7':'Moli 1.1.7','chromium':'Chromium 152'} + labels={'kitesurf':'Kitesurf','lightpanda':'Lightpanda','obscura':'Obscura','moli':'Moli','moli-1.1.7':'Moli 1.1.7','chromium':'Chromium 152'} metric_order=['overall','text_edit','code_edit','formula_edit','table_edit','table_TEDS'] corrected=teds_revision is not None display_score=_wmb_pct diff --git a/harness/web_fetch/report_build.py b/harness/web_fetch/report_build.py index bb543e4da..f05e42bc7 100644 --- a/harness/web_fetch/report_build.py +++ b/harness/web_fetch/report_build.py @@ -202,7 +202,7 @@ def _install_report_tree(staging: Path, output: Path): def build_report(source: Path, output: Path, *, include_english: bool = False, - local_audit: bool = False, authored_interpretation: bool = False, + local_audit: bool = False, authored_interpretation: bool | None = None, diagnostic_preview: bool = False, language: str | None = None) -> Path: from harness.report_languages import includes_english, select_report_language include_english = includes_english(language, include_english) @@ -224,6 +224,8 @@ def build_report(source: Path, output: Path, *, include_english: bool = False, if not local_audit: require_selection_history(source) dataset = json.loads(contained_path(source,'dataset.json').read_text()) + if authored_interpretation is None: + authored_interpretation = bool(dataset.get('comparison', {}).get('report_interpretation')) records = load_jsonl(contained_path(source,'results.jsonl')) validate_report_ids(dataset,records) require_risk_control_certification(source, dataset, records) @@ -264,7 +266,10 @@ def build_report(source: Path, output: Path, *, include_english: bool = False, for c in cases.values(): annotation=c.get('source_benchmark',{}).get('stable_annotation',{}) if annotation.get('anchor_evidence_sha256'): - if local_audit: + if (local_audit or dataset.get('comparison', {}).get('reference_binding_mode') == 'frozen_task_contracts' + or dataset.get('comparison', {}).get('source_reference_policy') in { + 'frozen_acceptance_contracts_with_live_exact_reference_checks', + 'frozen_benchmark_inputs_with_explicit_live_exact_checks'}): # The two historical dynamic cohorts retain their own source # audit receipts. Do not reinterpret those revisions with the # newer origin/main release gate in a local merged view. @@ -442,12 +447,17 @@ def build_report(source: Path, output: Path, *, include_english: bool = False, text=[r for r in subset if r['metrics'].get('content_check_coverage') is not None] row['text_checked_attempts']=len(text) row['text_checked_cases']=len({r['case_id'] for r in text}) - for name,data in [('dataset.json',dataset),('execution.json',{'batches':manifests}),('summary.json',summary)]: + # Preserve frozen input bytes across build and release profile checks. + shutil.copy2(contained_path(source,'dataset.json'), staging/'dataset.json') + for name,data in [('execution.json',{'batches':manifests}),('summary.json',summary)]: (staging/name).write_text(json.dumps(data,ensure_ascii=False,indent=2)+'\n') if (source / 'certification.json').is_file(): shutil.copy2(source / 'certification.json', staging / 'certification.json') if dataset.get('comparison', {}).get('kind') == 'risk_control_100_moli': - for name in ('dataset.json', 'manifest.json', 'execution.json', 'reference-proof-verification.json'): + from .risk_control import risk_control_verification_binding + receipt = json.loads((source / 'certification.json').read_text()) + verification_name, _ = risk_control_verification_binding(receipt) + for name in ('dataset.json', 'manifest.json', 'execution.json', verification_name): shutil.copy2(contained_path(source, name), staging / name) reliability=assess_reliability(dataset,records) (staging/'reliability.json').write_text(json.dumps(reliability,ensure_ascii=False,indent=2)+'\n') @@ -459,10 +469,25 @@ def build_report(source: Path, output: Path, *, include_english: bool = False, if local_audit: if (source/'local-evidence').is_dir(): shutil.copytree(source/'local-evidence', staging/'local-evidence') + content_binding = dataset.get('comparison', {}).get('report_interpretation') + if content_binding is not None: + copy_bound_assets(source, staging, {content_binding['path']: content_binding['sha256']}) copy_methodology_assets(staging) - provenance=Path(__file__).resolve().parents[2]/'research/headless-implementation-provenance.json' - if provenance.is_file(): - shutil.copy2(provenance,staging/'implementation-provenance.json') + implementation_binding = dataset.get('comparison', {}).get('implementation_provenance') + if implementation_binding is not None: + if implementation_binding['path'] != 'implementation-provenance.json': + raise ValueError('Implementation provenance uses implementation-provenance.json') + copy_bound_assets(source, staging, { + implementation_binding['path']: implementation_binding['sha256']}) + else: + provenance=Path(__file__).resolve().parents[2]/'research/headless-implementation-provenance.json' + if provenance.is_file(): + payload=json.loads(provenance.read_text()) + hashes={r.get('provider_metadata',{}).get('binary_sha256') for r in records} + matched=[item for item in payload.get('implementations',[]) if item.get('binary_sha256') in hashes] + if matched: + (staging/'implementation-provenance.json').write_text(json.dumps( + {**payload,'implementations':matched},ensure_ascii=False,indent=2)+'\n') if (source/'scoring-revisions.json').is_file(): scoring_path = source/'scoring-revisions.json' scoring_revisions = json.loads(scoring_path.read_text()) @@ -483,6 +508,12 @@ def build_report(source: Path, output: Path, *, include_english: bool = False, shutil.copy2(archived, target) if (source/'reference-revisions.json').is_file(): shutil.copy2(source/'reference-revisions.json', staging/'reference-revisions.json') + if (source/'report-binding-revisions.json').is_file(): + receipts = json.loads((source/'report-binding-revisions.json').read_text()) + bindings = {row[f'{side}_dataset_path']: row[f'{side}_dataset_sha256'] + for row in receipts for side in ('input', 'output')} + copy_bound_assets(source, staging, bindings) + shutil.copy2(source/'report-binding-revisions.json', staging/'report-binding-revisions.json') if (source/'latency-corrections.json').is_file(): shutil.copy2(source/'latency-corrections.json', staging/'latency-corrections.json') if (source/'request-status-corrections.json').is_file(): @@ -493,18 +524,21 @@ def build_report(source: Path, output: Path, *, include_english: bool = False, shutil.copytree(source/'history/file-baseline', staging/'history/file-baseline') if dataset.get("comparison", {}).get("kind") == "uniform_native_local_measurement": shutil.copy2(source / "local-run-contract.json", staging / "local-run-contract.json") - from harness.reporting.templates import resolve_template - (staging / "report-manifest.json").write_text(json.dumps({ - "schema_version": 1, "template": resolve_template("webfetch"), - "evidence_state": "local_measurement", "certified": False, - "languages": ["zh", "en"] if include_english else ["zh"], "provider_calls_added": 0, - "dataset_sha256": hashlib.sha256((source / "dataset.json").read_bytes()).hexdigest(), - "results_sha256": hashlib.sha256((source / "results.jsonl").read_bytes()).hexdigest(), - }, ensure_ascii=False, indent=2)) generate(staging,dataset,records,manifests,include_english=include_english, authored_interpretation=authored_interpretation) if language is not None: select_report_language(staging, language) + from harness.reporting.templates import resolve_template + from harness.report_languages import generated_languages + nonpublication = diagnostic_preview or dataset.get('comparison', {}).get('kind') == 'uniform_native_local_measurement' + template_manifest = {'schema_version': 1, 'template': None if nonpublication else resolve_template('webfetch'), + 'dataset_sha256': hashlib.sha256((staging/'dataset.json').read_bytes()).hexdigest(), + 'results_sha256': hashlib.sha256((staging/'results.jsonl').read_bytes()).hexdigest(), + 'generated_languages': generated_languages(staging)} + if dataset.get('comparison', {}).get('kind') == 'uniform_native_local_measurement': + template_manifest.update(evidence_state='local_measurement', certified=False, + languages=template_manifest['generated_languages'], provider_calls_added=0) + (staging/'report-manifest.json').write_text(json.dumps(template_manifest, ensure_ascii=False, indent=2)+'\n') (staging/marker).write_text('Generated from persisted evidence. Rebuild; do not edit.\n') # Replace only an explicitly marked generated directory; never merge stale pages. _install_report_tree(staging, output) diff --git a/harness/web_fetch/report_capabilities.py b/harness/web_fetch/report_capabilities.py index 886685ff5..1e83b363c 100644 --- a/harness/web_fetch/report_capabilities.py +++ b/harness/web_fetch/report_capabilities.py @@ -227,17 +227,51 @@ def c(status, detail): 'tested':'fetch --dump markdown --wait-until done；最终结果为准，失败重试链单独保留。', } -def render_capabilities(out, names, *, table, cell, link): - labels={'moli-1.0.6':'Moli 1.0.6','moli':'Moli 1.1.5','moli-1.1.3':'Moli 1.1.3','moli-1.1.7':'Moli 1.1.7','moli-1.1.11':'Moli 1.1.11'} - unknown = sorted(set(names) - set(CAPABILITIES)) +def task_capabilities(out, names, records): + """Describe the configured native task routes, without upgrading product claims.""" + from .report_interfaces import browser_task_interface_matrix + from .methods import native_browser_engine, browser_method_registry + measured=[r for r in records if r.get('provider_metadata',{}).get('output_contract')] + if (not measured or not all(native_browser_engine(name) in {'moli','lightpanda','obscura','chromium','kitesurf'} for name in names) + or not (out/'dataset.json').is_file()):return None + dataset=json.loads((out/'dataset.json').read_text()) + cases={case['id']:case for case in dataset['cases']} + matrix=browser_task_interface_matrix(cases,names) + configured={} + for name in names: + engine=native_browser_engine(name) + row={**CAPABILITIES[engine]} + row.update( + scope=matrix['capabilities'][name], + web=c('支持','原生网页正文输出' if engine!='chromium' else '原生 DOM 加共同 HTML 转换'), + document=c('原生输出','直接使用正文接口或浏览器内查看器文字；不使用外部 PDF/DOCX 解析器'), + image=c('资源地址 / 字节','原生 DOM 接口读取资源地址；直接文件通过原生响应取得'), + av=c('资源与元数据','原生接口读取地址、页面元数据或原始字节；不做媒体理解'), + file=c('原生获取','使用该浏览器原始响应或附件接口，按固定规则验收字节'), + structure='按任务交付 Markdown、文档正文、资源地址、媒体元数据或原始文件。', + notes='浏览器负责访问与原生产物获取；资源读取脚本由评测提供，通过原生接口读取已加载内容；只有 Chromium 使用标准 HTML 转 Markdown 包。这些调用步骤不会被写成浏览器内置能力。完整分类及边界见报告解读的接口表。', + tested='接口按任务输入预选；获取、等待、转换与解析共享 45 秒预算。失败不切换引擎，不按评分结果选择更好的返回。') + source='task-native-'+name + source_url={'moli':'https://github.com/lexmount/moli','lightpanda':'https://github.com/lightpanda-io/browser','obscura':'https://github.com/h4ckf0r0day/obscura','chromium':'https://developer.chrome.com/docs/automation-and-testing/headless','kitesurf':'https://developers.cloudflare.com/browser-run/kitesurf/'}[engine] + source_url=browser_method_registry()[name].get('source_url',source_url) + SOURCES[source]=(name+' · 官方接口',source_url) + row['sources']=[source] + configured[name]=row + return configured + +def render_capabilities(out, names, *, table, cell, link, records=()): + labels={'moli-1.0.6':'Moli 1.0.6','moli':'Moli','moli-1.1.3':'Moli 1.1.3','moli-1.1.7':'Moli 1.1.7','moli-1.1.11':'Moli 1.1.11'} + configured=task_capabilities(out,names,records) + capabilities=configured or CAPABILITIES + unknown = sorted(set(names) - set(capabilities)) evidence = {'checked_at': CHECKED_AT, 'scope': '官方契约/固定源码能力，不是实测成绩', - 'sources': SOURCES, 'methods': {n: CAPABILITIES[n] for n in names if n in CAPABILITIES}, 'unreviewed': unknown} + 'sources': SOURCES, 'methods': {n: capabilities[n] for n in names if n in capabilities}, 'unreviewed': unknown} (out / 'adoption' / 'capabilities.json').write_text(json.dumps(evidence, ensure_ascii=False, indent=2)+'\n') content_rows, runtime_rows, details = [], [], [] for name in names: - if name not in CAPABILITIES: + if name not in capabilities: continue - row = CAPABILITIES[name] + row = capabilities[name] method = cell(link('#capability-'+name, labels.get(name,name))) values = [] for key in ['web', 'document', 'image', 'av', 'file']: @@ -249,11 +283,13 @@ def render_capabilities(out, names, *, table, cell, link): sources = ' · '.join(link(SOURCES[s][1], SOURCES[s][0]+' ↗') for s in row['sources']) details.append(f'
{escape(labels.get(name,name))} · 范围与官方依据'+ f'

对应接口：{escape(row["scope"])}

{escape(row["notes"])}

'+ - f'

本轮配置：{escape(row["tested"])}

{sources}

') + f'

登记配置：{escape(row["tested"])}

{sources}

') legend = ('

原有资料核对日期：'+CHECKED_AT+'；本版新增 Cloudflare 与 Firecrawl 接口核对日期：'+ROUTE_CHECKED_AT+'。托管服务按官方文档；内置工具按固定源码。此处说明可选能力，不是实测成绩；没有启用的选项不补算进榜单。

' '
支持 / 需配置
有对应实现或契约；需配置表示要启用额外模式。
' '
仅链接 / 有限支持
仅返回地址，或只覆盖指定输入、正文区域。
' '
无此能力 / 未声明
前者有明确实现边界；后者是官方资料不足，不能当作不支持。
') + if configured: + legend='

下表说明本报告配置的接口与处理边界，浏览器版本及固定来源见“环境与复现”。共同解析、资源读取脚本和 HTML 转换由评测提供，不计作浏览器内置能力。接口可用不代表每题都能完成；实测结果见结果分析。

' content = ('

能力声明不是实测任务成绩；任务通过按登记的稳定内容锚点与字段结构等规则判定，不证明频繁变化值的事实准确性或实时性。文档列指 PDF / Office 中的文字；普通 TXT 不等于文档解析。图片、音视频列说明网页资源取得方式，不代表视觉理解或转写；直接媒体 URL 能否取得字节看原文件列。原文件不是页面正文、截图或下载链接本身。

'+ table(['方法 / 依据', '静态网站正文', '文档文字', '图片资源', '音视频资源', '原文件'], content_rows, cls='capability-matrix', metrics=[])) runtime = table(['方法 / 依据', 'JS 渲染', '输入范围', '结构化输出', '访问与会话'], runtime_rows, cls='capability-runtime', metrics=[]) diff --git a/harness/web_fetch/report_changes.py b/harness/web_fetch/report_changes.py index 62d8405a9..c5d8c1dee 100644 --- a/harness/web_fetch/report_changes.py +++ b/harness/web_fetch/report_changes.py @@ -26,6 +26,16 @@ def require_review_rounds(data): return rounds +def published_releases(releases, *, require_state=False): + public=[] + for release in releases: + state=release.get('publication_state',None if require_state else 'published') + if state not in {'published','unpublished'}: + raise ValueError('Unknown release publication state') + if state=='published':public.append(release) + return public + + def select_cohort_history(data, dataset, records): """Select public history for the measured cohort, including legacy untagged entries. @@ -40,7 +50,7 @@ def select_cohort_history(data, dataset, records): kind='headless_markdown' if not kind:kind='generic' headless=kind in {'headless_html','headless_markdown'} - releases=data.get('releases',[]) + releases=published_releases(data.get('releases',[]), require_state=data.get('schema_version',2)>=3) boundary=next((i for i,row in enumerate(releases) if row.get('id')==HEADLESS_HISTORY_START),None) selected=[] for index,release in enumerate(releases): @@ -74,7 +84,7 @@ def require_revision_release_bindings(history, releases): for receipt in history: digest = receipt.get('input_results_sha256') matches = [] - for release in releases: + for release in published_releases(releases): declared = release.get('scoring_revision_input_sha256') declared = declared if isinstance(declared, list) else [declared] if digest in declared: @@ -92,7 +102,7 @@ def release_details(release, table, cell, esc): def localized_release_history(data, kind): """Select one report's release records, newest first, with complete EN labels.""" - releases = [row for row in data.get('releases', []) if kind in row.get('comparison_kinds', [])] + releases = [row for row in published_releases(data.get('releases', []), require_state=data.get('schema_version',2)>=3) if kind in row.get('comparison_kinds', [])] if not releases or len({row['id'] for row in releases}) != len(releases): raise ValueError(kind + ' release history is absent or has duplicate IDs') for row in releases: diff --git a/harness/web_fetch/report_check.py b/harness/web_fetch/report_check.py index 5eec4fb54..3a7086b25 100644 --- a/harness/web_fetch/report_check.py +++ b/harness/web_fetch/report_check.py @@ -12,6 +12,7 @@ from harness.record_io import load_jsonl from harness.report_languages import check_language_contract, generated_languages, visible_numbers from harness.reporting.article_assets import ArticleAssetParity +from harness.reporting.presentation import check_presentation from harness.web_fetch.reference_lifecycle import validate_resource_source_evidence from harness.web_fetch.reliability import require_reliable from harness.web_fetch.safety import contained_path, reject_evidence_symlinks, safe_identifier @@ -133,7 +134,7 @@ def handle_data(self, text): self.numbers.extend(visible_numbers(text)) -def check(root: Path, *, authority_root: Path, profile: Path | None = None, presentation_binding: Path | None = None): +def check(root: Path, *, authority_root: Path, profile: Path | None = None, presentation_binding: Path | None = None, source_dataset: Path | None = None): reject_evidence_symlinks(root) root = root.resolve() repository = authority_root.resolve() @@ -158,9 +159,34 @@ def check(root: Path, *, authority_root: Path, profile: Path | None = None, pres if (profile_path.parent / "dataset.json").is_file() else repository / "evaluation" ) - require_canonical_inputs(root, canonical) + if not json.loads(profile_path.read_text()).get('frozen_inputs'): + require_canonical_inputs(root, canonical) d = json.loads((root / "dataset.json").read_text()) + manifest_path = root / 'report-manifest.json' + if manifest_path.is_file(): + from harness.reporting.templates import resolve_template + manifest = json.loads(manifest_path.read_text()) + expected_template = None if d.get('comparison', {}).get('kind') == 'uniform_native_local_measurement' else resolve_template('webfetch', root=repository) + if manifest.get('template') != expected_template: + raise ValueError('Report manifest differs from the maintained template') + for name in ('dataset', 'results'): + path = root / ('dataset.json' if name == 'dataset' else 'results.jsonl') + if manifest.get(name + '_sha256') != hashlib.sha256(path.read_bytes()).hexdigest(): + raise ValueError('Report manifest evidence changed: ' + name) + if manifest.get('generated_languages') != list(generated_languages(root)): + raise ValueError('Report manifest languages differ from generated routes') rs = load_jsonl(root / "results.jsonl") + if source_dataset: + if not profile_path: + raise ValueError('Frozen source dataset requires an explicit report profile') + profile_data = json.loads(profile_path.read_text()) + if source_dataset.is_symlink(): + raise ValueError('Frozen source dataset rejects symlinks') + source_raw = source_dataset.read_bytes() + if hashlib.sha256(source_raw).hexdigest() != profile_data.get('frozen_inputs', {}).get('dataset_sha256'): + raise ValueError('Frozen source dataset differs from current profile') + if json.loads(source_raw)['cases'] != d['cases']: + raise ValueError('Report cases differ from frozen source dataset') check_language_contract(root) has_english = "en" in generated_languages(root) if d.get("comparison", {}).get("kind") == "webmainbench_frozen_html_to_markdown": @@ -186,7 +212,7 @@ def check(root: Path, *, authority_root: Path, profile: Path | None = None, pres require_local_measurement(root, d, rs) inventory = {track: {"case_ids": [case["id"] for case in d["cases"] if case.get("track", "web") == track], "providers": providers} for track, providers in d["coverage_plan"].items()} canonical_dataset = repository / json.loads((root / "local-run-contract.json").read_text())["dataset_path"] - elif profile_path and (profile_path.parent / "dataset.json").is_file(): + elif source_dataset or (profile_path and (json.loads(profile_path.read_text()).get('frozen_inputs') or (profile_path.parent / "dataset.json").is_file())): inventory = { track: { "case_ids": [ @@ -196,7 +222,7 @@ def check(root: Path, *, authority_root: Path, profile: Path | None = None, pres } for track, providers in d.get("coverage_plan", {}).items() } - canonical_dataset = profile_path.parent / "dataset.json" + canonical_dataset = source_dataset or (root / 'dataset.json' if json.loads(profile_path.read_text()).get('frozen_inputs') else profile_path.parent / "dataset.json") else: inventory = json.loads((repository / "configs/evaluation-inventory.json").read_text()) canonical_dataset = repository / "evaluation/dataset.json" @@ -275,7 +301,6 @@ def check(root: Path, *, authority_root: Path, profile: Path | None = None, pres if c.get("track", "web") in {"web", "dynamic", "document"} and ( not c.get("ground_truth_main_content") ): - unavailable = c.get("source_verification", {}).get("state") == "unavailable" stable_checks = bool( c.get("required_facts") or c.get("required_patterns") @@ -284,7 +309,7 @@ def check(root: Path, *, authority_root: Path, profile: Path | None = None, pres if local_measurement: from .stable_tasks import validate_acceptance_contract validate_acceptance_contract(c["acceptance_contract"]) - elif not (unavailable and stable_checks): + elif not stable_checks: raise ValueError(c["id"]) if c.get("ground_truth_main_content"): expected = c["ground_truth_main_content"].encode() @@ -292,6 +317,9 @@ def check(root: Path, *, authority_root: Path, profile: Path | None = None, pres from .reference_inputs import read_dataset_bytes canonical_cases = {case["id"]: case for case in json.loads(read_dataset_bytes(canonical_dataset))["cases"]} valid_reference = expected == canonical_cases[c["id"]]["ground_truth_main_content"].encode() + elif source_dataset: + digest = hashlib.sha256(expected).hexdigest() + valid_reference = all(r['metrics'].get('scored_reference_sha256') == digest for r in rs if r['case_id'] == c['id']) else: valid_reference = hashlib.sha256(expected).hexdigest() == c["source_verification"]["reference_sha256"] if not valid_reference: @@ -400,6 +428,7 @@ def canonical_link(page, value): ) for p in root.rglob("*.html"): text = p.read_text() + check_presentation(text) reader = VisibleText() reader.feed(text) visible = " ".join(reader.parts) @@ -442,6 +471,8 @@ def canonical_link(page, value): if case.get("task_kind") == "file" else "标准答案与验收规则" if case.get("task_kind") == "structure" + else "验收规则（无全文参考）" + if case.get('task_kind', 'body') == 'body' and not (case.get('ground_truth_main_content') or case.get('reference_answer')) else "完整标准答案" ) actual_heading = ( @@ -449,6 +480,7 @@ def canonical_link(page, value): "目标文件": "Target file", "标准答案与验收规则": "Reference and acceptance contract", "完整标准答案": "Complete reference answer", + "验收规则（无全文参考）": "Acceptance rules (no full-text reference)", }[heading] if p.relative_to(root).parts[0] == "en" else heading @@ -485,11 +517,11 @@ def canonical_link(page, value): home = (root / "index.html").read_text() track_count = len({case.get("track", "web") for case in d["cases"]}) if ( - not home.index(f"

{track_count} 类任务概览

") - < home.index("

综合通过率总榜

") + not home.index("

综合通过率总榜

") + < home.index(f"

{track_count} 类任务概览

") < home.index(f"

{track_count} 类分榜

") ): - raise ValueError("Task overview and overall ranking must precede detailed rankings") + raise ValueError("Core comparison must precede the overview and detailed rankings") if not ("本类最高：" in home and "判分完整性" in home): raise ValueError("Report acceptance check failed") if (root / "resource-cohort/resource-comparison.json").is_file(): @@ -532,10 +564,11 @@ def main(argv=None, *, authority_root: Path | None = None): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("root", type=Path) parser.add_argument("--profile", type=Path) + parser.add_argument('--source-dataset', type=Path) parser.add_argument("--authority-root", type=Path, default=authority_root or Path.cwd()) parser.add_argument("--presentation-binding", type=Path) args = parser.parse_args(argv) - result = check(args.root, authority_root=args.authority_root, profile=args.profile, presentation_binding=args.presentation_binding) + result = check(args.root, authority_root=args.authority_root, profile=args.profile, presentation_binding=args.presentation_binding, source_dataset=args.source_dataset) if result is not None: print(result) @@ -640,6 +673,17 @@ def check_revision(out: Path, *, authority_root: Path): expected_failure = "Failure class" if is_english else "失败分类" if expected_failure not in [h.get_text() for h in soup.select("#filtered-table th")]: raise ValueError(path) + records = load_jsonl(out / 'results.jsonl') + if records and all(row.get('provider_type') == 'fixture' for row in records): + methods = BeautifulSoup((out / 'adoption/index.html').read_text(), 'html.parser') + for name in {row['provider'] for row in records}: + if methods.find(id='interface-' + name) is None: + raise ValueError('Synthetic adapter interface is missing: ' + name) + if dataset.get('comparison', {}).get('report_interpretation'): + from .report_explainer import check_current_browser_article + check_current_browser_article(out, {case['id']: case for case in dataset['cases']}, records, + dataset['comparison']['report_interpretation']) + return 'PASS: synthetic report presentation, declared adapters and bound interpretation verified' counts = json.loads((out / "adoption/counts.json").read_text())["methods"] names = {row["provider"] for row in load_jsonl(out / "results.jsonl")} from harness.web_fetch.report_methods import PROFILES diff --git a/harness/web_fetch/report_components.py b/harness/web_fetch/report_components.py index 3db213b2b..a4404f8b3 100644 --- a/harness/web_fetch/report_components.py +++ b/harness/web_fetch/report_components.py @@ -160,8 +160,9 @@ def task_pass_guide(track, base='', cases=None): text = rules.get(track, '按案例中登记的验收规则逐项检查。') coverage=[c['body_requirements'] for c in (cases or {}).values() if c.get('body_requirements')] if coverage: - thresholds=' / '.join(f'{v:.0%}' for v in sorted({r['minimum_coverage'] for r in coverage})) - text+=f' 已登记正文覆盖规则的案例，还须在最多 {max(len(r["regions"]) for r in coverage)} 个参考区域中逐区达到 {thresholds} 词语计数覆盖，不能只返回标题或开头。' + text+=' 已登记正文覆盖规则的案例，按各例指定的比较单位、区域门槛和通过区域数验收，不能只返回标题或开头。' + if any(r.get('unit')=='region_compact_hexagram_recall' for r in coverage): + text+=' 字符六元组规则忽略排版空白，保留文字顺序；各区域的要求见题目详情。' for c in (cases or {}).values(): fields=c.get('acceptance_contract',{}).get('field_minimum_records',{}) if fields: @@ -234,7 +235,7 @@ def examples(): ('结构怎么测','至少 3 个有名称的本站详情链接，按地址去重。仅网址、导航按钮和登录提示不计入；频繁变化的条目名称、名次、价格或 ID 不固定为旧快照取值，不免除登记的稳定内容核对。'), ('怎样算通过','稳定内容锚点与必需字段、类型、唯一性和数量全部通过。通过不证明变化值的事实准确性或实时性；缺少完整、可比的参考时完整度、纯净度和 F1 为 —。查看参考与实测 →'), ]), - ('risk_control','历史访问受限网址内容',[ + ('risk_control','风控网站正文',[ ('任务','打开指定网址，取得首次自动加载出的目标正文、列表和真实链接，不进行点击、输入、滚动、翻页或登录。'), ('核对依据','按独立保存的目标页面证据登记稳定文字、完整正文分区，以及名称—链接、路径、字段、去重和数量要求。'), ('怎么测','先排除请求失败、空返回、验证或拦截页面和页面外壳，再逐项检查本例声明的稳定文字、正文覆盖与组合规则。'), @@ -272,7 +273,7 @@ def list_rule_description(c): 'quote':'引文列表', 'product':'商品列表', 'catalog_product':'商品目录', 'review':'评论列表', 'name_description':'名称与说明', 'api_operation':'接口摘要', 'taxonomy_family':'科名列表', 'named_link':'具名详情入口', - 'linked_product':'具名商品详情', 'page_identity':'页面身份', + 'linked_product':'具名商品详情', 'page_identity':'目标页面核对', } FIELD_REQUIREMENTS = { 'magnitude':('震级','有效的数字震级，不固定某次地震的数值。'), @@ -329,7 +330,7 @@ def stable_rule_description(c): return ('页面正文按登记的稳定文字锚点核验；新闻、日期、轮播卡片、仓库状态等' '易变内容不固定为答案。历史完整快照只作诊断，不进入全文质量汇总。') if contract['schema'] == 'page_identity': - return '页面身份：所有独立登记的稳定文字组均须命中；不声明或虚构条目数量。' + return '目标页面核对：指定的稳定文字组都须命中；不要求页面包含固定条目数。' maximum = contract.get('maximum_records') limit = f'至少 {contract["minimum_records"]} 条' + (f'，最多 {maximum} 条' if maximum is not None else '') return (f'{SCHEMA_LABELS.get(contract["schema"], contract["schema"])}：{limit}完整且唯一的记录。' @@ -337,7 +338,7 @@ def stable_rule_description(c): '频繁变化的名单和价格不固定旧快照取值；通过不证明这些变化值真实或最新。') -def stable_contract_details(c): +def stable_contract_details(c, *, evidence_root=None): from .stable_tasks import SCHEMA_FIELDS, UNIQUE_FIELDS contract = c['acceptance_contract'] schema = contract['schema'] @@ -374,8 +375,10 @@ def stable_contract_details(c): or c.get('source_benchmark',{}).get('stable_annotation',{})) evidence = ((c.get('stable_text_evidence') or {}).get('path') or annotation.get('anchor_evidence')) + if evidence and evidence_root is not None and not (evidence_root/evidence).is_file(): + evidence = None provenance = ('

独立文字依据：查看限定范围原文。' - '这些核对项验证文字是否保留，不证明整页正文完整，也不以榜单当前成员为答案。

') if evidence else ('

本例未登记稳定文字依据；文字核对覆盖不计算，已执行的列表规则与结果仍保留。

' if annotation else '') + '这些核对项验证文字是否保留，不证明整页正文完整，也不以榜单当前成员为答案。

') if evidence else ('

固定文字核对项见下方；本报告未附早期标注的原始页面文件。

' if annotation and anchors else '

本例未登记稳定文字依据；文字核对覆盖不计算，已执行的列表规则与结果仍保留。

' if annotation else '') return ('

'+escape(stable_rule_description(c))+'

'+requirements + provenance + ('

稳定文本锚点

每组至少命中一种允许表达，所有组均须通过。

    '+anchors+'
' if anchors else '') diff --git a/harness/web_fetch/report_diagnostic.py b/harness/web_fetch/report_diagnostic.py index 8d8c32bc3..e31399f85 100644 --- a/harness/web_fetch/report_diagnostic.py +++ b/harness/web_fetch/report_diagnostic.py @@ -260,9 +260,9 @@ def add(kind, label, expected, actual): add('truncated', '输出声明被截断', '完整的任务内容', '；'.join(metrics['declared_truncation_hits'])) if metrics.get('script_payload_shell'): add('shell', '只有脚本或页面框架', '目标正文', '保存评分识别到脚本或空框架') - if metrics.get('page_identity_status') in ('mismatch', 'unknown', 'not_verified') or any( + if (metrics.get('page_identity_required') and metrics.get('page_identity_status') in ('mismatch', 'unknown', 'not_verified', 'unavailable')) or any( k in str(row.get('failure', '')) for k in ('page_identity_not_verified', 'page_identity_mismatch')): - add('identity', '页面身份未满足要求', '最终网址和页面语言符合任务', + add('identity', '最终网址或页面语言未满足要求', '最终网址和页面语言符合任务', '最终网址：' + str(log.get('final_url') or '记录未提供') + '；页面语言：' + str(metrics.get('observed_document_language') or log.get('document_language') or '记录未提供')) for check in metrics.get('stable_anchor_results') or []: if check.get('pass') is False: diff --git a/harness/web_fetch/report_explainer.py b/harness/web_fetch/report_explainer.py index 1280be6fd..1caab8d75 100644 --- a/harness/web_fetch/report_explainer.py +++ b/harness/web_fetch/report_explainer.py @@ -13,11 +13,12 @@ from harness.record_io import parse_jsonl from harness.web_fetch.reference_inputs import read_dataset_bytes -from .metrics import percentile, task_passed from .report_resources import render_resource_comparison +from .metrics import task_passed, percentile +from .methods import is_browser_comparison +from .scoring import score_attempt from .report_style import STYLE from .safety import contained_path, safe_identifier -from .scoring import score_attempt TITLE = '网页抓取成功之后，内容离可用还有多远' HEADLESS_TITLE = '五类浏览器、七种方法取同一批网页：速度、正文与交付边界' @@ -25,15 +26,15 @@ ROOT = Path(__file__).resolve().parents[2] TEMPLATE = Path(__file__).parent / 'templates/interpretation.html' -TRACK_NAMES = {'web': '静态网站正文', 'document': '文档文字', 'image': '图片', - 'video': '视频', 'challenge': '热门网站正文', 'dynamic': '动态网站正文'} +from .report import TRACKS as TRACK_NAMES + PROVIDER_NAMES = {'cowagent-built-in': 'CowAgent', 'moli-webfetch': 'Moli', 'cloudflare-markdown': 'Cloudflare Markdown', 'exa': 'Exa', 'langchain-webbase': 'LangChain WebBaseLoader', 'firecrawl': 'Firecrawl', 'crawl4ai': 'Crawl4AI', 'openclaw-built-in': 'OpenClaw', 'deepseek-harness-built-in': 'DeepSeek Harness', 'tavily': 'Tavily', 'lexmount-global': 'Lexmount 国际', 'lexmount-cn': 'Lexmount 中国区', - 'moli-1.0.6': 'Moli 1.0.6', 'moli': 'Moli 1.1.5', + 'moli-1.0.6': 'Moli 1.0.6', 'moli': 'Moli 1.1.5', 'moli-pr919': 'Moli · PR #919', 'moli-1.1.3': 'Moli 1.1.3', 'moli-1.1.11': 'Moli 1.1.11'} METHODOLOGY_FIGURES = ( @@ -459,7 +460,7 @@ def _dynamic_section(data, cases): def _is_headless_comparison(records): - return bool(records) and all(r.get('provider_type') in {'headless_html','headless_markdown'} for r in records) + return is_browser_comparison(records) def article_title(records): @@ -524,11 +525,16 @@ def passed(r): def _headless_article_input(cases, records, providers): """Bind measured article inputs while ignoring report-only annotations.""" derived_fields = {'diagnosis', 'failure_category', 'title_quality'} + def measured_record(record): + result = {key: value for key, value in record.items() if key not in derived_fields} + if 'provider_metadata' in result: + result['provider_metadata'] = {key: value for key, value in result['provider_metadata'].items() + if key != 'local_report_complete_diagnostic_links'} + return result return { 'cases': cases, 'records': sorted( - [{key: value for key, value in record.items() if key not in derived_fields} - for record in records if record['provider'] in providers], + [measured_record(record) for record in records if record['provider'] in providers], key=lambda record: record['attempt_id']), } @@ -749,14 +755,319 @@ def bars(values, unit='', maximum=None): return body + '' +def current_browser_article_model(out, cases, records, binding): + """Bind new cohort prose to measured inputs; derive every comparison value.""" + path = contained_path(out, binding['path']) + raw = path.read_bytes() + if hashlib.sha256(raw).hexdigest() != binding['sha256']: + raise ValueError('Browser interpretation content hash mismatch') + content = json.loads(raw) + providers = sorted({row['provider'] for row in records}) + if content.get('schema') != 'webfetch_browser_interpretation/1': + raise ValueError('Unsupported browser interpretation content schema') + if content.get('evidence_sha256') != _digest(_headless_article_input(cases, records, providers)): + raise ValueError('Browser interpretation evidence changed') + if set(content.get('providers', {})) != set(providers): + raise ValueError('Browser interpretation must explain every participant') + if not cases or not providers or any(c.get('track', 'web') not in TRACK_NAMES for c in cases.values()): + raise ValueError('Browser interpretation requires supported nonempty task families') + expected = {(name, cid) for name in providers for cid in cases} + indexed = {(row['provider'], row['case_id']): row for row in records} + if len(indexed) != len(records) or set(indexed) != expected: + raise ValueError('Browser interpretation requires the complete declared matrix') + def passed(row): + return task_passed(row['ok'], row['metrics'], cases[row['case_id']].get('task_kind', 'body')) + def distribution(rows): + times = [row['elapsed_ms'] for row in rows] + return {'n': len(rows), 'mean': statistics.mean(times) if times else None, + 'p50': statistics.median(times) if times else None, + 'p95': percentile(times, .95) if times else None} + tracks = [track for track in TRACK_NAMES if any(c.get('track', 'web') == track for c in cases.values())] + stats = {} + from .report_interfaces import method_provenance, method_version + provenance = method_provenance(out) + for name in providers: + rows = [indexed[name, cid] for cid in cases] + successes = [row for row in rows if passed(row)] + failures = [row for row in rows if not passed(row)] + chapter = content['providers'][name] + specimen = indexed.get((name, chapter.get('specimen_case_id'))) + if specimen is None or (failures and passed(specimen)): + raise ValueError('Browser chapter requires an evidenced failure specimen when failures exist') + explanation = chapter.get('explanation') + if not isinstance(explanation, str) or not explanation.strip(): + raise ValueError('Browser chapter requires a measured explanation') + versions = sorted({str(method_version(row,provenance) or '') for row in rows} - {''}) + groups = {track: [row for row in rows if cases[row['case_id']].get('track', 'web') == track] for track in tracks} + stats[name] = {'version': ' / '.join(versions) if versions else '托管服务，内部版本未公开' if name == 'kitesurf' else '未提供版本', + 'passed': len(successes), 'pass_rate': len(successes)/len(cases), + 'tracks': {track: {'passed': sum(passed(row) for row in subset), 'n': len(subset)} for track,subset in groups.items()}, + 'all': distribution(rows), 'successful': distribution(successes), + 'failure_counts': dict(Counter(row.get('error_type') or 'acceptance' for row in failures)), + 'specimen': specimen} + for field in ('title', 'intro', 'scoring', 'conclusions'): + if not isinstance(content.get(field), str) or not content[field].strip(): + raise ValueError('Browser interpretation is missing prose: ' + field) + tracks = [track for track in TRACK_NAMES if any(c.get('track', 'web') == track for c in cases.values())] + return {'content': content, 'providers': providers, 'stats': stats, 'tracks': tracks, + 'cases': len(cases), 'results': len(records)} + + +def _offline_browser_article(body, out, cases, model): + """Embed specimen evidence; an independent download needs no report host.""" + from bs4 import BeautifulSoup + from .artifacts import record_public_call_log_bindings, verify_public_call_log + document = BeautifulSoup(body, 'html.parser') + for script in document.select('script'): + script.decompose() + targets = {} + appendices = [] + def evidence(path, anchor, title, payload): + if path in targets: + return + targets[path] = '#'+anchor + appendices.append('
' + +escape(title)+''+payload+'
') + for name in model['providers']: + row = model['stats'][name]['specimen'] + case = cases[row['case_id']] + evidence('../datasets/'+case['id']+'.html', 'offline-task-'+case['id'], + '任务与固定验收规则 · '+case['id'], + '
'+escape(json.dumps(case,ensure_ascii=False,indent=2))+'
') + raw = contained_path(out,row['artifact_path']).read_text(encoding='utf-8',errors='replace') + payload = '

完整实际返回

'+escape(raw)+'
' + payload += '

结果与评分

'+escape(json.dumps(row,ensure_ascii=False,indent=2))+'
' + for binding in record_public_call_log_bindings(row): + log = verify_public_call_log(out,binding).read_text() + payload += '

调用日志

'+escape(log)+'
' + evidence('../attempts/'+row['attempt_id']+'.html', 'offline-result-'+row['attempt_id'], + '完整结果与调用证据 · '+name, payload) + if (out/'result-review.json').is_file(): + evidence('../review/index.html','offline-review','独立审查记录', + '
'+escape((out/'result-review.json').read_text())+'
') + for anchor in document.select('a[href]'): + href = anchor['href'] + if href in targets: + anchor['href'] = targets[href] + elif not href.startswith(('https://','http://','#')): + raise ValueError('Independent article has an unbound evidence target: '+href) + return str(document)+''.join(appendices) + + +def _render_current_browser_article(out, cases, records, binding): + model = current_browser_article_model(out, cases, records, binding) + content, names, stats = model['content'], model['providers'], model['stats'] + from .methods import browser_method_label, native_browser_engine + engine_count = len({native_browser_engine(name) or name for name in names}) + def label(name): + return escape(browser_method_label(name) if browser_method_label(name) != name else name.title()) + def paragraph(value): + return '

' + escape(value).replace('\n\n', '

') + '

' + def section(title, body, anchor): + return '

'+escape(title)+'

'+body+'
' + def metric(name, key, value, values, higher=False, suffix=''): + valid = [v for v in values if v is not None] + extreme = (max(valid) if higher else min(valid)) if valid else None + winner = value is not None and value == extreme and valid.count(extreme) == 1 + text = '—' if value is None else (f'{value:.1%}' if key == 'pass_rate' or key.startswith('family_') else f'{value:,.0f}{suffix}') + return (''+text+'') + def latency_table(population): + return _table(['浏览器', '样本数', '平均耗时 · ms', 'P50 · ms', 'P95 · ms'], [ + (label(name), str(stats[name][population]['n']), *( + metric(name, population+'_'+key, stats[name][population][key], + [stats[n][population][key] for n in names]) for key in ('mean','p50','p95'))) + for name in names], '同一行三个耗时指标使用相同样本；每个逻辑结果只计一次。') + body = '

'+escape(content['title'])+'

' + dates = sorted({row['started_at'][:10] for row in records}) + body += '

调用日期：'+escape('、'.join(dates))+'

' + body += '

'+str(model['cases'])+' 个任务；'+str(engine_count)+' 种浏览器、'+str(len(names))+' 种配置，'+str(model['results'])+' 个逻辑结果。

'+paragraph(content['intro']) + body += section('整体任务通过率', _table(['浏览器','实测版本','通过数 / 总数','任务通过率'], [ + (label(name), escape(stats[name]['version']), f'{stats[name]["passed"]}/{model["cases"]}', + metric(name,'pass_rate',stats[name]['pass_rate'],[stats[n]['pass_rate'] for n in names],True)) + for name in names], '全体任务每题等权；通过表示满足该题的正文、结构、资源或文件验收规则。'), 'overall') + def family_cell(name, track): + group = stats[name]['tracks'][track] + rate = group['passed']/group['n'] + values = [stats[n]['tracks'][track]['passed']/stats[n]['tracks'][track]['n'] for n in names] + return f"{group['passed']}/{group['n']}
" + metric(name,'family_'+track,rate,values,True) + body += section('各专项结果', _table(['任务类型',*(label(name) for name in names)], [ + (escape(TRACK_NAMES[track]), *(family_cell(name,track) for name in names)) + for track in model['tracks']], '分组单元格为通过数 / 该组总数及通过率；专项结果不能代替整体排名。'), 'families') + from .report_interfaces import browser_task_interface_matrix + interfaces = browser_task_interface_matrix(cases, names) + native = _table(['浏览器','Markdown 能力与本次调用'], [ + (''+label(name)+'', escape(interfaces['capabilities'][name])) for name in names], 'Markdown 能力指网页转换；不代表 PDF、DOCX 或媒体文件解析能力。') + route_rows = [] + for row in interfaces['rows']: + cells = [] + for name in names: + parts = row['cells'][name] + cells.append('
' + + '

'.join('' + +str(part['count'])+' 题：'+escape(part['description'])+'
等待：'+escape(part['wait'])+'' for part in parts)+'
') + route_rows.append((escape(TRACK_NAMES[row['track']])+f" · {row['count']} 题", *cells)) + matrix = _table(['任务类型',*(label(name) for name in names)], route_rows, '每格列出获取接口、额外处理及最终输出；同类中的不同交付格式分别列出题数。') + body += section('各类任务的获取与转换方式', + '

按题目类型使用参评浏览器合适的官方原生接口。正文直接采用该接口返回的文字；不会改取 HTML、JSON DOM 或文档字节后替它提取正文。Chromium 没有原生 Markdown，网页 HTML 使用标准转换包。下表按固定任务输入列出实际选择的处理链，失败不补成成功，也不切换引擎。每例完成了哪些步骤、返回什么内容，均可在结果详情中查看。文档正文、资源地址、原始文件和媒体元数据有各自的交付格式，不强行转成 Markdown。

' + +native+'
'+matrix+'
' + +'

Obscura 的正文提取脚本与官方 CLI、LP.getMarkdown 相同；评测包装只附加最终网址和标题，不修改转换算法。JSON 包装只解包原生输出，不改写内容。只有 Chromium 的网页 HTML 使用 node-html-markdown 2.0.0；所有浏览器都不使用外部 PDF/DOCX 解析器补正文。资源脚本只读取已加载页面，嵌入文档只读浏览器内已加载的查看器文字。最终网址作为辅助信息，不单独决定正文任务是否通过。获取、等待、转换、解析与附件读取计入同一 45 秒预算。Moli 附件的原生 CDP 导航与原始附件接口是两步操作，次数和耗时保留在结果记录，不因评分失败而触发。所有结果采用同一评分器和固定验收规则。

', 'interfaces') + flow = 'flowchart LR\n A[同一目标网址与验收规则] --> B[各浏览器原生访问与提取]\n B --> C[保留完整输出与调用日志]\n C --> D[统一评分]\n D --> E[整体与专项结果]' + fallback = ' → '.join(''+x+'' for x in ('目标网址与规则','浏览器原生访问','完整输出与日志','统一评分','整体与专项')) + mechanism = '
'+fallback+'
' + body += section('怎样判断一次获取是否可用', mechanism+paragraph(content['scoring']), 'scoring') + body += section('全部调用的耗时', '

所有浏览器使用相同任务分母，包含成功与失败调用。耗时较短只说明更快返回，不等于更高的任务通过率。

'+latency_table('all'), 'all-latency') + body += section('通过任务的耗时', '

只计各浏览器自己通过的任务。各行样本可能不同，须结合通过数阅读；这张表不改变任务范围或整体排名。

'+latency_table('successful'), 'successful-latency') + resource = contained_path(out, 'resource-cohort/resource-comparison.json') + if resource.is_file(): + table, explanation = _render_article_resources(json.loads(resource.read_text()), names) + body += section('客户端资源消耗', table+paragraph(explanation), 'resources') + errors = {'acceptance':'内容或交付验收未通过','timeout':'访问或提取超时', + 'browser_exit':'浏览器进程返回错误','browser_error':'浏览器访问或运行错误', + 'http_status':'网页返回错误状态','empty_response':'返回内容为空','tool_error':'工具执行错误'} + for name in names: + stat, chapter = stats[name], content['providers'][name] + counts = stat['failure_counts'] + distribution = _table(['程序分类','未通过数量'], [ + (escape(errors.get(key, key)), ''+str(count)+'') + for key,count in sorted(counts.items())], f'{name} 的全部未通过结果，每个结果归入一个程序分类。') if counts else '

本轮所有任务均通过。

' + row = stat['specimen'];case = cases[row['case_id']] + raw = contained_path(out,row['artifact_path']).read_text(encoding='utf-8',errors='replace') + reference = case.get('ground_truth_main_content') or case.get('reference_answer') or json.dumps({ + key:case[key] for key in ('required_facts','body_requirements','acceptance_contract','expected_resources','resource_output_contract') if case.get(key)},ensure_ascii=False,indent=2) + error = row.get('error_message') or ('未满足完整验收规则；逐项期望与实际值见结果详情。' if counts else '满足该题验收规则。') + example = '
'+escape(case.get('name',case['id']))+' · '+escape(case['id'])+'' + example += '

原始页面 · 任务与验收规则 · 完整结果、评分与日志

' + example += '

参考范围或验收要求（节选）

'+escape(reference[:1800])+'

实际返回（节选）

'+escape(raw[:1800])+'

程序返回或验收结果

'+escape(error)+'
' + body += section((browser_method_label(name) if browser_method_label(name) != name else name.title())+'：'+str(stat['passed'])+'/'+str(model['cases'])+' 题通过', + '
'+paragraph(chapter['explanation'])+distribution+example+'
', 'browser-'+name) + review_path = contained_path(out,'result-review.json') + review = json.loads(review_path.read_text()) if review_path.is_file() else None + body += section('结论与审查范围', paragraph(content['conclusions'])+ + ('

独立审查范围见审查记录，未审查结果仍按程序验收结果展示。

' if review else + '

独立结果审查：0/'+str(model['results'])+'。未经独立复判的程序通过结果：'+str(sum(stats[n]['passed'] for n in names))+'。本轮报告使用统一评分器的原始验收结果。

'), 'conclusions') + body += '
' + evidence = {'schema':'webfetch_browser_interpretation_evidence/1','input_sha256':content['evidence_sha256'], + 'cases':model['cases'],'results':model['results'],'providers':names,'interfaces':interfaces, + 'stats':{n:{k:v for k,v in stats[n].items() if k!='specimen'} for n in names}} + guide = contained_path(out,'guide');guide.mkdir(parents=True,exist_ok=True) + (guide/'interpretation-evidence.json').write_text(json.dumps(evidence,ensure_ascii=False,indent=2)+'\n') + from .report import EXTRA + standalone = _offline_browser_article(body,out,cases,model) + (guide/'article.html').write_text(''+escape(content['title'])+''+standalone+'') + return body+'' + + +def check_current_browser_article(out, cases, records, binding): + """Verify numerical populations, unique highlights and complete drilldowns.""" + from html.parser import HTMLParser + from urllib.parse import urlsplit + model = current_browser_article_model(out, cases, records, binding) + from .report_interfaces import browser_task_interface_matrix + interfaces = browser_task_interface_matrix(cases, model['providers']) + expected_interfaces = {(name,row['track']):[(part['mode'],part['count'],str(part['count'])+' 题：'+part['description']+'等待：'+part['wait']) for part in row['cells'][name]] + for row in interfaces['rows'] for name in model['providers']} + expected = {} + failures = {} + for name in model['providers']: + stats = model['stats'][name] + expected[name, 'pass_rate'] = stats['pass_rate'] + for population in ('all','successful'): + for key in ('mean','p50','p95'): + expected[name, population+'_'+key] = stats[population][key] + for track, group in stats['tracks'].items(): + expected[name, 'family_'+track] = group['passed']/group['n'] + failures.update({(name,key):count for key,count in stats['failure_counts'].items()}) + class Inspect(HTMLParser): + def __init__(self): + super().__init__();self.metrics=[];self.failures=[];self.chapters=[];self.errors=[];self.links=[];self.active=None + self.interfaces={};self.interface=None;self.route=None + def handle_starttag(self, tag, attrs): + data=dict(attrs) + if 'data-browser-interface' in data: + self.interface=(data['data-browser-interface'],data['data-track']) + if self.interface in self.interfaces:raise ValueError('Duplicate browser task interface') + self.interfaces[self.interface]=[] + if 'data-route-mode' in data: + if self.interface is None:raise ValueError('Browser route has no task interface') + self.route=[data['data-route-mode'],int(data['data-case-count']),''] + self.interfaces[self.interface].append(self.route) + if 'data-browser-metric' in data: + self.active=[data,''];self.metrics.append(self.active) + if 'data-browser-failure' in data: + self.failures.append((data['data-browser-failure'],data['data-category'],int(data['data-count']))) + if 'data-browser-chapter' in data:self.chapters.append(data['data-browser-chapter']) + if 'data-browser-error' in data:self.errors.append(data['data-browser-error']) + if 'data-browser-result' in data:self.links.append((data['data-browser-result'],data.get('href'))) + def handle_data(self,text): + if self.active is not None:self.active[1]+=text + if self.route is not None:self.route[2]+=text + def handle_endtag(self,tag): + if tag=='span':self.active=None;self.route=None + for relative in ('guide/index.html','guide/article.html'): + page=contained_path(out,relative) + inspect=Inspect();inspect.feed(page.read_text()) + actual_interfaces={key:[tuple(part) for part in parts] for key,parts in inspect.interfaces.items()} + if actual_interfaces != expected_interfaces: + raise ValueError('Browser task interface matrix is missing or differs from declared tasks') + actual={} + for data,text in inspect.metrics: + key=(data['data-provider'],data['data-browser-metric']) + if key in actual or key not in expected:raise ValueError('Duplicate or unknown browser metric') + value=expected[key] + if data.get('data-value') != ('' if value is None else str(value)): + raise ValueError('Browser metric value differs from evidence') + peers=[v for (name,metric),v in expected.items() if metric==key[1] and v is not None] + high=key[1]=='pass_rate' or key[1].startswith('family_') + extreme=(max(peers) if high else min(peers)) if peers else None + winner=value is not None and value==extreme and peers.count(extreme)==1 + if (data.get('data-winner')=='true') != winner: + raise ValueError('Browser metric unique highlight is missing or wrong') + visible='—' if value is None else (f'{value:.1%}' if high else f'{value:,.0f}') + if text.strip()!=visible:raise ValueError('Visible browser metric differs from evidence') + actual[key]=value + if actual != expected:raise ValueError('Browser metric population is incomplete') + if len(inspect.failures)!=len(failures) or {(n,k):v for n,k,v in inspect.failures}!=failures: + raise ValueError('Browser failure distribution differs from evidence') + for values in (inspect.chapters,inspect.errors): + if Counter(values)!=Counter(model['providers']):raise ValueError('Browser chapter or program error evidence is missing') + if Counter(n for n,link in inspect.links)!=Counter(model['providers']): + raise ValueError('Browser result drilldown is missing') + for name,link in inspect.links: + attempt = model['stats'][name]['specimen']['attempt_id'] + expected_link = '#offline-result-'+attempt if relative.endswith('article.html') else '../attempts/'+attempt+'.html' + if link!=expected_link or (not link.startswith('#') and not (page.parent/urlsplit(link).path).is_file()): + raise ValueError('Browser result evidence target is broken') + if relative.endswith('article.html'): + from bs4 import BeautifulSoup + offline = BeautifulSoup(page.read_text(),'html.parser') + if offline.select('script,link[rel="stylesheet"]'): + raise ValueError('Independent article requires external runtime') + for name in model['providers']: + row = model['stats'][name]['specimen'] + anchor = offline.find(id='offline-result-'+row['attempt_id']) + output = anchor.select_one('[data-standalone-output]') if anchor else None + if output is None or output.get_text() != contained_path(out,row['artifact_path']).read_text(encoding='utf-8',errors='replace'): + raise ValueError('Independent article omits complete specimen output') + return model + + def supports_authored_explainer(cases): """Do not apply a mixed-track article to a different task family.""" - return all(case.get('track', 'web') in TRACK_NAMES for case in cases.values()) + # Historical authored prose has no risk-control chapter. New bound cohort + # content is selected before this legacy presentation guard. + return all(case.get('track', 'web') in set(TRACK_NAMES) - {'risk_control'} for case in cases.values()) -def render_explainer(out, cases, records, *, table=None, cell=None, link=None, authored=True): +def render_explainer(out, cases, records, *, table=None, cell=None, link=None, authored=True, content_binding=None): """One authored body; only image URL serialization differs for the offline export.""" out = Path(out) + dataset_path = out / 'dataset.json' + if content_binding is None and dataset_path.is_file(): + content_binding = json.loads(dataset_path.read_text()).get('comparison', {}).get('report_interpretation') + if authored and content_binding is not None: + return _render_current_browser_article(out, cases, records, content_binding) if (not authored or not supports_authored_explainer(cases) or (records and all(row.get('provider_type') == 'fixture' for row in records))): passes = sum(task_passed(row['ok'], row['metrics'], diff --git a/harness/web_fetch/report_i18n.py b/harness/web_fetch/report_i18n.py index 95f751223..6b0144922 100644 --- a/harness/web_fetch/report_i18n.py +++ b/harness/web_fetch/report_i18n.py @@ -134,6 +134,17 @@ '合成示例': 'Synthetic example', '本报告由本地测试适配器生成。没有正式评测发布记录。': 'This report was generated by a local test adapter and has no formal benchmark release history.', + "Lightpanda、Obscura 与 Moli 的普通网页使用 CLI 正文输出。Moli 的直接文档和媒体 URL 使用裸 fetch：正文任务在预算内解析，文件任务校验字节。": "Lightpanda, Obscura and Moli use CLI body output for ordinary web pages. Moli uses bare fetch for direct document and media URLs: body parsing stays within the time budget and file tasks validate bytes.", + "Chromium 由 Playwright 取完整渲染 DOM，再用 node-html-markdown 2.0.0 转换。五种方法都不调用模型。": "Playwright obtains Chromium's complete rendered DOM, then node-html-markdown 2.0.0 converts it. None of these five methods calls a model.", + "实际版本、配置和调用路线见“环境与复现”及逐次调用记录。": "See Environment & reproduction and the individual call records for measured versions, settings and execution routes.", + "保留同一次调用的原生 Markdown 和最终页面地址。Lightpanda 解包原生 JSON；Obscura 执行固定哈希的官方 Markdown 脚本，并读取当前地址；页面完成条件不变。": "Retain native Markdown and the final page URL from the same call. Lightpanda unwraps its native JSON; Obscura evaluates the hash-pinned upstream Markdown script and reads the current URL, preserving the page completion conditions.", + "安装与本报告版本和 SHA-256 一致的浏览器，配置 CHROMIUM_BINARY；详见“环境与复现”。": "Install the browser matching this report's version and SHA-256, and configure CHROMIUM_BINARY; see Environment & reproduction.", + "登记配置：": "Registered configuration:", + "安装与本报告版本和 SHA-256 一致的二进制，配置 MOLI_BINARY；详见“环境与复现”。": "Install the binary matching this report's version and SHA-256, and configure MOLI_BINARY; see Environment & reproduction.", + "只有 Kitesurf 的账户级 Cloudflare Browser Run 路线需要供应商账号与凭据；其余方法是本地二进制，不需要账号，但仍需安装并明确指定可执行文件。": "Only the account-level Kitesurf Cloudflare Browser Run route requires a provider account and credentials; the other methods use local binaries that must be installed and explicitly selected.", + "查看各方法执行结果 →": "View execution results for all methods →", + "浏览器版本与二进制来源": "Browser versions and binary sources", + "风控网页正文": "Risk-control page body text", "本报告的任务范围与验收规则": "Task scope and acceptance rules", "查看全部任务及验收规则": "View all tasks and acceptance rules", "查看结果、实际返回及失败证据": "View results, actual outputs, and failure evidence", @@ -150,8 +161,7 @@ "从官方 v1.1.7 release 获取二进制，配置 MOLI_1_1_7_BINARY。": "Obtain the binary from the official v1.1.7 release and configure MOLI_1_1_7_BINARY.", "Moli 1.1.7 正式发布": "Moli 1.1.7 official release", "本方法只使用官方 1.1.7 发行二进制，与其他 Moli 版本分开计分。普通网页使用原生 Markdown；直接文档与媒体的裸 fetch 路线在正式二进制上实测确认。正文任务若取得受支持的原始文档，会在同一计时预算内解析文字；文件任务直接验收字节。不在失败后切换引擎或输出模式。": "This method uses only the official 1.1.7 release binary and is scored separately from other Moli versions. Ordinary web pages use native Markdown; bare-fetch routes for direct documents and media were verified on the release binary. If a body-text task retrieves a supported original document, text is parsed within the same timing budget; file tasks validate bytes directly. The method does not switch engines or output modes after a failure.", - "历史访问受限网址正文": "Historically access-restricted URL body text", - "历史访问受限网址内容": "Historically access-restricted URL content", + "风控网站正文": "Risk-control website body text", "请求指定网址": "Request the specified URL", "识别目标页或拦截页": "Identify the target page or a blocked page", "提取首次加载内容": "Extract the initially loaded content", @@ -1217,7 +1227,7 @@ "完整报告中的任务均明确判为通过或失败。诊断构建可显示未测，但会被标为不可靠且不能发布。通过不证明事实准确性、实时性或整页完整性。": "Every task in a complete report has an explicit pass or fail verdict. Diagnostic builds may show unmeasured tasks, but are marked unreliable and cannot be published. A pass does not prove factual accuracy, freshness, or whole-page completeness.", "所有分数由本轮调用记录计算。网页与服务可能变化；结论仅适用于本轮样本和配置。": "All scores are computed from the recorded calls. Web pages and services may change; conclusions apply only to this sample and configuration.", "先比较方法在同一任务上的成败组合，再筛选全部执行记录。": "Compare methods on the same tasks first, then filter the complete execution record.", - "按同一任务逐项比较，不用各方法不同的成功样本拼接结论。成功与失败均沿用主榜已核验判定；点击方法可查看标准答案、实际返回和原因。": "Methods are compared task by task; conclusions never splice together different success samples. Pass and fail states use the reviewed primary verdicts. Open a method to inspect its reference, output, and rationale.", + "按同一任务逐项比较，不用各方法不同的成功样本拼接结论。成功与失败均沿用主榜的程序判定；点击方法可查看标准答案、实际返回和原因。": "Methods are compared task by task; conclusions never splice together different success samples. Pass and fail states use the primary program verdicts. Open a method to inspect its reference, output, and rationale.", "记录每轮核验的范围、发现、纠正和未决事项。": "Each review round records its scope, findings, corrections, and unresolved items.", "本轮没有绑定独立核验记录。": "No independent review record is bound to this build.", "代码、评测集和复现说明以 browser-eval 仓库为准；本页只列本轮必要配置。": "The browser-eval repository is authoritative for code, dataset, and reproduction instructions. This page lists only the configuration required for this run.", @@ -3364,7 +3374,7 @@ def release_translations(history: dict) -> dict[str, str]: + " / sent " + match.group(2).replace("、", ", "), )) REGEX.insert(0, ( - re.compile(r"^(?:Chromium|Lightpanda|Moli 1\.1\.1|Obscura)(?:、(?:Chromium|Lightpanda|Moli 1\.1\.1|Obscura))+$"), + re.compile(r"^(?:Chromium|Lightpanda|Moli(?: \d+(?:\.\d+)*)?|Obscura)(?:、(?:Chromium|Lightpanda|Moli(?: \d+(?:\.\d+)*)?|Obscura))+$"), lambda match: match.group(0).replace("、", ", "), )) REGEX.insert(0, ( @@ -3414,6 +3424,11 @@ def release_translations(history: dict) -> dict[str, str]: (re.compile(r"^(\d+) 个案例 · (\d+) 次调用，机器判分。本轮未重新核验来源。$"), r"\1 cases · \2 calls, machine-scored. Sources were not reverified for this run."), (re.compile(r"^每种方法均使用同一组 (\d+) 个任务，包含通过、未通过、错误和超时。耗时是单次调用从输入到返回的时间，不包含调用之间的统一冷却间隔。各方法串行执行，仍受本地与远程环境、网络和产品内部限制影响。$"), r"Each method uses the same \1 tasks, including passes, failures, errors, and timeouts. Latency runs from input to return for a single call, excluding shared cooldown intervals. Each method runs serially and remains subject to local and remote environments, networks, and product limits."), ] +TEXT.update({ + '普通正文使用 Markdown；动态完成条件与正文提取范围分离。直接文档使用 Moli 原生正文接口；文件使用原生 CDP 的原始响应；附件事件使用同一二进制的原始附件接口，计入同一时限。图片、视频资源只从已取得的 DOM 中读取，不另抓模块挖掘链接。文档原生文字读取计入同一截止预算；私网保护默认开启，只对冻结公开测试域名使用精确白名单；不在失败后切换格式或借用其他 provider。': 'Ordinary body text uses Markdown; readiness conditions are separate from extraction scope. Direct documents use Moli native body output. Files use native CDP responses; attachment events use the same binary’s raw attachment interface within the deadline. Image and video resources are read only from the retrieved DOM, without fetching modules for more links. Native document text retrieval shares the deadline. Private-network protection is enabled by default, with exact exceptions for frozen public test domains. Failures do not trigger format changes or another provider.', + '读取预声明范围；资源和嵌入文档使用各浏览器共用的脚本': 'Read the declared scope; resource and embedded-document tasks use the same scripts across browsers.', + '普通正文使用 Markdown；动态完成条件与正文提取范围分离。直接文档与文件使用 Moli 原生 CDP 的原始响应；附件事件使用同一二进制的原始附件接口，计入同一时限。图片、视频资源只从已取得的 DOM 中读取，不另抓模块挖掘链接。文档解析计入同一截止预算；私网保护默认开启，只对冻结公开测试域名使用精确白名单；不在失败后切换格式或借用其他 provider。': 'Ordinary pages use Markdown; readiness conditions are separate from extraction scope. Direct documents and files use Moli native CDP responses; attachment events use the same binary’s raw attachment interface within the same deadline. Image and video resources are read only from the retrieved DOM, without crawling modules for more links. Document parsing shares the deadline. Private-network protection is enabled by default, with exact exceptions for frozen public test domains. Failures do not trigger format changes or another provider.', +}) REGEX.extend([ (re.compile(r"^本报告纳入 ([\d,]+) 个逻辑结果。运行参数见下表；保留的物理调用与输出见各结果详情。$"), diff --git a/harness/web_fetch/report_interfaces.py b/harness/web_fetch/report_interfaces.py index d11938dd9..1fe1320ba 100644 --- a/harness/web_fetch/report_interfaces.py +++ b/harness/web_fetch/report_interfaces.py @@ -1,7 +1,9 @@ """Interfaces actually used by the benchmark, not a claim to cover entire SDKs.""" import json +import re from html import escape from .safety import contained_path +from .methods import native_browser_engine, browser_method_registry URL='https://example.com/' HEADLESS_ACCESS={ @@ -14,21 +16,21 @@ 'lightpanda':{'account':'不需要供应商账号或 API Key。','configuration':'从官方项目获取二进制，配置 LIGHTPANDA_BINARY；不使用托管服务凭据。','env_vars':['LIGHTPANDA_BINARY'],'links':[('官方项目与安装说明','https://github.com/lightpanda-io/browser')]}, 'obscura':{'account':'不需要供应商账号或 API Key。','configuration':'从官方项目获取程序及配套 worker，配置 OBSCURA_BINARY。','env_vars':['OBSCURA_BINARY'],'links':[('官方项目与安装说明','https://github.com/h4ckf0r0day/obscura')]}, 'moli-1.0.6':{'account':'不需要供应商账号或 API Key。','configuration':'从官方 v1.0.6 release 获取二进制，配置 MOLI_1_0_6_BINARY；发布归档与可执行文件均校验 SHA-256。','env_vars':['MOLI_1_0_6_BINARY'],'links':[('Moli 1.0.6 正式发布','https://github.com/lexmount/moli/releases/tag/v1.0.6')]}, - 'moli':{'account':'不需要供应商账号或 API Key。','configuration':'从官方 v1.1.5 release 获取二进制，配置 MOLI_BINARY；不是 Lexmount 云服务接口。','env_vars':['MOLI_BINARY'],'links':[('Moli 1.1.5 正式发布','https://github.com/lexmount/moli/releases/tag/v1.1.5')]}, + 'moli':{'account':'不需要供应商账号或 API Key。','configuration':'安装与本报告版本和 SHA-256 一致的二进制，配置 MOLI_BINARY；详见“环境与复现”。','env_vars':['MOLI_BINARY'],'links':[('官方项目与安装说明','https://github.com/lexmount/moli')]}, 'moli-1.1.7':{'account':'不需要供应商账号或 API Key。','configuration':'从官方 v1.1.7 release 获取二进制，配置 MOLI_1_1_7_BINARY。','env_vars':['MOLI_1_1_7_BINARY'],'links':[('Moli 1.1.7 正式发布','https://github.com/lexmount/moli/releases/tag/v1.1.7')]}, 'moli-1.1.11':{'account':'不需要供应商账号或 API Key。','configuration':'从官方 v1.1.11 release 获取二进制，配置 MOLI_1_1_11_BINARY。','env_vars':['MOLI_1_1_11_BINARY'],'links':[('Moli 1.1.11 正式发布','https://github.com/lexmount/moli/releases/tag/v1.1.11')]}, 'moli-pr643':{'account':'不需要供应商账号或 API Key。','configuration':'从 Moli PR #643 的固定源码提交构建，配置 MOLI_PR643_BINARY；此方法是候选构建版。','env_vars':['MOLI_PR643_BINARY'],'links':[('Moli 源码 PR','https://github.com/lexmount/moli/pull/643')]}, 'moli-1.1.3':{'account':'不需要供应商账号或 API Key。','configuration':'从官方 v1.1.3 release 获取二进制，配置 MOLI_1_1_3_BINARY；发布包与可执行文件均校验 SHA-256。','env_vars':['MOLI_1_1_3_BINARY'],'links':[('Moli 1.1.3 正式发布','https://github.com/lexmount/moli/releases/tag/v1.1.3')]}, - 'chromium':{'account':'不需要 Google 账号或 API Key；不加载个人登录会话。','configuration':'通过 Playwright 安装匹配的 Chromium，配置 CHROMIUM_BINARY。','env_vars':['CHROMIUM_BINARY'],'links':[('官方浏览器安装说明','https://playwright.dev/python/docs/browsers')]}, + 'chromium':{'account':'不需要 Google 账号或 API Key；不加载个人登录会话。','configuration':'安装与本报告版本和 SHA-256 一致的浏览器，配置 CHROMIUM_BINARY；详见“环境与复现”。','env_vars':['CHROMIUM_BINARY'],'links':[('Chrome for Testing','https://googlechromelabs.github.io/chrome-for-testing/'),('官方浏览器安装说明','https://playwright.dev/python/docs/browsers')]}, } def spec(entry,auth,params,fields,request,source,notes): return dict(entry=entry,auth=auth,params=params,fields=fields,request=request,source=source,notes=notes) SPECS={ 'moli-webfetch':spec('CLI · 按任务预选 Markdown、JSON 诊断、DOM eval 或裸响应路线','本地二进制；无 API Key', - [('URL','string · 必填','公开 HTTP(S) 页面或直接文件'),('--dump markdown','普通正文','直接取得 Markdown'),('--eval','动态 / 资源页 / DOCX','一次返回有界记录文本、结构化 URL 或页面运行时文档字节'),('--wait-until / --wait-selector / --wait-script','string','使用预声明的生命周期或动态完成条件'),('--timeout / --http-timeout','number','毫秒；导航、HTTP、进程与本地文档解析共享同一端到端预算'),('--block-private-networks','flag','默认启用；仅对冻结公开测试域名的精确白名单省略')], + [('URL','string · 必填','公开 HTTP(S) 页面或直接文件'),('--dump markdown','普通正文','直接取得 Markdown'),('--eval','动态 / 资源页 / 嵌入文档','读取预声明范围；资源和嵌入文档使用各浏览器共用的脚本'),('--wait-until / --wait-selector / --wait-script','string','使用预声明的生命周期或动态完成条件'),('--timeout / --http-timeout','number','毫秒；导航、HTTP、进程与本地文档解析共享同一端到端预算'),('--block-private-networks','flag','默认启用；仅对冻结公开测试域名的精确白名单省略')], [('stdout','UTF-8 string / bytes','Markdown、eval 结果或直接文件原始字节'),('stderr','string','独立诊断；不混入正文'),('exit_code','integer','0 表示工具请求完成；正文或文件为空仍由任务规则单独判失败'),('status_code / final_url','可能未提供','CLI 未提供时保持未知，不伪造 HTTP 200 或最终 URL')], - {'ordinary_body':'moli fetch --dump markdown … URL','challenge':'moli fetch --dump json --trace-network … URL → 最终 DOM 本地转 Markdown','dynamic_or_resource_page':'moli fetch --eval … URL','direct_pdf_or_media':'moli fetch … URL','direct_docx':'moli fetch --eval … URL'}, - 'harness/web_fetch/providers/moli.py','普通正文使用 Markdown；动态完成条件与正文提取范围分离。直接 PDF 与媒体使用不带 --dump 的原始响应路线；DOCX 因当前 Moli CLI 的 OOXML 误分类，在同一次 CLI 运行中由页面运行时二次 fetch 原始字节并明确披露。文档解析计入同一截止预算；私网保护默认开启，只对冻结公开测试域名使用精确白名单；不在失败后切换格式或借用其他 provider。'), + {'ordinary_or_popular_body':'moli fetch --dump markdown … URL','exact_page_identity':'moli fetch --dump markdown … URL；官方同次日志无法确认最终网址时保留空值','dynamic_or_resource_page':'moli fetch --eval … URL','direct_document_or_file':'Moli 原生 CDP → 原始响应；附件事件 → 同一 Moli 原始附件接口；文档 → 原生正文接口'}, + 'harness/web_fetch/providers/moli.py','普通正文使用 Markdown；动态完成条件与正文提取范围分离。直接文档使用 Moli 原生正文接口；文件使用原生 CDP 的原始响应；附件事件使用同一二进制的原始附件接口，计入同一时限。图片、视频资源只从已取得的 DOM 中读取，不另抓模块挖掘链接。文档原生文字读取计入同一截止预算；私网保护默认开启，只对冻结公开测试域名使用精确白名单；不在失败后切换格式或借用其他 provider。'), 'lexmount-global':spec('POST https://api.lexmount.com/v1/extract','X-API-Key: ；X-Project-Id: ', [('extract.url','string · 必填','目标网页 URL'),('extract.engine','string · 可选','指定提取引擎；本轮不传，使用服务端选择')], [('result.main_text','string','正文 → content'),('result.title / final_url','string','页面标题 / 最终地址'),('result.links / images','array','页面链接和图片 → provider_metadata'),('result.status_code','number','源站状态；与 HTTP API 状态分开'),('metadata.server_elapsed_ms','number','服务端耗时')], @@ -78,22 +80,22 @@ def spec(entry,auth,params,fields,request,source,notes): [('url','string','唯一任务输入'),('gotoOptions.waitUntil','string','networkidle2'),('browser','query','kitesurf'),('timeout','seconds','45 秒客户端端到端截止')], [('result','string','Cloudflare 原生 Markdown → content'),('success / errors','boolean / array','接口状态与错误'),('browser_ms_used','number-like','服务端浏览器计费时间；不等于客户端耗时')], {'url':URL,'gotoOptions':{'waitUntil':'networkidle2'}},'harness/web_fetch/providers/cloudflare.py', - '账号版 Browser Run 原生 Markdown 路径。客户端耗时包含公网往返、远端排队与执行；browser_ms_used 仅作附加诊断。') + '正文使用账号版 Browser Run 原生 Markdown；图片、视频资源通过指定 Kitesurf 的官方 CDP 执行共用 DOM 脚本，可读取已加载的同源 iframe。直接文档使用官方 Markdown 接口，嵌入文档通过该 CDP 读取已加载的查看器文字；直接文件通过 CDP 获取原始响应。Workers AI 文档转换不计入 Kitesurf 成绩。客户端耗时包含公网往返、排队与执行。') for engine,wait in (('lightpanda','done'),('obscura','load + adaptive settle'),('moli','done')): SPECS[engine]=spec( 'CLI · '+engine+' fetch --dump markdown','本地独立浏览器；不需要供应商 API Key', [('URL','string','唯一任务输入'),('--dump','string','markdown'),('wait_until','string',wait),('timeout','seconds','45 秒外层截止；CLI 参数按各自单位换算')], [('stdout','UTF-8 string','浏览器原生 Markdown → content'),('native_markdown_ms','number','CLI 调用及原生序列化阶段耗时'),('browser_stderr','string','浏览器诊断；不进入正文')], {'url':URL,'engine':engine,'timeout':45,'wait_until':wait,'dump':'markdown'},'harness/web_fetch/providers/headless_worker.py', - '不点击、不滚动、不补抓资源，不在失败后切换引擎或输出模式。CLI 未提供可靠 HTTP 状态和最终 URL 时保持空值。') + '正文使用原生 Markdown。文件使用浏览器已有的原始响应接口；资源与嵌入文档使用共用 DOM 脚本。嵌入文档只读取浏览器内已经加载的文字，不下载后外部解析。不在失败后切换引擎或请求外部服务补正文。接口未提供 HTTP 状态或最终 URL 时保留空值。') def moli_release_spec(version): base=SPECS['moli'] return {**base, - 'entry':f'CLI · Moli {version} 普通网页 Markdown / 直接文档与媒体自动原始响应', - 'params':[('URL','string · 必填','唯一任务输入'),('--dump markdown','普通网页正文','直接取得 Markdown'),('不传 --dump','直接文档或媒体','由 CLI 自动选择原始响应或序列化页面；不附加页面等待参数'),('timeout','seconds','45 秒外层端到端截止；文档解析包含在内')], + 'entry':f'CLI · Moli {version} 普通网页 Markdown / 直接文档与文件原生字节读取', + 'params':[('URL','string · 必填','唯一任务输入'),('--dump markdown','普通网页正文','直接取得 Markdown'),('原始字节接口','直接文档或文件','使用固定版本公开支持的接口；实际路线与原始字节见逐次记录'),('timeout','seconds','45 秒外层端到端截止；文档解析包含在内')], 'fields':[('stdout','UTF-8 string / bytes','普通网页为 Markdown；直接文件为原始字节'),('document_extractor','string','直接文档正文使用的本地解析器'),('raw_stdout_path / sha256','string','保存的原始文档或媒体证据'),('file_evaluation','object','文件格式、长度和目标 SHA-256 验收')], - 'request':{'ordinary_web':'moli fetch --dump markdown --wait-until done --timeout 45000 URL','direct_document_or_media':'moli fetch --timeout 45000 --http-timeout 45000 URL','version':version}, - 'notes':f'本方法只使用官方 {version} 发行二进制，与其他 Moli 版本分开计分。普通网页使用原生 Markdown；直接文档与媒体的裸 fetch 路线在正式二进制上实测确认。正文任务若取得受支持的原始文档，会在同一计时预算内解析文字；文件任务直接验收字节。不在失败后切换引擎或输出模式。'} + 'request':{'ordinary_web':'moli fetch --dump markdown --wait-until done --timeout 45000 URL','direct_document_or_media':'原始字节接口；具体 CDP、响应或附件路线见逐次调用记录','version':version}, + 'notes':f'本方法只使用官方 {version} 发行二进制，与其他 Moli 版本分开计分。普通网页使用原生 Markdown；直接文档与文件的原始字节和接口路线保留在逐次记录中。正文任务若取得受支持的原始文档，会在同一计时预算内解析文字；文件任务直接验收字节。不在失败后切换引擎或输出模式。'} SPECS['moli-1.0.6']=moli_release_spec('1.0.6') SPECS['moli']=moli_release_spec('1.1.5') SPECS['moli-1.1.7']=moli_release_spec('1.1.7') @@ -101,10 +103,157 @@ def moli_release_spec(version): SPECS['moli-1.1.11']=moli_release_spec('1.1.11') SPECS['chromium']=spec( 'Playwright · Chromium → DOM HTML → node-html-markdown','本地独立浏览器；不需要 Google 账号，不加载个人会话', - [('url','string','唯一任务输入'),('wait_until','string','domcontentloaded'),('timeout','seconds','45 秒端到端截止'),('converter','string','node-html-markdown 2.0.0')], + [('url','string','唯一任务输入'),('wait_until','string','load'),('timeout','seconds','45 秒端到端截止'),('converter','string','node-html-markdown 2.0.0')], [('content','string','转换后的 Markdown'),('rendered_html_path / sha256','string','转换前 DOM HTML 证据'),('fetch_html_ms / conversion_ms','number','分阶段耗时'),('browser_stderr','string','浏览器诊断；不进入正文')], - {'url':URL,'engine':'chromium','timeout':45,'wait_until':'domcontentloaded','dump':'html','converter':'node-html-markdown 2.0.0'},'harness/web_fetch/providers/headless_worker.py', - 'Chromium/CDP 没有原生 Markdown 输出，因此取完整渲染 DOM 后转换；转换计入端到端耗时，HTML 与转换器版本、哈希均保留。') + {'url':URL,'engine':'chromium','timeout':45,'wait_until':'load','dump':'html','converter':'node-html-markdown 2.0.0'},'harness/web_fetch/providers/headless_worker.py', + '网页正文取完整渲染 DOM 后转为 Markdown；图片、视频资源使用共用 DOM 脚本。直接文档使用浏览器自身返回的页面内容，嵌入文档读取已加载的查看器文字；不使用外部文档解析器。转换、下载和解析均计入耗时；原始字节、HTML 与转换器哈希均保留。') + +def method_provenance(out): + path=out/'implementation-provenance.json' + return {item['binary_sha256']:item for item in json.loads(path.read_text()).get('implementations',[])} if path.is_file() else {} + + +def method_version(record, provenance=None): + """Use executable identity, never a different engine's CDP compatibility number.""" + meta=record.get('provider_metadata') or {} + name=record.get('provider') + if name=='kitesurf':return 'Beta · 托管后端版本未公开' + bound=(provenance or {}).get(meta.get('binary_sha256'),{}) + if bound.get('version'):return bound['version'] + if meta.get('reported_version'):return meta['reported_version'] + value=meta.get('browser_version') + if name in {'moli','lightpanda','obscura'} and re.fullmatch(r'\d+\.\d+\.\d+\.\d+',str(value)): + return None + return value or meta.get('binary_version') or meta.get('moli_version') or meta.get('library_version') or meta.get('version') + + +def method_access(name, records): + engine=native_browser_engine(name) or name + access=HEADLESS_ACCESS.get(name) or HEADLESS_ACCESS.get(engine) + if access and engine!=name: + source=browser_method_registry()[name].get('source_url') + return {**access,'configuration':'使用本轮绑定提交构建的二进制；提交、构建环境与 SHA-256 见环境与复现。','links':[(name+' · 源码分支',source)] if source else access['links']} + if access is None or name!='moli':return access + versions=set() + for row in records: + if row['provider']!=name:continue + match=re.fullmatch(r'moli (\d+\.\d+\.\d+)',row.get('provider_metadata',{}).get('reported_version','')) + if match:versions.add(match.group(1)) + if len(versions)!=1:return access + version=next(iter(versions)) + return {**access,'links':[(f'Moli {version}',f'https://github.com/lexmount/moli/releases/tag/v{version}')]} + + +def method_spec(name, record): + engine=native_browser_engine(name) or name + if record and record.get('provider_type')=='moli_webfetch': + base=SPECS['moli-webfetch'] + return {**base,'entry':'CLI · Moli · 按任务预选原生正文、DOM eval 或原始响应接口'} if name not in SPECS else base + base=SPECS.get(name) or SPECS[engine] + meta=(record or {}).get('provider_metadata',{}) + if name=='moli': + match=re.fullmatch(r'moli (\d+\.\d+\.\d+)',meta.get('reported_version','')) + if match:base=moli_release_spec(match.group(1)) + else: + base={**base,'entry':'CLI · Moli','request':{k:v for k,v in base['request'].items() if k!='version'}} + return {**base,'notes':'实际版本、配置和调用路线见“环境与复现”及逐次调用记录。'} + if name=='chromium' and meta.get('navigation_wait_until') in {'load','domcontentloaded','networkidle','commit'}: + wait=meta['navigation_wait_until'] + return {**base,'request':{**base['request'],'wait_until':wait}, + 'params':[(key,kind,wait if key=='wait_until' else description) + for key,kind,description in base['params']]} + native={k:v for k,v in meta.get('native_metadata',{}).items() if k in {'format','script_sha256'}} + format_name=native.get('format') + if (name,format_name) not in {('lightpanda','lightpanda_json'),('obscura','obscura_eval')}:return base + command=('lightpanda fetch URL --dump markdown --json' if name=='lightpanda' + else 'obscura fetch URL --eval ') + request={**base['request'],'command':command,'native_metadata':native} + if name=='obscura':request.pop('dump',None) + return {**base,'entry':'CLI · '+command,'request':request, + 'params':[row for row in base['params'] if name!='obscura' or row[0]!='--dump'], + 'notes':'正文保留同一次调用的原生 Markdown 和最终地址。Lightpanda 解包原生 JSON；Obscura 执行固定哈希的官方 Markdown 脚本。直接文件分别使用 Lightpanda CDP 与 Obscura --dump original；资源和嵌入文档使用共用 DOM 读取脚本，不外部解析文档；获取与输出处理均计入耗时。'} + + + +def browser_task_interface_matrix(cases, providers): + """Describe selected task routes without receiving answers or measured scores.""" + from collections import Counter + from types import SimpleNamespace + from .providers.headless import output_mode_for_case + capabilities={ + 'moli':'内置 Rust Markdown 转换；CLI --dump markdown。最终网址仅使用同次官方日志可明确确认的值；无法确认时保留空值，不切换 JSON 或共同转换。', + 'lightpanda':'内置 Markdown 转换；CLI --dump markdown --json，同时返回正文、最终网址及错误信息。', + 'obscura':'内置 Markdown 与 LP.getMarkdown 共用官方 JS 实现；本次执行固定版本的同一脚本，并在同次 --eval 中返回最终网址。', + 'chromium':'没有直接输出 Markdown 的原生接口；取得渲染后的 DOM，再使用共同的 node-html-markdown 2.0.0。', + 'kitesurf':'官方 Browser Run /markdown?browser=kitesurf 返回 Markdown；这是托管服务的转换能力，不是客户端共同转换。', + } + markdown={ + 'moli':'CLI fetch --dump markdown → 内置转换 → Markdown', + 'lightpanda':'CLI fetch --dump markdown --json → 内置转换 → 解包 Markdown', + 'obscura':'CLI --eval 官方固定版本脚本与 location.href → 官方转换 → Markdown', + 'chromium':'Playwright/CDP 导航 → 最终 DOM → 共同转换 → Markdown', + 'kitesurf':'/markdown?browser=kitesurf → 官方托管转换 → Markdown', + } + original={ + 'moli':'原生 CDP 响应；附件事件使用同一 Moli 原始附件接口', + 'lightpanda':'原生 CDP Response.body()', + 'obscura':'CLI fetch --dump original（原始 HTTP 响应，不运行页面 JS）', + 'chromium':'Playwright/CDP 原始响应或原生下载', + 'kitesurf':'指定 Kitesurf 的官方 CDP 原始响应', + } + resource={ + 'moli':'CLI --eval 执行共用 DOM 元素读取脚本', + 'lightpanda':'原生 CDP 执行共用 DOM 元素读取脚本', + 'obscura':'CLI --eval 执行共用 DOM 元素读取脚本；内置 assets 缺少所需 srcset/picture 地址', + 'chromium':'Playwright/CDP 执行共用 DOM 元素读取脚本', + 'kitesurf':'指定 Kitesurf 的官方 CDP 执行共用 DOM 元素读取脚本', + } + runtime={'moli':'原生 CDP','obscura':'原生 CDP','lightpanda':'原生 CDP', + 'chromium':'Playwright/CDP','kitesurf':'指定 Kitesurf 的官方 CDP'} + waits={ + 'moli':'CLI 等待 done', + 'lightpanda':'Markdown CLI 等待 done', + 'obscura':'CLI 使用官方默认 load 加自适应静默等待（最多 5 秒）', + 'chromium':'页面提取等待 load', + 'kitesurf':'Markdown API 等待 networkidle2', + } + tracks=[] + for case in cases.values(): + track=case.get('track','web') + if track not in tracks:tracks.append(track) + from .report import TRACKS + order=list(TRACKS) + tracks.sort(key=lambda track:order.index(track) if track in order else len(order)) + rows=[] + for track in tracks: + group=[case for case in cases.values() if case.get('track','web')==track] + cells={} + for provider in providers: + engine=native_browser_engine(provider) + if engine not in capabilities:raise ValueError('Unregistered native browser participant: '+provider) + parts=[];counts=Counter() + for case in group: + mode=output_mode_for_case(engine,'headless_markdown',SimpleNamespace( + url=case['url'],track=track,task_kind=case.get('task_kind','body'))) + counts[mode]+=1 + for mode,count in counts.items(): + if mode=='markdown':description=markdown[engine]+('；文档必须由此接口直接返回文字，不使用外部 PDF/DOCX 解析器' if track=='document' else '') + elif mode=='document':description=original[engine]+' → 原始 PDF/DOCX → 共同 pypdf / DOCX XML 解析 → 文档正文' + elif mode=='file':description=original[engine]+' → 原始文件字节（不转 Markdown）' + elif mode=='resource':description=resource[engine]+' → 图片或视频资源地址（不转 Markdown）' + elif mode=='document_page':description=runtime[engine]+' 读取嵌入文档 → 已加载 PDF 查看器原生文字，或页面可见文字 → 文档正文；不下载后另行解析' + elif mode=='media_metadata':description=runtime[engine]+' 执行共用脚本，读取正文、作者及 JSON-LD → 页面正文与媒体元数据（不转 Markdown）' + else:raise ValueError('Unsupported browser task interface mode: '+mode) + wait=('读取原始响应或下载文件，不等待查看器绘制' if mode in {'file','document'} else waits[engine]) + if mode not in {'file','document','markdown'}: + if engine in {'lightpanda','kitesurf'} or (engine=='obscura' and mode in {'document_page','media_metadata'}): + wait='CDP 页面提取等待 load' + if mode=='document_page':wait+='；随后等待查看器文档就绪或页面内容' + parts.append({'count':count,'mode':mode,'description':description,'wait':wait}) + cells[provider]=parts + rows.append({'track':track,'count':len(group),'cells':cells}) + return {'providers':providers,'capabilities':{name:capabilities[native_browser_engine(name)] for name in providers},'rows':rows} + def render_interfaces(out,records): from .artifacts import record_public_call_log_bindings, verify_public_call_log @@ -113,19 +262,20 @@ def render_interfaces(out,records): def pre(value):return '
'+escape(value if isinstance(value,str) else json.dumps(value,ensure_ascii=False,indent=2))+'
' def fields(rows):return '
'+''.join('
'+escape(k)+' '+escape(t)+'
'+escape(d)+'
' for k,t,d in rows)+'
' measured={r['provider'] for r in records} - names=sorted(SPECS,key=lambda name:(name not in measured,name)) + names=sorted(set(SPECS)|{name for name in measured if native_browser_engine(name)},key=lambda name:(name not in measured,name)) body='

按本轮适配器介绍，不混用完整产品能力。输入示例展示正文配置，资源配置差异单列；它们不是每条历史请求的逐字回放。各调用所属批次及实际配置见“环境与复现”。输出示例取存档调用，经适配器整理后的真实节选，不冒充原始 HTTP 响应。Key 和账号均用占位符。

' if native_default: body='

已测方法的输入展示保存的公开调用字段，输出展示同一次调用的实际节选。完整配置、原始响应和日志见环境与调用详情。未参评方法保留仓库登记示例。

' for i,name in enumerate(names): - s=SPECS[name];r=next((r for r in records if r['provider']==name and r['case_id']=='static-simple'),next((r for r in records if r['provider']==name),None)) + r=next((r for r in records if r['provider']==name and r['case_id']=='static-simple'),next((r for r in records if r['provider']==name),None)) + s=method_spec(name,r) if r and native_default: bindings=record_public_call_log_bindings(r) if not bindings: raise ValueError("Measured native interface requires a saved call log: "+name) request=json.loads(verify_public_call_log(out,bindings[-1]).read_text())["request"] s={**s,"params":[],"request":request,"notes":"此处展示该次公开调用记录保存的请求字段；完整配置和日志见环境与调用详情。产品内部超时以冻结配置和实际请求为准。"} - access=HEADLESS_ACCESS.get(name) + access=method_access(name,records) if r: content=contained_path(out,r['artifact_path']).read_text();sample={k:r.get(k) for k in ['ok','status_code','title','final_url']};sample['content_excerpt']=content[:450];sample['content_characters']=len(content) else:sample={'measurement':None,'note':'仓库登记接口，未纳入本轮；此处不提供虚构的运行输出。'} @@ -135,7 +285,7 @@ def fields(rows):return '
'+''.join('
body+=f'
'+escape(name)+'' body+='

'+escape(s['entry'])+'

'+escape(s['auth'])+'

' if access: - body+='

账号与获取入口

'+escape(access['account'])+'

'+escape(access['configuration'])+'

'+ ' · '.join(''+escape(label)+'' for label,url in access['links'])+'

官方说明核验：2026-09-07。只列账号要求、获取入口与占位符，不公开真实凭据。

' + body+='

账号与获取入口

'+escape(access['account'])+'

'+escape(access['configuration'])+'

'+ ' · '.join(''+escape(label)+'' for label,url in access['links'])+'

实际版本和接口见运行记录；下方链接指向官方说明。

' sample_label='本轮输出节选 · 统一记录' if r else '本轮未执行' body+='

输入

'+fields(s['params'])+('

本轮保存的调用字段

' if r and native_default else '

调用输入示例

')+pre(s['request'])+'

输出

'+fields(s['fields'])+'

'+sample_label+'

'+pre(sample)+'
' measured_link='该次完整输出与判定 →' if r else '未纳入本轮；参数说明为仓库登记模式，不计入本轮结果' diff --git a/harness/web_fetch/report_methods.py b/harness/web_fetch/report_methods.py index 0e7bb7af3..30aa26ddd 100644 --- a/harness/web_fetch/report_methods.py +++ b/harness/web_fetch/report_methods.py @@ -1,7 +1,7 @@ """Provider-centred introduction and reproducible, sample-scoped adoption counts.""" import json from pathlib import Path -from .methods import method_info, method_page_model +from .methods import method_info, method_page_model, is_browser_comparison, browser_method_label from .report_components import tabs,cards from .report_interfaces import render_interfaces from .report_capabilities import render_capabilities @@ -12,12 +12,12 @@ 'lightpanda':('本地浏览器 CLI','Lightpanda fetch --dump markdown，直接输出浏览器原生 Markdown。','https://github.com/lightpanda-io/browser'), 'obscura':('本地浏览器 CLI','Obscura fetch --dump markdown；标准模式，不启用 stealth 或备用引擎。','https://github.com/h4ckf0r0day/obscura'), 'moli-1.0.6':('本地浏览器 CLI','普通网页使用 fetch --dump markdown；直接文档和媒体使用裸 fetch。原始文档的本地正文解析计入耗时，文件任务直接校验字节。','https://github.com/lexmount/moli/releases/tag/v1.0.6'), - 'moli':('本地浏览器 CLI','普通网页使用 fetch --dump markdown；直接文档和媒体使用裸 fetch。原始文档的本地正文解析计入耗时，文件任务直接校验字节。','https://github.com/lexmount/moli/releases/tag/v1.1.5'), + 'moli':('本地浏览器 CLI','普通网页使用 fetch --dump markdown；直接文档和媒体使用裸 fetch。原始文档的本地正文解析计入耗时，文件任务直接校验字节。','https://github.com/lexmount/moli'), 'moli-1.1.7':('本地浏览器 CLI','普通网页使用 fetch --dump markdown；直接文档和媒体使用裸 fetch。原始文档的本地正文解析计入耗时，文件任务直接校验字节。','https://github.com/lexmount/moli/releases/tag/v1.1.7'), 'moli-1.1.11':('本地浏览器 CLI','普通网页使用 fetch --dump markdown；本轮风险网站评测固定 debug 日志并逐次完整留存。','https://github.com/lexmount/moli/releases/tag/v1.1.11'), 'moli-1.1.3':('本地浏览器 CLI','普通网页使用 fetch --dump markdown；直接文档和媒体使用裸 fetch。原始文档的本地正文解析计入耗时，文件任务直接校验字节。','https://github.com/lexmount/moli/releases/tag/v1.1.3'), 'chromium':('本地浏览器 CDP','Playwright 启动 Chromium，取完整渲染 DOM，再由 node-html-markdown 2.0.0 生成 Markdown。','https://playwright.dev/python/docs/browsers'), - 'moli-webfetch':('本地 CLI','Moli：普通正文用 Markdown，挑战题用 JSON 返回最终 DOM、状态和跳转诊断；动态等待与正文范围分离。DOCX 在同一次 CLI 运行中二次 fetch 原始字节；不调用模型。','https://github.com/lexmount/moli/tree/v1.1.4/skills/moli-webfetch'), + 'moli-webfetch':('本地 CLI','Moli：普通正文用 Markdown，挑战题用 JSON 返回最终 DOM、状态和跳转诊断；动态等待与正文范围分离。DOCX 在同一次 CLI 运行中二次 fetch 原始字节；不调用模型。','https://github.com/lexmount/moli/tree/main/skills/moli-webfetch'), 'cowagent-built-in':('项目内置','CowAgent 内的 HTTP 抓取和文本、文档解析工具。','https://github.com/zhayujie/CowAgent/blob/main/agent/tools/web_fetch/web_fetch.py'), 'lexmount-global':('托管 API','Lexmount 国际 API，返回网页主正文与元数据。','https://browser.lexmount.com/en/docs/webfetch'), 'lexmount-cn':('托管 API','Lexmount 中国区 API；与国际 API 分开实测，不重复计算品牌采用。','https://browser.lexmount.com/en/docs/webfetch'), @@ -30,8 +30,8 @@ 'langchain-webbase':('本地开源库','WebBaseLoader 使用 HTTP 与 BeautifulSoup 取得网页文字。','https://reference.langchain.com/python/langchain-community/document_loaders/web_base/WebBaseLoader'), 'crawl4ai':('本地开源库','本地浏览器渲染，再抽取 Markdown、媒体及链接信息。','https://docs.crawl4ai.com/'), } -DISPLAY_NAMES={'moli-1.0.6':'Moli 1.0.6','moli':'Moli 1.1.5','moli-1.1.3':'Moli 1.1.3','moli-1.1.7':'Moli 1.1.7','moli-1.1.11':'Moli 1.1.11'} -def display_name(name):return DISPLAY_NAMES.get(name,name) +DISPLAY_NAMES={'moli-1.0.6':'Moli 1.0.6','moli':'Moli','moli-1.1.3':'Moli 1.1.3','moli-1.1.7':'Moli 1.1.7','moli-1.1.11':'Moli 1.1.11'} +def display_name(name):return DISPLAY_NAMES.get(name,browser_method_label(name)) def adoption_counts(data,names): result=[] @@ -93,7 +93,11 @@ def render_methods(out,records,*,hero,section,table,cell,link,esc,authored=True) (dest/'counts.json').write_text(json.dumps({'snapshot':data['generated_at'],'scope':'签名命中线索，不是运行调用次数或全网采用量','methods':counts},ensure_ascii=False,indent=2)+'\n') intro=[];rows=[];details='' for name in names: - kind,desc,url=PROFILES.get(name,('测试适配器','本地可重复测试实现。','https://example.com'));r=next(r for r in records if r['provider']==name);info=method_info(r['provider_type']) + from .methods import native_browser_engine, browser_method_registry + engine=native_browser_engine(name) + kind,desc,url=PROFILES.get(name,PROFILES.get(engine,('测试适配器','本地可重复测试实现。','https://example.com'))) + if engine and engine!=name:url=browser_method_registry()[name].get('source_url',url) + r=next(r for r in records if r['provider']==name);info=method_info(r['provider_type']) intro.append(('',cell(link('../providers/'+name+'.html',display_name(name)))+cell(kind)+cell(desc)+cell(esc(info['output']))+cell(esc(info['freshness']))+cell(link(url,'文档 / 源码 ↗')))) for item in counts: name=item['provider'];n=item['repository_count'];known=n is not None @@ -114,13 +118,13 @@ def render_methods(out,records,*,hero,section,table,cell,link,esc,authored=True) ('方法归属规则','逐一读取命中的 Git 文件并校验内容。通用 web_fetch 和同名函数不能证明归属；Exa 要求 Contents 端点或 SDK 导入及 get_contents。内置方法也包括源码复制与官方仓库，不能冒充第三方集成。'), ])+'

'+link('evidence.json','完整检索证据')+' · '+link('counts.json','逐方法统计数据')+'

' body=hero('方法介绍','参评方法的官方能力、价格、接口与开源采用；实测表现见各方法执行记录。') - if records and all(r.get('provider_type') in {'headless_html','headless_markdown'} for r in records): - body+=section('参评方法',cards([('远程正文接口','Kitesurf 使用 Cloudflare 账户级 Browser Run /markdown API；耗时包含公网往返、远端排队与执行。'),('本地原生产物','Lightpanda、Obscura 与 Moli 1.1.1 的普通网页使用 CLI 正文输出。Moli 的直接文档和媒体 URL 使用裸 fetch：正文任务在预算内解析，文件任务校验字节。'),('本地 DOM 转换','Chromium 152 由 Playwright 取完整渲染 DOM，再用 node-html-markdown 2.0.0 转换。五种方法都不调用模型。')])) + if is_browser_comparison(records): + body+=section('参评方法',cards([('远程接口','Kitesurf 使用指定 browser=kitesurf 的 Cloudflare Browser Run API；耗时包含公网往返、远端排队与执行。'),('本地原生接口','Lightpanda、Obscura 与 Moli 按任务使用各自官方接口。正文、文档和媒体任务的具体调用及输出格式见报告解读中的接口表。'),('Chromium 的正文转换','Chromium 通过原生接口读取渲染后的 DOM，再用 node-html-markdown 2.0.0 转成 Markdown。其他浏览器的正文不经过共同转换；文档任务不使用外部 PDF / DOCX 解析器。所有方法均不调用模型。')])) body+=section('Kitesurf 能否本地部署',cards([('当前结论','截至 2026-09-07，Cloudflare 尚未发布 Kitesurf 源码或自行部署包；本轮只能通过 Cloudflare 账号版 Browser Run 远程测试，不能把它当作本地开源浏览器。'),('后续意向','Cloudflare 官方博客表示计划在准备好后开源，并希望允许客户部署到自己的账户。这是未来计划，不是当前已有能力。')])+'

当前官方接入说明 · 官方开源计划说明

') else: body+=section('参评方法',cards([('托管 API','Lexmount 两个区域、Firecrawl、Tavily、Exa、Cloudflare；通过各自 API 调用。'),('项目内置','CowAgent、OpenClaw、DeepSeek Harness；按本轮保留的实现与配置执行。'),('本地运行','LangChain WebBaseLoader、Crawl4AI、Moli WebFetch；使用本机网络与运行环境。')])) body+=section('能力、价格与接口',tabs('method-guide',[ - ('capabilities','官方能力',render_capabilities(out,names,table=table,cell=cell,link=link)), + ('capabilities','官方能力',render_capabilities(out,names,table=table,cell=cell,link=link,records=records)), ('pricing','价格与计费',render_pricing(out,names,table=table,cell=cell,link=link,esc=esc)), ('adoption','采用统计',f'

{data["generated_at"][:10]} 核验的公开源码候选集，包括搜索结果与登记文件。计数是已核验下限，不是全网采用量；点击数字查看仓库与文件。

'+stats+details), ('profiles','方法详情','

下表介绍本轮实际调用的实现；方法名称可打开其全部已测案例。各组覆盖数量以执行记录为准，功能范围以本轮配置为准。

'+table(['方法','形态','做什么','本轮输出','本轮新鲜度设置','来源'],intro)), diff --git a/harness/web_fetch/report_resources.py b/harness/web_fetch/report_resources.py index 1acdb214e..d67dc4893 100644 --- a/harness/web_fetch/report_resources.py +++ b/harness/web_fetch/report_resources.py @@ -6,7 +6,7 @@ PROVIDER_NAMES = { 'chromium': 'Chromium', 'lightpanda': 'Lightpanda', - 'moli': 'Moli 1.1.1', + 'moli': 'Moli', 'obscura': 'Obscura', } diff --git a/harness/web_fetch/report_style.py b/harness/web_fetch/report_style.py index 0f505332c..79764195e 100644 --- a/harness/web_fetch/report_style.py +++ b/harness/web_fetch/report_style.py @@ -1,5 +1,6 @@ """Shared visual styling for the current report and standalone editorial export.""" STYLE = r""" +.browser-interface-matrix table{min-width:1440px;table-layout:fixed}.browser-interface-matrix th:first-child{width:140px}.browser-interface-matrix td{overflow-wrap:anywhere}.browser-interface-matrix th{white-space:nowrap} :root{color-scheme:light;--bg:#f6f8fb;--panel:#fff;--panel2:#eef3f8;--ink:#17212b;--muted:#5d6b78;--line:#d9e2ea;--accent:#1268d8;--accent-soft:#e9f2ff;--violet:#6756d8;--lime:#3f7b26;--coral:#bd402f;--good:#117a47;--bad:#c22e43;--warn:#8a5a00;--callout:#fff6d8;--failure-bg:#fff0f2;--success-bg:#eaf8f0;--code-bg:#101820;--code-ink:#e9f1f5;--shadow:0 8px 28px rgba(30,55,80,.08);--max:1180px} :root[data-theme="dark"]{color-scheme:dark;--bg:#071015;--panel:#0d1a21;--panel2:#122630;--ink:#edf6f7;--muted:#9aadb4;--line:#263c46;--accent:#49d6ed;--accent-soft:#10303a;--violet:#8b7cff;--lime:#c6ed3d;--coral:#ff8168;--good:#67df9f;--bad:#ff6b7a;--warn:#ffd166;--callout:#251f13;--failure-bg:#2a1419;--success-bg:#10271d;--code-bg:#050d11;--code-ink:#d7e8eb;--shadow:none} *{box-sizing:border-box}html{scroll-behavior:smooth}body{margin:0;background:linear-gradient(180deg,var(--accent-soft) 0,transparent 240px),var(--bg);color:var(--ink);font:15px/1.65 Inter,system-ui,-apple-system,"Segoe UI","PingFang SC",sans-serif;transition:background .2s,color .2s}a{color:var(--accent);text-decoration:none}a:hover{text-decoration:underline}.wrap{width:min(var(--max),calc(100% - 40px));margin:auto}.top{border-bottom:1px solid var(--line);background:color-mix(in srgb,var(--panel) 92%,transparent);position:sticky;top:0;z-index:5;backdrop-filter:blur(14px)}nav{display:flex;align-items:center;gap:17px;overflow:auto;padding:10px 0;white-space:nowrap}.brand{font-weight:850;color:var(--ink);margin-right:auto}.theme-toggle{border:1px solid var(--line);background:var(--panel);color:var(--ink);border-radius:999px;padding:7px 12px;font:inherit;font-size:12px;cursor:pointer}.theme-toggle:hover{border-color:var(--accent)}.hero{padding:42px 0 30px}.eyebrow{font:750 12px/1.2 ui-monospace,monospace;letter-spacing:.12em;color:var(--accent)}h1{font-size:clamp(34px,5vw,58px);line-height:1.08;letter-spacing:-.04em;margin:13px 0;max-width:980px}h2{font-size:clamp(25px,3vw,36px);line-height:1.16;letter-spacing:-.025em;margin:0 0 12px}h3{font-size:19px;margin:24px 0 8px}.lead,.sub{color:var(--muted);max-width:900px}.meta{display:flex;gap:8px;flex-wrap:wrap}.pill,.tag{border:1px solid var(--line);background:var(--panel);border-radius:999px;padding:5px 10px;color:var(--muted);font-size:12px}.tag{padding:2px 7px}.grid{display:grid;grid-template-columns:repeat(12,1fr);gap:14px}.card{background:var(--panel);border:1px solid var(--line);border-radius:14px;padding:20px;box-shadow:var(--shadow)}.kpi{grid-column:span 3;border-top:3px solid var(--accent)}.kpi:nth-child(2){border-top-color:var(--violet)}.kpi:nth-child(3){border-top-color:var(--lime)}.kpi:nth-child(4){border-top-color:var(--coral)}.n{font-size:32px;font-weight:850;letter-spacing:-.04em;margin:7px 0}.section{padding:38px 0;border-top:1px solid var(--line)}.table-wrap{overflow:auto;border:1px solid var(--line);border-radius:13px;background:var(--panel);box-shadow:var(--shadow)}table{border-collapse:collapse;width:100%;min-width:760px;background:var(--panel)}th,td{padding:11px 13px;border-bottom:1px solid var(--line);text-align:left;vertical-align:top}th{background:var(--panel2);font-size:12px;letter-spacing:.03em;position:sticky;top:0}td{color:var(--ink)}tbody tr:hover{background:var(--accent-soft)}.good{color:var(--good);font-weight:750}.bad{color:var(--bad);font-weight:750}.warn{color:var(--warn);font-weight:750}.bar{height:9px;background:var(--panel2);border-radius:99px;overflow:hidden;min-width:120px}.bar>i{display:block;height:100%;background:linear-gradient(90deg,var(--accent),var(--violet));border-radius:99px}.callout{border-left:4px solid var(--warn);background:var(--callout);padding:14px 17px;border-radius:0 12px 12px 0}.failure{border-left-color:var(--bad);background:var(--failure-bg)}.success{border-left-color:var(--good);background:var(--success-bg)}pre,code{font-family:ui-monospace,SFMono-Regular,Menlo,monospace}code{color:var(--accent);overflow-wrap:anywhere}pre{white-space:pre-wrap;overflow-wrap:anywhere;background:var(--code-bg);border:1px solid var(--line);padding:18px;border-radius:12px;color:var(--code-ink);max-height:70vh;overflow:auto}.attempt-grid{display:grid;grid-template-columns:repeat(auto-fit,minmax(190px,1fr));gap:10px}.kv{background:var(--panel);border:1px solid var(--line);border-radius:11px;padding:13px}.kv b{display:block;font-size:12px;color:var(--muted);margin-bottom:4px}.footer{padding:36px 0 64px;color:var(--muted)}.empty{color:var(--muted);font-style:italic}.tiny{font-size:12px;color:var(--muted)} diff --git a/harness/web_fetch/retrieval_hints.py b/harness/web_fetch/retrieval_hints.py index b9c704eba..346ddbca4 100644 --- a/harness/web_fetch/retrieval_hints.py +++ b/harness/web_fetch/retrieval_hints.py @@ -3,9 +3,9 @@ from __future__ import annotations import json -from copy import deepcopy import re from collections.abc import Mapping +from copy import deepcopy from dataclasses import dataclass from typing import Any @@ -23,20 +23,22 @@ def native_fetch_dataset(dataset: dict) -> dict: """Select default WebFetch execution without changing scenario acceptance. - ``dynamic`` is the selector-guided execution track, not merely a description - of a site's contents. A native-completion experiment uses the ordinary web - execution track while retaining the original family and every scoring rule. + Keep the task family independent of completion strategy. An explicit + version-3 native contract selects ordinary provider completion without + selectors, delays, extraction hints or changes to acceptance rules. """ result = deepcopy(dataset) for case in result['cases']: if case.get('retrieval_hints'): - raise ValueError('Native fetch cannot discard declared retrieval hints: ' + case['id']) + try: + plan = parse_retrieval_hints(case.get('track', 'web'), case['retrieval_hints'], identifier=case['id']) + except ValueError as error: + raise ValueError('Native fetch cannot discard declared retrieval hints: ' + case['id']) from error + if plan is not None: + raise ValueError('Native fetch cannot discard declared retrieval hints: ' + case['id']) if case.get('track') == 'dynamic': - case['track'] = 'web' + case['retrieval_hints'] = {'version': 3, 'mode': 'native'} case.setdefault('source_benchmark', {})['scenario_track'] = 'dynamic' - if 'dynamic' in result.get('coverage_plan', {}): - dynamic = result['coverage_plan'].pop('dynamic') - result['coverage_plan']['web'] = list(dict.fromkeys(result['coverage_plan'].get('web', []) + dynamic)) result.setdefault('comparison', {})['retrieval_mode'] = 'native_default_completion' return result @@ -177,6 +179,12 @@ def parse_retrieval_hints( if not isinstance(hints, Mapping): raise ValueError(f"{identifier}: retrieval_hints must be an object") version = hints.get("version") + if isinstance(version, int) and not isinstance(version, bool) and version == 3: + if set(hints) != {"version", "mode"} or hints["mode"] != "native": + raise ValueError( + f"{identifier}: native retrieval_hints must contain only version=3 and mode=native" + ) + return None expected_keys = ( {"version", "wait_until", "completion", "extraction"} if version == 2 and not isinstance(version, bool) diff --git a/harness/web_fetch/risk_control.py b/harness/web_fetch/risk_control.py index ca18c2a74..9e62f4941 100644 --- a/harness/web_fetch/risk_control.py +++ b/harness/web_fetch/risk_control.py @@ -27,6 +27,47 @@ def _case_ids_digest(case_ids: list[str]) -> str: return hashlib.sha256(("\n".join(case_ids) + "\n").encode()).hexdigest() +RISK_CONTROL_CERTIFICATIONS = { + "certified_risk_control_100_moli_v1": ( + "reference-proof-verification.json", "reference_proof_verification_sha256"), + "certified_risk_control_100_moli_v2": ( + "frozen-input-verification.json", "frozen_input_verification_sha256"), +} + + +def risk_control_verification_binding(receipt: dict) -> tuple[str, str]: + try: + return RISK_CONTROL_CERTIFICATIONS[receipt["kind"]] + except (KeyError, TypeError) as error: + raise ValueError("Unsupported risk-control certification") from error + + +def frozen_risk_control_input_snapshot(profile_path: Path) -> dict: + """Verify executable inputs without reopening historical source qualification.""" + root = profile_path.resolve().parents[2] + profile_raw = profile_path.read_bytes() + profile = json.loads(profile_raw) + files = [{"role": "profile", "path": profile_path.resolve().relative_to(root).as_posix(), + "sha256": hashlib.sha256(profile_raw).hexdigest(), "bytes": len(profile_raw)}] + bindings = {"dataset": profile["dataset"], + "provider_config": {"path": profile["provider"]["config_path"], + "sha256": profile["provider"]["config_sha256"]}, + **profile["source_bindings"]} + for role, binding in sorted(bindings.items()): + path = contained_path(root, binding["path"]) + if path.is_symlink(): + raise ValueError("Missing frozen input: " + binding["path"]) + raw = read_dataset_bytes(path) if role == 'dataset' else path.read_bytes() + digest = hashlib.sha256(raw).hexdigest() + if digest != binding["sha256"]: + raise ValueError("Changed frozen input: " + binding["path"]) + files.append({"role": role, "path": binding["path"], "sha256": digest, "bytes": len(raw)}) + dataset = json.loads(read_dataset_bytes(contained_path(root, profile["dataset"]["path"]))) + inventory = json.dumps(files, ensure_ascii=False, sort_keys=True, separators=(",", ":")) + return {"case_count": len(dataset["cases"]), + "inventory_sha256": hashlib.sha256(inventory.encode()).hexdigest(), "files": files} + + def risk_control_comparison(profile: dict[str, Any], profile_sha256: str) -> dict[str, Any]: execution = profile["execution"] return { @@ -115,54 +156,67 @@ def validate_risk_control_certification( raise ValueError("Risk-control result provider identity or retry policy changed") manifest_path = contained_path(source, "manifest.json") execution_path = contained_path(source, "execution.json") - proof_verification_path = contained_path(source, "reference-proof-verification.json") + verification_name, verification_field = risk_control_verification_binding(receipt) + verification_path = contained_path(source, verification_name) manifest = json.loads(manifest_path.read_text(encoding="utf-8")) execution_record = json.loads(execution_path.read_text(encoding="utf-8")) - proof_verification = json.loads(proof_verification_path.read_text(encoding="utf-8")) - proof_rows = proof_verification.get("before", {}).get("files", []) - expected_proofs = [ - {"case_id": case["id"], **case["source_benchmark"]["risk_control"]["reference_proof"]} - for case in frozen_dataset["cases"] - ] - observed_proofs = [ - {key: row.get(key) for key in ("case_id", "root", "path", "sha256")} for row in proof_rows - ] - proof_inventory = json.dumps( - proof_rows, ensure_ascii=False, sort_keys=True, separators=(",", ":") - ) - if ( - proof_verification.get("schema_version") != 1 - or proof_verification.get("kind") != "risk_control_reference_proof_verification" - or proof_verification.get("unchanged") is not True - or (proof_verification.get("after") != proof_verification.get("before")) - or (proof_verification.get("before", {}).get("case_count") != 100) - or ( - proof_verification.get("before", {}).get("inventory_sha256") - != hashlib.sha256(proof_inventory.encode()).hexdigest() + proof_verification = json.loads(verification_path.read_text(encoding="utf-8")) + if receipt["kind"] == "certified_risk_control_100_moli_v2": + expected_inputs = frozen_risk_control_input_snapshot(profile_path) + if (proof_verification.get("schema_version") != 1 + or proof_verification.get("kind") != "risk_control_frozen_input_verification" + or proof_verification.get("unchanged") is not True + or proof_verification.get("before") != expected_inputs + or proof_verification.get("after") != expected_inputs): + raise ValueError("Risk-control frozen input verification is incomplete or mismatched") + else: + proof_rows = proof_verification.get("before", {}).get("files", []) + expected_proofs = [ + { + "case_id": case["id"], + **case["source_benchmark"]["risk_control"]["reference_proof"], + } + for case in frozen_dataset["cases"] + ] + observed_proofs = [ + {key: row.get(key) for key in ("case_id", "root", "path", "sha256")} + for row in proof_rows + ] + proof_inventory = json.dumps( + proof_rows, ensure_ascii=False, sort_keys=True, separators=(",", ":") ) - or (observed_proofs != expected_proofs) - or any(type(row.get("bytes")) is not int or row["bytes"] < 1 for row in proof_rows) - ): - raise ValueError("Risk-control reference proof verification is incomplete or mismatched") - proof_verification_sha256 = _digest(proof_verification_path) + if ( + proof_verification.get("schema_version") != 1 + or proof_verification.get("kind") != "risk_control_reference_proof_verification" + or proof_verification.get("unchanged") is not True + or proof_verification.get("after") != proof_verification.get("before") + or proof_verification.get("before", {}).get("case_count") != 100 + or proof_verification.get("before", {}).get("inventory_sha256") + != hashlib.sha256(proof_inventory.encode()).hexdigest() + or observed_proofs != expected_proofs + or any(type(row.get("bytes")) is not int or row["bytes"] < 1 for row in proof_rows) + ): + raise ValueError("Risk-control reference proof verification is incomplete or mismatched") + proof_verification_sha256 = _digest(verification_path) provider_rows = manifest.get("providers") parameters = manifest.get("parameters", {}) if ( manifest.get("attempt_count") != 100 or manifest.get("attempt_ids") != expected_attempt_ids or manifest.get("unexecuted") != [] - or (manifest.get("dataset_sha256") != frozen_dataset_sha256) - or (manifest.get("provider_config_sha256") != profile["provider"]["config_sha256"]) - or (manifest.get("reference_proof_verification_sha256") != proof_verification_sha256) - or (not isinstance(provider_rows, list)) - or (len(provider_rows) != 1) - or (provider_rows[0].get("name") != provider_name) - or (provider_rows[0].get("implementation_identity") != expected_identity) - or (parameters.get("runs") != execution["runs_per_case"]) - or (parameters.get("timeout") != execution["timeout_seconds"]) - or (parameters.get("delay") != execution["delay_seconds"]) - or (parameters.get("provider_concurrency") != execution["concurrency"]) - or (parameters.get("max_failure_retries") != execution["max_failure_retries"]) + or manifest.get("dataset_sha256") != frozen_dataset_sha256 + or manifest.get("provider_config_sha256") != profile["provider"]["config_sha256"] + or manifest.get(verification_field) + != proof_verification_sha256 + or not isinstance(provider_rows, list) + or len(provider_rows) != 1 + or provider_rows[0].get("name") != provider_name + or provider_rows[0].get("implementation_identity") != expected_identity + or parameters.get("runs") != execution["runs_per_case"] + or parameters.get("timeout") != execution["timeout_seconds"] + or parameters.get("delay") != execution["delay_seconds"] + or parameters.get("provider_concurrency") != execution["concurrency"] + or parameters.get("max_failure_retries") != execution["max_failure_retries"] ): raise ValueError("Risk-control manifest does not match the frozen execution contract") aggregate = execution_record.get("aggregate", {}) @@ -180,18 +234,19 @@ def validate_risk_control_certification( results_path = contained_path(source, "results.jsonl") if ( receipt.get("schema_version") != 1 - or receipt.get("kind") != "certified_risk_control_100_moli_v1" + or receipt.get("kind") not in RISK_CONTROL_CERTIFICATIONS or receipt.get("profile_sha256") != profile_sha256 - or (receipt.get("dataset_sha256") != _digest(contained_path(source, "dataset.json"))) - or (receipt.get("results_sha256") != _digest(results_path)) - or (receipt.get("manifest_sha256") != _digest(manifest_path)) - or (receipt.get("execution_sha256") != _digest(execution_path)) - or (receipt.get("reference_proof_verification_sha256") != proof_verification_sha256) - or (receipt.get("provider_identity_before") != expected_identity) - or (receipt.get("provider_identity_after") != expected_identity) - or (receipt.get("logical_results") != 100) - or (receipt.get("physical_calls") != 100) - or (receipt.get("complete_diagnostic_logs") is not True) + or receipt.get("dataset_sha256") != _digest(contained_path(source, "dataset.json")) + or receipt.get("results_sha256") != _digest(results_path) + or receipt.get("manifest_sha256") != _digest(manifest_path) + or receipt.get("execution_sha256") != _digest(execution_path) + or receipt.get(verification_field) + != proof_verification_sha256 + or receipt.get("provider_identity_before") != expected_identity + or receipt.get("provider_identity_after") != expected_identity + or receipt.get("logical_results") != 100 + or receipt.get("physical_calls") != 100 + or receipt.get("complete_diagnostic_logs") is not True ): raise ValueError("Risk-control certification receipt is incomplete or mismatched") @@ -237,48 +292,6 @@ def require_complete_diagnostic_logs(records: list[dict]) -> None: ) -def reference_proof_roots(profile: dict) -> dict[str, Path]: - """Resolve the external, Git-out source roots declared by the bound manifest.""" - binding = profile["source_bindings"]["qualification_manifest"] - manifest = json.loads(rooted(binding["path"]).read_text(encoding="utf-8")) - return { - key: (data_home() / relative).resolve() - for key, relative in manifest["evidence_roots"].items() - } - - -def reference_proof_snapshot(cases: list[dict], roots: dict[str, Path]) -> dict: - """Hash every declared reference text; missing or changed files fail closed.""" - files = [] - for case in cases: - binding = case["source_benchmark"]["risk_control"]["reference_proof"] - if binding.get("root") not in roots: - raise ValueError(f"Unknown reference proof root: {case['id']}") - path = contained_path(roots[binding["root"]], binding["path"]) - if not path.is_file(): - raise ValueError(f"Missing reference proof: {case['id']}") - observed = _digest(path) - if observed != binding.get("sha256"): - raise ValueError(f"Changed reference proof: {case['id']}") - files.append( - { - "case_id": case["id"], - "root": binding["root"], - "path": binding["path"], - "sha256": observed, - "bytes": path.stat().st_size, - } - ) - if len(files) != 100 or len({row["case_id"] for row in files}) != 100: - raise ValueError("Reference proof scope is not the exact 100-case cohort") - inventory = json.dumps(files, ensure_ascii=False, sort_keys=True, separators=(",", ":")) - return { - "case_count": 100, - "inventory_sha256": hashlib.sha256(inventory.encode()).hexdigest(), - "files": files, - } - - def validate(profile_path: Path) -> tuple[dict, dict, list, object, Path]: profile = json.loads(profile_path.read_text(encoding="utf-8")) if profile.get("schema_version") != 1 or profile.get("kind") != "risk_control_100_moli_v1": @@ -404,8 +417,7 @@ def main(argv: list[str] | None = None, *, certified: bool = False) -> None: profile_sha256 = _digest(profile_path) measured_dataset = measured_risk_control_dataset(profile, dataset, profile_sha256) execution = profile["execution"] - proof_roots = reference_proof_roots(profile) - proof_before = reference_proof_snapshot(dataset["cases"], proof_roots) + proof_before = frozen_risk_control_input_snapshot(profile_path) pre_identity = implementation_identity(provider) from harness.web_fetch.runner import run_evaluation @@ -428,15 +440,15 @@ def main(argv: list[str] | None = None, *, certified: bool = False) -> None: post_identity = implementation_identity(provider, refresh=True) if post_identity != pre_identity: raise ValueError("Moli implementation identity changed during execution") - proof_after = reference_proof_snapshot(dataset["cases"], proof_roots) + proof_after = frozen_risk_control_input_snapshot(profile_path) if proof_after != proof_before: - raise ValueError("Reference proof inventory changed during execution") - proof_verification_path = output / "reference-proof-verification.json" + raise ValueError("Frozen benchmark inputs changed during execution") + proof_verification_path = output / "frozen-input-verification.json" proof_verification_path.write_text( json.dumps( { "schema_version": 1, - "kind": "risk_control_reference_proof_verification", + "kind": "risk_control_frozen_input_verification", "before": proof_before, "after": proof_after, "unchanged": True, @@ -450,7 +462,7 @@ def main(argv: list[str] | None = None, *, certified: bool = False) -> None: proof_verification_sha256 = _digest(proof_verification_path) manifest = json.loads((output / "manifest.json").read_text(encoding="utf-8")) manifest["run_profile_sha256"] = profile_sha256 - manifest["reference_proof_verification_sha256"] = proof_verification_sha256 + manifest["frozen_input_verification_sha256"] = proof_verification_sha256 (output / "manifest.json").write_text( json.dumps(manifest, ensure_ascii=False, indent=2) + "\n", encoding="utf-8" ) @@ -476,13 +488,13 @@ def main(argv: list[str] | None = None, *, certified: bool = False) -> None: results_path = output / "results.jsonl" certification = { "schema_version": 1, - "kind": "certified_risk_control_100_moli_v1", + "kind": "certified_risk_control_100_moli_v2", "profile_sha256": profile_sha256, "dataset_sha256": _digest(output / "dataset.json"), "results_sha256": _digest(results_path), "manifest_sha256": _digest(output / "manifest.json"), "execution_sha256": _digest(output / "execution.json"), - "reference_proof_verification_sha256": proof_verification_sha256, + "frozen_input_verification_sha256": proof_verification_sha256, "provider_identity_before": pre_identity, "provider_identity_after": post_identity, "logical_results": len(records), diff --git a/harness/web_fetch/scoring.py b/harness/web_fetch/scoring.py index 7c0b75577..ab983802d 100644 --- a/harness/web_fetch/scoring.py +++ b/harness/web_fetch/scoring.py @@ -187,7 +187,7 @@ def score_attempt( else "unavailable" ) result.update( - page_identity_required=True, + page_identity_required=False, page_identity_status=identity_status, final_url_consistent=final_consistent, language_consistent=language_consistent, @@ -205,7 +205,7 @@ def score_attempt( and any(_same_page_url(final_url, expected) for expected in acceptable_urls) ) result.update( - page_identity_required=True, + page_identity_required=False, page_identity_status=( "verified" if final_consistent else "mismatch" if final_url else "unavailable" ), @@ -255,13 +255,6 @@ def score_attempt( body_requirements, sort_keys=True, ensure_ascii=False).encode()).hexdigest() if not result["content_score_applicable"] or not ok: result["content_pass"] = False - if result.get("page_identity_required") and result.get("page_identity_status") != "verified": - result["content_pass"] = False - if contract: - result["stable_task_pass"] = False - result["stable_content_pass"] = False - if "page_identity_not_verified" not in result["stable_task_errors"]: - result["stable_task_errors"].append("page_identity_not_verified") if not ok: if contract: result["stable_task_pass"] = False diff --git a/harness/web_fetch/scoring_regressions.py b/harness/web_fetch/scoring_regressions.py index 16f71c9af..eb807446e 100644 --- a/harness/web_fetch/scoring_regressions.py +++ b/harness/web_fetch/scoring_regressions.py @@ -46,6 +46,8 @@ def load_scoring_regressions(path: Path | None = None) -> dict[str, dict[str, An raise ValueError(f"{identifier}: artifact must be synthetic text") if not isinstance(row.get("ok"), bool) or not isinstance(row.get("expected_task_pass"), bool): raise ValueError(f"{identifier}: expected verdict fields must be booleans") + if "expected_task_measured" in row and not isinstance(row["expected_task_measured"], bool): + raise ValueError(f"{identifier}: expected_task_measured must be a boolean") if not isinstance(row.get("expected_metrics", {}), dict): raise ValueError(f"{identifier}: expected_metrics must be an object") rows[identifier] = row @@ -54,6 +56,22 @@ def load_scoring_regressions(path: Path | None = None) -> dict[str, dict[str, An return rows +def active_scoring_regressions(path: Path | None = None) -> dict[str, dict[str, Any]]: + """Run current policy controls while preserving immutable historical inputs.""" + path = default_registry_path() if path is None else path + rows = load_scoring_regressions(path) + mapping = json.loads(path.read_text()).get("superseded_cases", {}) + if not isinstance(mapping, dict): + raise ValueError("Scoring supersessions must map explicit control IDs") + for old, new in mapping.items(): + if old not in rows or new not in rows or old == new or new in mapping: + raise ValueError("Invalid scoring policy supersession") + if any(rows[old].get(field) != rows[new].get(field) for field in + ("case", "artifact", "ok", "metadata", "title", "final_url")): + raise ValueError("Policy supersession must preserve synthetic inputs") + return {key: row for key, row in rows.items() if key not in mapping} + + def require_scoring_regressions(identifiers: list[str] | tuple[str, ...], *, category: str | None = None) -> list[str]: unique = list(dict.fromkeys(identifiers)) diff --git a/harness/web_fetch/scoring_revision.py b/harness/web_fetch/scoring_revision.py index df56e1764..4d35d1fa7 100644 --- a/harness/web_fetch/scoring_revision.py +++ b/harness/web_fetch/scoring_revision.py @@ -224,6 +224,43 @@ def _revise_unlocked( return receipt +def rebind_report_interpretation(source: Path, binding: dict): + """Bind presentation content while preserving the scored output bytes.""" + from .reference_lifecycle import valid_scoring_revision + source = source.resolve() + if not valid_scoring_revision(source): + raise ValueError('Report binding requires a valid existing scoring chain') + if set(binding) != {'path', 'sha256'}: + raise ValueError('Invalid report interpretation binding') + asset = contained_path(source, binding['path']) + if hashlib.sha256(asset.read_bytes()).hexdigest() != binding['sha256']: + raise ValueError('Report interpretation content hash mismatch') + dataset_path = contained_path(source, 'dataset.json') + before = dataset_path.read_bytes() + dataset = json.loads(before) + dataset.setdefault('comparison', {})['report_interpretation'] = binding + after = (json.dumps(dataset, ensure_ascii=False, indent=2)+'\n').encode() + results_hash = hashlib.sha256((source/'results.jsonl').read_bytes()).hexdigest() + receipt = {'kind': 'report_interpretation_binding_revision', 'provider_calls': 0, + 'input_results_sha256': results_hash, 'output_results_sha256': results_hash} + for side, raw in (('input', before), ('output', after)): + digest = hashlib.sha256(raw).hexdigest() + relative = f'revision-history/dataset-{digest}.json' + target = contained_path(source, relative) + target.parent.mkdir(parents=True, exist_ok=True) + if target.exists() and target.read_bytes() != raw: + raise ValueError('Dataset history collision') + target.write_bytes(raw) + receipt[f'{side}_dataset_sha256'] = digest + receipt[f'{side}_dataset_path'] = relative + path = source/'report-binding-revisions.json' + history = json.loads(path.read_text()) if path.exists() else [] + history.append(receipt) + path.write_text(json.dumps(history, ensure_ascii=False, indent=2)+'\n') + dataset_path.write_bytes(after) + return receipt + + def revise( source: Path, canonical_path: Path, diff --git a/harness/web_fetch/stable_tasks.py b/harness/web_fetch/stable_tasks.py index be0ea4661..5783cb99c 100644 --- a/harness/web_fetch/stable_tasks.py +++ b/harness/web_fetch/stable_tasks.py @@ -6,17 +6,16 @@ """ from __future__ import annotations -from datetime import date, datetime -from html import unescape import hashlib import math import re import unicodedata +from datetime import date, datetime +from html import unescape from urllib.parse import urlsplit from .dynamic_lists import extract_named_entries, named_entry_source_pair_hash -from .reference_quality import clean_reference_text - +from .reference_quality import clean_reference_text, markdown_delimiter_pairs SCHEMA_FIELDS = { "table_row": (), @@ -62,29 +61,13 @@ def _key(text): _CARD_BOUNDARY = "\ue000" -def _closing(text, start, opening, closing): - depth = 1 - i = start + 1 - while i < len(text): - if text[i] == "\\": - i += 2 - continue - if text[i] == opening: - depth += 1 - elif text[i] == closing: - depth -= 1 - if not depth: - return i - i += 1 - return None - - def _unwrap_links(text, schema): """Unwrap balanced inline links, including multiline card labels. Only explicit markup creates boundaries; text from separate cards is never flattened together. Image alt text is not a substitute for a visible title. """ + pairs = markdown_delimiter_pairs(text) output, i = [], 0 while i < len(text): image = text.startswith("![", i) @@ -93,9 +76,9 @@ def _unwrap_links(text, schema): output.append(text[i]) i += 1 continue - end = _closing(text, start, "[", "]") + end = pairs.get(start) destination = end + 1 if end is not None else len(text) - close = _closing(text, destination, "(", ")") if text[destination:destination + 1] == "(" else None + close = pairs.get(destination) if text[destination:destination + 1] == "(" else None if close is None: output.append(text[i]) i += 1 @@ -539,7 +522,8 @@ def _parse_records(visible, schema): def _non_record_visible(content, contract, records, base_url): """Remove owned record links, without erasing identical navigation text.""" from urllib.parse import urljoin - from .dynamic_lists import named_entry_identity, _scoped_html + + from .dynamic_lists import _scoped_html, named_entry_identity from .reference_quality import markdown_links identities = {named_entry_identity(r['fields']['url'], contract) diff --git a/harness/web_fetch/storage.py b/harness/web_fetch/storage.py index e0d6283d9..68a3e6652 100644 --- a/harness/web_fetch/storage.py +++ b/harness/web_fetch/storage.py @@ -54,6 +54,26 @@ def require_selection_history(evidence: Path) -> None: ) +def frozen_input_bindings(evidence: Path) -> dict[str, str]: + """Compact publication binding for a composed, frozen dataset and its assets.""" + dataset = evidence / "dataset.json" + if evidence.is_symlink() or dataset.is_symlink(): + raise ValueError("Frozen input bindings reject symlinks") + auxiliary = {} + for name in ("sources", "list-references"): + tree = evidence / name + if tree.is_symlink(): + raise ValueError("Frozen input bindings reject symlinks") + for path in tree.rglob("*"): + if path.is_symlink(): + raise ValueError("Frozen input bindings reject symlinks") + if path.is_file(): + auxiliary[path.relative_to(evidence).as_posix()] = hashlib.sha256(path.read_bytes()).hexdigest() + return {"dataset_sha256": hashlib.sha256(dataset.read_bytes()).hexdigest(), + "auxiliary_sha256": hashlib.sha256(json.dumps(auxiliary, sort_keys=True, + separators=(",", ":")).encode()).hexdigest()} + + def require_canonical_inputs(evidence: Path, canonical: Path) -> None: """Reject a report build when its frozen inputs differ from Git.""" from .reference_inputs import input_tree, read_dataset_bytes @@ -93,6 +113,25 @@ def require_canonical_inputs(evidence: Path, canonical: Path) -> None: raise ValueError(f"Persistent evidence differs from canonical input tree: {relative}") +def require_frozen_case_source(evidence: Path, profile_path: Path, source: Path) -> None: + """Verify all task bytes against the profile-bound pre-call dataset.""" + profile = json.loads(profile_path.read_text()) + expected_hash = profile.get("source_native_dataset_sha256") + if not expected_hash or not profile.get("frozen_inputs"): + raise ValueError("Pre-call case verification requires both source and report input bindings") + if source.is_symlink() or not source.is_file(): + raise ValueError("Pre-call dataset must be a regular, non-symlink file") + raw = source.read_bytes() + if hashlib.sha256(raw).hexdigest() != expected_hash: + raise ValueError("Pre-call dataset differs from the profile source binding") + if frozen_input_bindings(evidence) != profile["frozen_inputs"]: + raise ValueError("Report differs from frozen input bindings") + original = json.loads(raw) + current = json.loads((evidence / "dataset.json").read_bytes()) + if current.get("cases") != original.get("cases") or not original.get("cases"): + raise ValueError("Report task definitions or answers differ from pre-call input") + + def require_current_report_profile(evidence: Path, profile_path: Path) -> None: """Reject evidence from another valid cohort when building the current report.""" profile = json.loads(profile_path.read_text()) @@ -100,6 +139,8 @@ def require_current_report_profile(evidence: Path, profile_path: Path) -> None: raise ValueError("WCXB evidence uses the certified cohort validator, not the legacy current-report profile validator") dataset = json.loads((evidence / "dataset.json").read_text()) records = load_jsonl(evidence / "results.jsonl") + if profile.get("frozen_inputs") is not None and frozen_input_bindings(evidence) != profile["frozen_inputs"]: + raise ValueError("Evidence differs from the profile frozen input bindings") comparison = dataset.get("comparison", {}) providers = set(profile["providers"]) if len(providers) != len(profile["providers"]): @@ -108,6 +149,11 @@ def require_current_report_profile(evidence: Path, profile_path: Path) -> None: cases = dataset.get("cases", []) case_ids = [case.get("id") for case in cases] failures = [] + if profile.get("frozen_inputs") is not None: + inventory = {track: [case['id'] for case in cases if case.get('track', 'web') == track] + for track in {case.get('track', 'web') for case in cases}} + if profile.get('case_inventory') != inventory: + failures.append('case_inventory') if dataset.get("id") != profile["dataset_id"]: failures.append("dataset_id") if len(dataset.get("cases", [])) != profile["case_count"]: @@ -179,6 +225,15 @@ def require_current_report_profile(evidence: Path, profile_path: Path) -> None: attempt_ids = [row.get("attempt_id") for row in records] if any(not identifier for identifier in attempt_ids) or len(attempt_ids) != len(set(attempt_ids)): failures.append("attempt_ids") + route_contract = profile.get("native_task_route_contract") + from .methods import native_browser_engine + if profile.get("comparison_kind") == "headless_markdown" and any(native_browser_engine(name) for name in providers) and route_contract is None: + raise ValueError("Native browser report requires native_task_route_contract") + if route_contract is not None: + if route_contract != 4: + raise ValueError("Unsupported native task route contract") + from .task_outputs import require_native_task_routes + require_native_task_routes(cases, records) resource_profile = profile.get("resource_profile") or {} if resource_profile.get("required"): resource_root = evidence / "resource-cohort" diff --git a/harness/web_fetch/task_outputs.py b/harness/web_fetch/task_outputs.py new file mode 100644 index 000000000..17bfd3e69 --- /dev/null +++ b/harness/web_fetch/task_outputs.py @@ -0,0 +1,230 @@ +"""Shared, answer-free projections of already-rendered browser content. + +Resource and metadata projections only read already-loaded content. The document +projection reads text already loaded in a browser viewer or page; it never +downloads document bytes for external conversion. +""" + +RESOURCE_PAGE_EXPRESSION = r""" +(() => { + const urls = new Set(); + const add = (value, base = location.href) => { + const raw = String(value || '').trim(); + if (!raw || /^(?:data|blob|javascript|mailto):/i.test(raw)) return; + try { urls.add(new URL(raw, base).href); } catch {} + }; + const addSrcset = (value, base) => { + // HTML srcset URLs end at whitespace; commas inside a URL are literal. + // Trailing commas delimit candidates without descriptors. + const text = String(value || ''); + let cursor = 0; + while (cursor < text.length) { + while (cursor < text.length && /[\s,]/.test(text[cursor])) cursor++; + const start = cursor; + while (cursor < text.length && !/\s/.test(text[cursor])) cursor++; + const token = text.slice(start, cursor); + add(token.replace(/,+$/, ''), base); + if (token.endsWith(',')) continue; + while (cursor < text.length && text[cursor] !== ',') cursor++; + cursor++; + } + }; + + // Enumerate the top document and bounded, already-loaded same-origin frame + // documents. This is answer-free: it follows the page's own document + // hierarchy and never receives an expected URL or case-specific selector. + const roots = [document]; + for (let cursor = 0; cursor < roots.length && roots.length < 16; cursor++) { + for (const frame of roots[cursor].querySelectorAll('iframe,frame')) { + try { + if (frame.contentDocument && !roots.includes(frame.contentDocument)) { + roots.push(frame.contentDocument); + if (roots.length >= 16) break; + } + } catch {} + } + } + for (const root of roots) { + for (const node of root.querySelectorAll(__RESOURCE_SELECTOR__)) { + for (const name of ['src', 'href', 'data', 'poster', 'srcset']) { + const value = node.getAttribute(name); + if (__VIDEO_TASK__ && name === 'href') { + let path; + try { path = new URL(value, location.href).pathname.toLowerCase(); } catch { continue; } + const media = /\.(?:mp4|webm|mov|m4v|mp3|wav|ogg)(?:\.zip)?$/.test(path); + const detail = __DETAIL_LINKS__ && /\/(?:video|videos|watch)(?:\/|$)/.test(path); + if (!media && !detail) continue; + } + const base = node.baseURI || root.baseURI || location.href; + if (name === 'srcset') addSrcset(value, base); + else add(value, base); + } + } + } + return JSON.stringify({links: [...urls]}); +})() +""" + + +def resource_detail_links(case): + """Read only the public resource kind; expected URLs never reach the browser.""" + return case.track == 'video' and any(item.get('kind') == 'video-link' for item in getattr(case, 'expected_resources', [])) + + +def resource_expression(track, detail_links=False): + """Select resource element types from the declared task, never its answers.""" + import json + selectors = {'image':'img,picture source', + 'video':'video,video source,audio,audio source,iframe,a[href]'} + if track not in selectors: + raise ValueError('Unsupported resource task family') + selector = 'a[href]' if track == 'video' and detail_links else selectors[track] + return (RESOURCE_PAGE_EXPRESSION.replace('__RESOURCE_SELECTOR__', json.dumps(selector)) + .replace('__VIDEO_TASK__', 'true' if track == 'video' else 'false') + .replace('__DETAIL_LINKS__', 'true' if detail_links else 'false')) + +MEDIA_METADATA_EXPRESSION = r""" +(() => { + const values = []; + const add = value => { + const text = String(value || '').trim(); + if (text && !values.includes(text)) values.push(text); + }; + add(document.title); + add(document.body?.innerText); + for (const node of document.querySelectorAll( + 'meta[name="author"],meta[property$=":author"],meta[itemprop="author"],' + + '[itemprop="author"],[itemprop="creator"],[itemprop="publisher"]' + )) { + add(node.content); + add(node.getAttribute?.('content')); + add(node.innerText); + for (const name of node.querySelectorAll?.('[itemprop="name"]') || []) { + add(name.content); + add(name.getAttribute?.('content')); + add(name.innerText); + } + } + let visited = 0; + const visit = (value, depth = 0) => { + if (++visited > 2048 || depth > 6 || value == null) return; + if (Array.isArray(value)) { + for (const item of value.slice(0, 64)) visit(item, depth + 1); + return; + } + if (typeof value !== 'object') return; + for (const [key, item] of Object.entries(value).slice(0, 128)) { + if (/^(?:author|creator|publisher)$/i.test(key)) { + if (typeof item === 'string') add(item); + else if (item && typeof item === 'object') add(item.name); + } + visit(item, depth + 1); + } + }; + for (const script of [...document.querySelectorAll('script[type="application/ld+json"]')] + .slice(0, 16)) { + try { visit(JSON.parse(script.textContent)); } catch {} + } + return values.join('\n\n'); +})() +""" + + +DOCUMENT_PAGE_EXPRESSION = r""" +(async () => { + const app = globalThis.PDFViewerApplication; + if (app?.pdfDocument) { + const pdf = app.pdfDocument; + if (pdf) { + const pages = []; + for (let number = 1; number <= pdf.numPages; number++) { + const page = await pdf.getPage(number), text = await page.getTextContent(); + pages.push(text.items.map(item => item.str).join(' ')); + } + return JSON.stringify({ + kind: 'text', source: new URL(app.url, location.href).href, + text: pages.join('\n\n') + }); + } + } + return JSON.stringify({kind: 'page_text', text: document.body?.innerText || ''}); +})() +""" + +DOCUMENT_PAGE_READY = r""" +() => { + if (globalThis.PDFViewerApplication) return !!globalThis.PDFViewerApplication.pdfDocument; + return !!document.querySelector('embed[src],iframe[src],object[data],a[href*=".pdf"]') + || !!document.body?.innerText?.trim(); +} +""" + + + +def require_native_task_routes(cases, records): + """Reject substitutions and stale task adapters before report publication.""" + import hashlib + from types import SimpleNamespace + from .providers.headless import output_mode_for_case + + inventory = {case['id']: SimpleNamespace(id=case['id'], url=case['url'], track=case.get('track','web'), task_kind=case.get('task_kind','body'), source_benchmark=case.get('source_benchmark',{}), expected_resources=[{'kind':r.get('kind')} for r in case.get('expected_resources',[])]) for case in cases} + expressions = {'resource':RESOURCE_PAGE_EXPRESSION, 'media_metadata':MEDIA_METADATA_EXPRESSION, + 'document_page':DOCUMENT_PAGE_EXPRESSION} + failures = [] + for row in records: + from .methods import native_browser_engine + provider = native_browser_engine(row['provider']) + if provider not in {'moli','lightpanda','obscura','chromium','kitesurf'}: + failures.append(row['provider'] + '/' + row['case_id']) + continue + if row.get('error_type') == 'configuration': + failures.append(row['provider'] + '/' + row['case_id']) + continue + case = inventory[row['case_id']] + mode = output_mode_for_case(provider, 'headless_markdown', case) + meta = row.get('provider_metadata') or {} + # The runner may expire before a worker returns metadata. Its retained + # call and cohort receipts still bind the invocation and deadline. + runner_fields = {'call_log', 'configured_min_interval_seconds', 'client_throttle_wait_ms'} + if row.get('ok') is False and row.get('error_type') in {'timeout','network'} and meta.get('call_log') and not (set(meta) - runner_fields): + continue + valid = True + if mode == 'markdown': + if provider == 'moli': + valid = meta.get('route') == 'ordinary_markdown' and meta.get('output') == 'markdown' + elif provider == 'chromium': + valid = (meta.get('extractor') == 'node-html-markdown' + and (meta.get('extractor_version') == '2.0.0' + or (row.get('ok') is False and meta.get('extractor_version') is None))) + elif provider in {'lightpanda','obscura'}: + valid = meta.get('extractor') == provider+'_native_markdown' + elif provider == 'kitesurf': + valid = (meta.get('route') == 'browser_rendering_markdown' + and meta.get('extractor') == 'cloudflare_native_markdown' + and meta.get('browser') == 'kitesurf') + elif mode in {'document','file'}: + if provider == 'moli': + valid = (meta.get('route') == ('raw_document' if mode == 'document' else 'raw_file') + and meta.get('extractor') == 'moli_cdp_response' + ('+document_text' if mode == 'document' else '')) + else: + prefixes = {'lightpanda':'lightpanda_cdp_response', 'obscura':'obscura_original_response', + 'chromium':'browser_native_response_or_download', 'kitesurf':'kitesurf_cdp_response'} + valid = meta.get('extractor') == prefixes[provider] + ('+document_text' if mode == 'document' else '') + valid = valid and meta.get('route') != 'workers_ai_tomarkdown' + elif mode in expressions: + expression = resource_expression(case.track, resource_detail_links(case)) if mode == 'resource' else expressions[mode] + valid = meta.get('output_projection_sha256') == hashlib.sha256(expression.encode()).hexdigest() + if provider == 'kitesurf': + valid = (valid and meta.get('browser') == 'kitesurf' + and meta.get('endpoint_route') == '/browser-run/devtools/browser?browser=kitesurf') + if provider == 'moli' and mode == 'media_metadata' and not meta.get('output_projection_sha256'): + # Historical calls used this exact same shared expression; the + # native route and its source-bound cohort remain authoritative. + valid = meta.get('route') == 'media_metadata_runtime' + if case.track == 'document': + valid = valid and meta.get('document_extractor') in {None, 'pdfjs_runtime', 'rendered_page_text'} + valid = valid and '+document_text' not in str(meta.get('extractor', '')) + if not valid: + failures.append(row['provider'] + '/' + case.id) + if failures: + raise ValueError('Native task route contract violated: ' + ', '.join(failures)) diff --git a/pyproject.toml b/pyproject.toml index a1a00e300..5a2121744 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -105,6 +105,6 @@ include = ["harness*"] "harness.environments.webarena" = ["runtime/*.ini"] [tool.setuptools.data-files] -"share/browser-harness-eval" = ["evaluation/scoring-regressions.json"] +"share/browser-harness-eval" = ["evaluation/scoring-regressions.json", "configs/providers.headless.json"] "share/browser-harness-eval/webmainbench" = ["evaluation/webmainbench-full/scoring-regressions.json"] "share/browser-harness-eval/public-webarena" = ["evaluation/browser-use-public/adoption-evidence.json", "evaluation/browser-use-public/benchmark-catalog.json", "evaluation/browser-use-public/report-changes.json"] diff --git a/research/moli-111-three-cohort-evaluation-revision.json b/research/moli-111-three-cohort-evaluation-revision.json index 11fae2891..80cdd2767 100644 --- a/research/moli-111-three-cohort-evaluation-revision.json +++ b/research/moli-111-three-cohort-evaluation-revision.json @@ -331,38 +331,61 @@ "scope": "Scorer and regression bindings; current diagnostic rescoring uses separate receipts, not historical certification." }, { - "root_cause":"Normalize CommonMark escaped punctuation in list prefixes and reuse unreserved URL canonicalization for composite field ownership; preserve prose rejection and record boundaries.", - "provider_calls":0, - "predecessor_source_sha256":"5af22f03e56ecc28e41511c2ad23736508e40981f7914b54865741df43dcfffb", - "revised_sha256":"6d092487f414f55287fc2b63be09660e8e5b30e0ec1f1f4dec04df714aa01c13", - "scoring_regressions_sha256":"574867b68a701232dd1b3074841c6c5dacccd924435fd4fae550aa25bfd1ebd8", - "regression_ids":[ - "html-doctype-hidden-challenge-is-not-visible", "html-doctype-visible-challenge-still-blocks", - "html-inline-code-preserves-visible-anchor", "html-hidden-script-cannot-satisfy-anchor", - "html-inline-code-counts-for-body-coverage", "html-hidden-script-cannot-satisfy-body-coverage", - "html-visible-body-ignores-hidden-challenge-heading", "html-hidden-script-cannot-inflate-visible-length", + "root_cause": "Normalize CommonMark escaped punctuation in list prefixes and reuse unreserved URL canonicalization for composite field ownership; preserve prose rejection and record boundaries.", + "provider_calls": 0, + "predecessor_source_sha256": "5af22f03e56ecc28e41511c2ad23736508e40981f7914b54865741df43dcfffb", + "revised_sha256": "6d092487f414f55287fc2b63be09660e8e5b30e0ec1f1f4dec04df714aa01c13", + "scoring_regressions_sha256": "574867b68a701232dd1b3074841c6c5dacccd924435fd4fae550aa25bfd1ebd8", + "regression_ids": [ + "html-doctype-hidden-challenge-is-not-visible", + "html-doctype-visible-challenge-still-blocks", + "html-inline-code-preserves-visible-anchor", + "html-hidden-script-cannot-satisfy-anchor", + "html-inline-code-counts-for-body-coverage", + "html-hidden-script-cannot-satisfy-body-coverage", + "html-visible-body-ignores-hidden-challenge-heading", + "html-hidden-script-cannot-inflate-visible-length", "html-visible-challenge-heading-blocks-body", - "dynamic-undeclared-source-language-can-verify-identity", "dynamic-undeclared-source-language-rejects-declared-language", - "long-metadata-title-keeps-visible-html-anchor", "long-metadata-title-cannot-expose-script-anchor", - "html-inline-challenge-explanation-is-prose", "html-independent-challenge-heading-still-blocks", - "combined-problem-cells-across-sections-pass", "combined-problem-cells-cannot-omit-name", - "html-head-title-cannot-supply-body", "html-body-anchor-with-head-title-passes", - "card-single-line-distinct-typed-fields-pass", "card-unrelated-quantity-cannot-supply-typed-fields", - "metadata-title-body-anchor-missing-fails", "metadata-title-body-anchor-visible-passes", - "metadata-title-body-coverage-missing-fails", "metadata-title-body-coverage-visible-passes", - "metadata-title-composite-missing-fails", "metadata-title-composite-visible-passes", - "legacy-title-marker-missing-fails", "legacy-title-marker-visible-passes", - "legacy-title-pattern-missing-fails", "legacy-title-pattern-visible-passes", - "html-misplaced-title-missing-body-fails", "html-misplaced-title-visible-body-passes", - "declared-page-chrome-is-required-stable-text", "missing-declared-page-chrome-fails-full-page-contract", - "emphasized-numeric-rank-is-valid-entry", "emphasized-recommendation-is-not-list-entry", - "markdown-heading-opens-declared-list-scope", "outside-section-links-do-not-fill-declared-scope", - "inline-layout-breaks-preserve-table-fields", "inline-layout-breaks-do-not-repair-missing-table-fields", - "declared-redirect-link-base-resolves-relative-records", "declared-redirect-link-base-does-not-allow-other-hosts", - "escaped-list-punctuation-retains-entry", "escaped-recommendation-remains-prose", - "unreserved-path-encoding-retains-card-fields", "encoded-card-cannot-borrow-next-price" + "dynamic-undeclared-source-language-can-verify-identity", + "dynamic-undeclared-source-language-rejects-declared-language", + "long-metadata-title-keeps-visible-html-anchor", + "long-metadata-title-cannot-expose-script-anchor", + "html-inline-challenge-explanation-is-prose", + "html-independent-challenge-heading-still-blocks", + "combined-problem-cells-across-sections-pass", + "combined-problem-cells-cannot-omit-name", + "html-head-title-cannot-supply-body", + "html-body-anchor-with-head-title-passes", + "card-single-line-distinct-typed-fields-pass", + "card-unrelated-quantity-cannot-supply-typed-fields", + "metadata-title-body-anchor-missing-fails", + "metadata-title-body-anchor-visible-passes", + "metadata-title-body-coverage-missing-fails", + "metadata-title-body-coverage-visible-passes", + "metadata-title-composite-missing-fails", + "metadata-title-composite-visible-passes", + "legacy-title-marker-missing-fails", + "legacy-title-marker-visible-passes", + "legacy-title-pattern-missing-fails", + "legacy-title-pattern-visible-passes", + "html-misplaced-title-missing-body-fails", + "html-misplaced-title-visible-body-passes", + "declared-page-chrome-is-required-stable-text", + "missing-declared-page-chrome-fails-full-page-contract", + "emphasized-numeric-rank-is-valid-entry", + "emphasized-recommendation-is-not-list-entry", + "markdown-heading-opens-declared-list-scope", + "outside-section-links-do-not-fill-declared-scope", + "inline-layout-breaks-preserve-table-fields", + "inline-layout-breaks-do-not-repair-missing-table-fields", + "declared-redirect-link-base-resolves-relative-records", + "declared-redirect-link-base-does-not-allow-other-hosts", + "escaped-list-punctuation-retains-entry", + "escaped-recommendation-remains-prose", + "unreserved-path-encoding-retains-card-fields", + "encoded-card-cannot-borrow-next-price" ], - "scope":"Shared scoring implementation provenance only; measured score revisions retain separate per-run receipts and immutable call bytes." + "scope": "Shared scoring implementation provenance only; measured score revisions retain separate per-run receipts and immutable call bytes." }, { "root_cause": "Accept equivalent plus bullets, adjacent multiline Markdown cards, and declared headings in Markdown table headers without accepting prose or unscoped rows.", @@ -426,9 +449,249 @@ "markdown-table-header-cannot-bypass-declared-scope" ], "scope": "Shared parser and regression bindings only. Frozen measured reports retain their declared scoring revision; this record does not rescore or recertify historical calls." + }, + { + "root_cause": "Merge the shared scorer corrections and the independently retained delimiter performance correction without altering historical provider calls or stored verdicts.", + "provider_calls": 0, + "predecessor_source_sha256": "b97dc5f788fc6da194c6e3b8987f039468960a68328d2d061f3b941acca5750f", + "parallel_predecessor_source_sha256": "c0fce0df529aec9d024ea82dde7caf31a69f0297fb9c79e1ea5b36e25ead9372", + "revised_sha256": "d9d961022d66a29703b262fe2db987d0d17c6a915c896cf8825ab46221a24044", + "scoring_regressions_sha256": "b1f61cdfc5e2be265b944cfd6c767b9dfd1823446378e73a89eb08ed8980a2ad", + "regression_ids": [ + "adjacent-multiline-markdown-cards-keep-records", + "card-single-line-distinct-typed-fields-pass", + "card-unrelated-quantity-cannot-supply-typed-fields", + "combined-problem-cells-across-sections-pass", + "combined-problem-cells-cannot-omit-name", + "declared-page-chrome-is-required-stable-text", + "declared-redirect-link-base-does-not-allow-other-hosts", + "declared-redirect-link-base-resolves-relative-records", + "dynamic-undeclared-source-language-can-verify-identity", + "dynamic-undeclared-source-language-rejects-declared-language", + "emphasized-numeric-rank-is-valid-entry", + "emphasized-recommendation-is-not-list-entry", + "encoded-card-cannot-borrow-next-price", + "escaped-list-punctuation-retains-entry", + "escaped-recommendation-remains-prose", + "html-body-anchor-with-head-title-passes", + "html-doctype-hidden-challenge-is-not-visible", + "html-doctype-visible-challenge-still-blocks", + "html-head-title-cannot-supply-body", + "html-hidden-script-cannot-inflate-visible-length", + "html-hidden-script-cannot-satisfy-anchor", + "html-hidden-script-cannot-satisfy-body-coverage", + "html-independent-challenge-heading-still-blocks", + "html-inline-challenge-explanation-is-prose", + "html-inline-code-counts-for-body-coverage", + "html-inline-code-preserves-visible-anchor", + "html-misplaced-title-missing-body-fails", + "html-misplaced-title-visible-body-passes", + "html-visible-body-ignores-hidden-challenge-heading", + "html-visible-challenge-heading-blocks-body", + "inline-layout-breaks-do-not-repair-missing-table-fields", + "inline-layout-breaks-preserve-table-fields", + "legacy-title-marker-missing-fails", + "legacy-title-marker-visible-passes", + "legacy-title-pattern-missing-fails", + "legacy-title-pattern-visible-passes", + "long-metadata-title-cannot-expose-script-anchor", + "long-metadata-title-keeps-visible-html-anchor", + "markdown-heading-opens-declared-list-scope", + "markdown-plus-list-keeps-named-records", + "markdown-plus-prose-is-not-a-record-list", + "markdown-table-header-cannot-bypass-declared-scope", + "markdown-table-header-opens-declared-scope", + "metadata-title-body-anchor-missing-fails", + "metadata-title-body-anchor-visible-passes", + "metadata-title-body-coverage-missing-fails", + "metadata-title-body-coverage-visible-passes", + "metadata-title-composite-missing-fails", + "metadata-title-composite-visible-passes", + "missing-declared-page-chrome-fails-full-page-contract", + "multiline-card-followed-by-prose-does-not-borrow-record", + "outside-section-links-do-not-fill-declared-scope", + "unmatched-markdown-brackets-cannot-supply-missing-prose", + "unmatched-markdown-brackets-retains-required-prose", + "unreserved-path-encoding-retains-card-fields" + ] + }, + { + "provider_calls": 0, + "predecessor_source_sha256": "d9d961022d66a29703b262fe2db987d0d17c6a915c896cf8825ab46221a24044", + "revised_sha256": "30731d9556b150ec91c48074a4b12d37e14acc1569d78194d1aa6f6b7d6f9788", + "task_verdicts_unchanged": true, + "root_cause": "Completed calls lacking required final-URL or language observations are failed measurements, not missing benchmark references. Unresolved benchmark rules remain unavailable.", + "scoring_regressions_sha256": "d3a9cffb5a476dcbb73958ef1f2c48a1d0414ff7a46d1dd1dbf3f2750f2d9e1a", + "regression_ids": [ + "adjacent-multiline-markdown-cards-keep-records", + "card-single-line-distinct-typed-fields-pass", + "card-unrelated-quantity-cannot-supply-typed-fields", + "combined-problem-cells-across-sections-pass", + "combined-problem-cells-cannot-omit-name", + "declared-page-chrome-is-required-stable-text", + "declared-redirect-link-base-does-not-allow-other-hosts", + "declared-redirect-link-base-resolves-relative-records", + "dynamic-undeclared-source-language-can-verify-identity", + "dynamic-undeclared-source-language-rejects-declared-language", + "emphasized-numeric-rank-is-valid-entry", + "emphasized-recommendation-is-not-list-entry", + "encoded-card-cannot-borrow-next-price", + "escaped-list-punctuation-retains-entry", + "escaped-recommendation-remains-prose", + "html-body-anchor-with-head-title-passes", + "html-doctype-hidden-challenge-is-not-visible", + "html-doctype-visible-challenge-still-blocks", + "html-head-title-cannot-supply-body", + "html-hidden-script-cannot-inflate-visible-length", + "html-hidden-script-cannot-satisfy-anchor", + "html-hidden-script-cannot-satisfy-body-coverage", + "html-independent-challenge-heading-still-blocks", + "html-inline-challenge-explanation-is-prose", + "html-inline-code-counts-for-body-coverage", + "html-inline-code-preserves-visible-anchor", + "html-misplaced-title-missing-body-fails", + "html-misplaced-title-visible-body-passes", + "html-visible-body-ignores-hidden-challenge-heading", + "html-visible-challenge-heading-blocks-body", + "inline-layout-breaks-do-not-repair-missing-table-fields", + "inline-layout-breaks-preserve-table-fields", + "legacy-title-marker-missing-fails", + "legacy-title-marker-visible-passes", + "legacy-title-pattern-missing-fails", + "legacy-title-pattern-visible-passes", + "long-metadata-title-cannot-expose-script-anchor", + "long-metadata-title-keeps-visible-html-anchor", + "markdown-heading-opens-declared-list-scope", + "markdown-plus-list-keeps-named-records", + "markdown-plus-prose-is-not-a-record-list", + "markdown-table-header-cannot-bypass-declared-scope", + "markdown-table-header-opens-declared-scope", + "metadata-title-body-anchor-missing-fails", + "metadata-title-body-anchor-visible-passes", + "metadata-title-body-coverage-missing-fails", + "metadata-title-body-coverage-visible-passes", + "metadata-title-composite-missing-fails", + "metadata-title-composite-visible-passes", + "missing-declared-page-chrome-fails-full-page-contract", + "multiline-card-followed-by-prose-does-not-borrow-record", + "outside-section-links-do-not-fill-declared-scope", + "required-final-url-missing-is-a-measured-failure", + "required-final-url-present-keeps-valid-body", + "unmatched-markdown-brackets-cannot-supply-missing-prose", + "unmatched-markdown-brackets-retains-required-prose", + "unreserved-path-encoding-retains-card-fields" + ] + }, + { + "provider_calls": 0, + "predecessor_source_sha256": "30731d9556b150ec91c48074a4b12d37e14acc1569d78194d1aa6f6b7d6f9788", + "revised_sha256": "181578a2be34f26ddc22dfd1590a84b3367a3b161f01656f0215914e54412dd4", + "task_verdicts_unchanged": false, + "root_cause": "Web Fetch body acceptance incorrectly required returned final-URL and HTML-language metadata. Target content and blocked-page checks are authoritative; optional metadata remains diagnostic. All methods are rescored from immutable saved output, without changes to measured tasks or references.", + "scoring_regressions_sha256": "3c82b56fc8bd13ab3d3b25168b6dafd5b24c29c3409fb41c2f5521e6aabd9346", + "regression_ids": [ + "adjacent-multiline-markdown-cards-keep-records", + "body-without-final-url-passes", + "card-single-line-distinct-typed-fields-pass", + "card-unrelated-quantity-cannot-supply-typed-fields", + "combined-problem-cells-across-sections-pass", + "combined-problem-cells-cannot-omit-name", + "content-policy-dynamic-final-page-mismatch-cannot-be-counted-as-a-pass", + "content-policy-dynamic-search-query-mismatch-is-a-different-page", + "content-policy-dynamic-source-document-language-mismatch-is-not-verified", + "content-policy-dynamic-undeclared-source-language-rejects-declared-language", + "content-policy-required-final-url-missing-is-a-measured-failure", + "content-policy-required-final-url-present-keeps-valid-body", + "declared-page-chrome-is-required-stable-text", + "declared-redirect-link-base-does-not-allow-other-hosts", + "declared-redirect-link-base-resolves-relative-records", + "dynamic-undeclared-source-language-can-verify-identity", + "dynamic-undeclared-source-language-rejects-declared-language", + "emphasized-numeric-rank-is-valid-entry", + "emphasized-recommendation-is-not-list-entry", + "encoded-card-cannot-borrow-next-price", + "escaped-list-punctuation-retains-entry", + "escaped-recommendation-remains-prose", + "html-body-anchor-with-head-title-passes", + "html-doctype-hidden-challenge-is-not-visible", + "html-doctype-visible-challenge-still-blocks", + "html-head-title-cannot-supply-body", + "html-hidden-script-cannot-inflate-visible-length", + "html-hidden-script-cannot-satisfy-anchor", + "html-hidden-script-cannot-satisfy-body-coverage", + "html-independent-challenge-heading-still-blocks", + "html-inline-challenge-explanation-is-prose", + "html-inline-code-counts-for-body-coverage", + "html-inline-code-preserves-visible-anchor", + "html-misplaced-title-missing-body-fails", + "html-misplaced-title-visible-body-passes", + "html-visible-body-ignores-hidden-challenge-heading", + "html-visible-challenge-heading-blocks-body", + "inline-layout-breaks-do-not-repair-missing-table-fields", + "inline-layout-breaks-preserve-table-fields", + "legacy-title-marker-missing-fails", + "legacy-title-marker-visible-passes", + "legacy-title-pattern-missing-fails", + "legacy-title-pattern-visible-passes", + "long-metadata-title-cannot-expose-script-anchor", + "long-metadata-title-keeps-visible-html-anchor", + "markdown-heading-opens-declared-list-scope", + "markdown-plus-list-keeps-named-records", + "markdown-plus-prose-is-not-a-record-list", + "markdown-table-header-cannot-bypass-declared-scope", + "markdown-table-header-opens-declared-scope", + "metadata-title-body-anchor-missing-fails", + "metadata-title-body-anchor-visible-passes", + "metadata-title-body-coverage-missing-fails", + "metadata-title-body-coverage-visible-passes", + "metadata-title-composite-missing-fails", + "metadata-title-composite-visible-passes", + "missing-declared-page-chrome-fails-full-page-contract", + "multiline-card-followed-by-prose-does-not-borrow-record", + "outside-section-links-do-not-fill-declared-scope", + "required-final-url-missing-is-a-measured-failure", + "required-final-url-present-keeps-valid-body", + "unmatched-markdown-brackets-cannot-supply-missing-prose", + "unmatched-markdown-brackets-retains-required-prose", + "unreserved-path-encoding-retains-card-fields", + "verification-page-without-final-url-fails", + "wrong-body-content-policy-dynamic-final-page-mismatch-cannot-be-counted-as-a-pass", + "wrong-body-content-policy-dynamic-search-query-mismatch-is-a-different-page", + "wrong-body-content-policy-dynamic-source-document-language-mismatch-is-not-verified", + "wrong-body-content-policy-dynamic-undeclared-source-language-rejects-declared-language" + ], + "superseded_regression_ids": { + "dynamic-final-page-mismatch-cannot-be-counted-as-a-pass": "content-policy-dynamic-final-page-mismatch-cannot-be-counted-as-a-pass", + "dynamic-search-query-mismatch-is-a-different-page": "content-policy-dynamic-search-query-mismatch-is-a-different-page", + "dynamic-source-document-language-mismatch-is-not-verified": "content-policy-dynamic-source-document-language-mismatch-is-not-verified", + "dynamic-undeclared-source-language-rejects-declared-language": "content-policy-dynamic-undeclared-source-language-rejects-declared-language", + "required-final-url-missing-is-a-measured-failure": "content-policy-required-final-url-missing-is-a-measured-failure", + "required-final-url-present-keeps-valid-body": "content-policy-required-final-url-present-keeps-valid-body" + } + } + ], + "dynamic_dataset_followups": [ + { + "case_id": "dynamic-065-franceinfo-home", + "field_path": [ + "acceptance_contract", + "link_base_url" + ], + "value": "https://www.franceinfo.fr/", + "provider_calls": 0, + "predecessor_sha256": "57a138d2c79c9f107124cad3997a7350aa33e40cee3ac1137c5d627f1db098ce", + "revised_sha256": "b32c7ad678ddef0377868f88d8fb82b0f84c09b708f67e5688317db96eab9571", + "source_authority": { + "path": "evaluation/sources/dynamic-065-franceinfo-home/mixed-session-stable-text.json", + "sha256": "254111452c2cec1b5a5a9481c5204b01aa029cb06c97563092db9d12208ba58b", + "provider_output_used": false + }, + "regression_ids": [ + "declared-redirect-link-base-resolves-relative-records", + "declared-redirect-link-base-does-not-allow-other-hosts" + ] } ], - "dynamic_dataset_followups": [{"case_id":"dynamic-065-franceinfo-home","field_path":["acceptance_contract","link_base_url"],"value":"https://www.franceinfo.fr/","provider_calls":0,"predecessor_sha256":"57a138d2c79c9f107124cad3997a7350aa33e40cee3ac1137c5d627f1db098ce","revised_sha256":"b32c7ad678ddef0377868f88d8fb82b0f84c09b708f67e5688317db96eab9571","source_authority":{"path":"evaluation/sources/dynamic-065-franceinfo-home/mixed-session-stable-text.json","sha256":"254111452c2cec1b5a5a9481c5204b01aa029cb06c97563092db9d12208ba58b","provider_output_used":false},"regression_ids":["declared-redirect-link-base-resolves-relative-records","declared-redirect-link-base-does-not-allow-other-hosts"]}], "shared_scorer": { "main_baseline_sha256": "bd0b0118c84b0ba03d9241aba74be84330a9a9dd20f98f62a3ac8be396c79725", "revised_sha256": "130d8de3c553e7f8d7ecc1ba3832aa6c03ad55c9f9b53ad4661a3faaddae1a39", @@ -446,5 +709,76 @@ "dynamic-undeclared-source-language-can-verify-identity", "dynamic-undeclared-source-language-rejects-declared-language" ] - } + }, + "shared_scorer_parallel_followups": [ + { + "root_cause": "Both visible Markdown links and stable-task link unwrapping repeatedly scanned suffixes for unmatched delimiters. A shared delimiter index preserves balanced, escaped, nested and span-bounded parsing while avoiding quadratic work.", + "change_kind": "performance_only", + "provider_calls": 0, + "provider_outputs_used": false, + "stored_scores_changed": false, + "predecessor_source_sha256": "6d092487f414f55287fc2b63be09660e8e5b30e0ec1f1f4dec04df714aa01c13", + "revised_sha256": "c0fce0df529aec9d024ea82dde7caf31a69f0297fb9c79e1ea5b36e25ead9372", + "scoring_regressions_sha256": "f8ab69c9eb61ee5048d9af3385e19816e3327a98f372f0b5f8163fe37a9d8348", + "synthetic_input_count": 10000, + "synthetic_metric_changes": 0, + "synthetic_task_decision_changes": 0, + "compatibility_evidence": { + "path": "evidence/research/webfetch-430-refresh-20261008/markdown-parser-equivalence-receipt.json", + "sha256": "b9adadf5c5e549219af9bf84c0d08c910ed19db965181fab962261f859ab7e19", + "baseline_commit": "a205217775cb1c4dba476013a8bbb1f36e7f9b59" + }, + "regression_ids": [ + "html-doctype-hidden-challenge-is-not-visible", + "html-doctype-visible-challenge-still-blocks", + "html-inline-code-preserves-visible-anchor", + "html-hidden-script-cannot-satisfy-anchor", + "html-inline-code-counts-for-body-coverage", + "html-hidden-script-cannot-satisfy-body-coverage", + "html-visible-body-ignores-hidden-challenge-heading", + "html-hidden-script-cannot-inflate-visible-length", + "html-visible-challenge-heading-blocks-body", + "dynamic-undeclared-source-language-can-verify-identity", + "dynamic-undeclared-source-language-rejects-declared-language", + "long-metadata-title-keeps-visible-html-anchor", + "long-metadata-title-cannot-expose-script-anchor", + "html-inline-challenge-explanation-is-prose", + "html-independent-challenge-heading-still-blocks", + "combined-problem-cells-across-sections-pass", + "combined-problem-cells-cannot-omit-name", + "html-head-title-cannot-supply-body", + "html-body-anchor-with-head-title-passes", + "card-single-line-distinct-typed-fields-pass", + "card-unrelated-quantity-cannot-supply-typed-fields", + "metadata-title-body-anchor-missing-fails", + "metadata-title-body-anchor-visible-passes", + "metadata-title-body-coverage-missing-fails", + "metadata-title-body-coverage-visible-passes", + "metadata-title-composite-missing-fails", + "metadata-title-composite-visible-passes", + "legacy-title-marker-missing-fails", + "legacy-title-marker-visible-passes", + "legacy-title-pattern-missing-fails", + "legacy-title-pattern-visible-passes", + "html-misplaced-title-missing-body-fails", + "html-misplaced-title-visible-body-passes", + "declared-page-chrome-is-required-stable-text", + "missing-declared-page-chrome-fails-full-page-contract", + "emphasized-numeric-rank-is-valid-entry", + "emphasized-recommendation-is-not-list-entry", + "markdown-heading-opens-declared-list-scope", + "outside-section-links-do-not-fill-declared-scope", + "inline-layout-breaks-preserve-table-fields", + "inline-layout-breaks-do-not-repair-missing-table-fields", + "declared-redirect-link-base-resolves-relative-records", + "declared-redirect-link-base-does-not-allow-other-hosts", + "escaped-list-punctuation-retains-entry", + "escaped-recommendation-remains-prose", + "unreserved-path-encoding-retains-card-fields", + "encoded-card-cannot-borrow-next-price", + "unmatched-markdown-brackets-retains-required-prose", + "unmatched-markdown-brackets-cannot-supply-missing-prose" + ] + } + ] } diff --git a/research/provider-pricing.json b/research/provider-pricing.json index 2b4c87624..a4458ea46 100644 --- a/research/provider-pricing.json +++ b/research/provider-pricing.json @@ -121,8 +121,8 @@ } ], "other_methods":[ - {"ids":["kitesurf"], "label":"Kitesurf · Cloudflare Browser Run /markdown", "status":"Beta 期免费；账户级限制", "note":"2026-09-07 核验的 Kitesurf 专项文档明确写明 Beta 期免费且受每账户限制。通用 Browser Run 另有浏览器时长计价；在专项文档改口前，不把通用超额价格当成 Kitesurf Beta 的已证实单价。", "sources":[{"label":"官方 Kitesurf 文档", "url":"https://developers.cloudflare.com/browser-run/kitesurf/"},{"label":"Browser Run 通用定价", "url":"https://developers.cloudflare.com/browser-run/pricing/"}]}, - {"ids":["lightpanda","obscura","moli-1.0.6","moli","moli-1.1.3","moli-1.1.7","chromium"], "label":"Lightpanda / Obscura / Moli 1.0.6 / 1.1.5 / 1.1.3 / 1.1.7 / Chromium", "status":"本轮本地浏览器；无托管调用单价", "note":"核验于 2026-09-17。算力、网络、存储、代理和维护成本自担；没有测量这些成本，不把本地调用等同于零成本。官方托管产品或服务价格不属于本轮执行模式。", "sources":[{"label":"Lightpanda", "url":"https://github.com/lightpanda-io/browser"},{"label":"Obscura", "url":"https://github.com/h4ckf0r0day/obscura"},{"label":"Moli", "url":"https://github.com/lexmount/moli/releases/tag/v1.1.7"},{"label":"Chrome Headless", "url":"https://developer.chrome.com/docs/automation-and-testing/headless"}]}, + {"ids":["kitesurf"], "label":"Kitesurf · Cloudflare Browser Run", "status":"Beta 期免费；账户级限制", "note":"2026-10-09 核验：Kitesurf 专项文档仍标明 Beta 期免费，受账户用量限制。通用 Browser Run 另有浏览器时长计价；在专项文档改口前，不把通用超额价格当成 Kitesurf Beta 的已证实单价。", "sources":[{"label":"官方 Kitesurf 文档", "url":"https://developers.cloudflare.com/browser-run/kitesurf/"},{"label":"Browser Run 通用定价", "url":"https://developers.cloudflare.com/browser-run/pricing/"}]}, + {"ids":["lightpanda","obscura","moli-1.0.6","moli","moli-1.1.3","moli-1.1.7","chromium"], "label":"Lightpanda / Obscura / Moli / Chromium", "status":"本轮本地浏览器；无托管调用单价", "note":"核验于 2026-09-17。算力、网络、存储、代理和维护成本自担；没有测量这些成本，不把本地调用等同于零成本。官方托管产品或服务价格不属于本轮执行模式。", "sources":[{"label":"Lightpanda", "url":"https://github.com/lightpanda-io/browser"},{"label":"Obscura", "url":"https://github.com/h4ckf0r0day/obscura"},{"label":"Moli", "url":"https://github.com/lexmount/moli/releases/tag/v1.1.7"},{"label":"Chrome Headless", "url":"https://developer.chrome.com/docs/automation-and-testing/headless"}]}, {"ids":["moli-1.1.11"], "label":"Moli 1.1.11", "status":"本轮本地浏览器；无托管调用单价", "note":"核验于 2026-09-28。算力、网络、存储、代理和维护成本自担；没有测量这些成本，不把本地调用等同于零成本。官方托管产品或服务价格不属于本轮执行模式。", "sources":[{"label":"Moli 1.1.11", "url":"https://github.com/lexmount/moli/releases/tag/v1.1.11"}]}, {"ids":["lexmount-cn","lexmount-global"], "label":"Lexmount CN / Global", "status":"未查到公开价格", "note":"官方 WebFetch 文档标记内测；两个区域分别实测，不能据此推断价格相同。价格未公布不代表免费。", "sources":[{"label":"官方 WebFetch 文档", "url":"https://browser.lexmount.com/en/docs/webfetch"}]}, {"ids":["moli-webfetch"], "label":"Moli WebFetch", "status":"本轮本地 CLI；无托管 API 单价", "note":"Moli 可独立运行；本地算力、网络和代理自担。Lexmount 托管浏览器是另一个产品。", "sources":[{"label":"官方项目说明", "url":"https://github.com/lexmount/moli"}]}, diff --git a/research/report-changes.json b/research/report-changes.json index 0ed73646c..b1df9c92c 100644 --- a/research/report-changes.json +++ b/research/report-changes.json @@ -1,5 +1,5 @@ { - "schema_version": 2, + "schema_version": 3, "history_scope": "本页从有完整审计依据的修订开始记录。更早的站点版本仍保留在 Artifact Hub，但没有逐项变更说明，故不补编历史。", "history_scope_i18n": { "en": "This history begins with revisions supported by a complete audit trail. Earlier Artifact Hub versions remain available, but no itemized history is reconstructed where the supporting record is incomplete." @@ -1275,8 +1275,166 @@ } ], "releases": [ + { + "id": "2026-10-10-native-content-acceptance-correction", + "publication_state": "published", + "date": "2026-10-10", + "comparison_kinds": [ + "headless_markdown" + ], + "scoring_revision_input_sha256": [ + "47a1494ee2c6d55aeb33ce726f68fac5829ab16d6c6a077103834a06131ee4bf" + ], + "title": "统一原生内容接口与正文验收", + "summary": "五种浏览器仍各有 430 个结果：Lightpanda 238、Moli 227、Chromium 191、Obscura 189、Kitesurf 171。文档正文不再借助外部解析器，最终网址缺失不单独扣分。", + "changes": [ + [ + "正文接口", + "Moli 风控任务含 JSON DOM 转换", + "100 个风控任务改用原生 Markdown", + "保留旧调用，不用其他接口补正文。" + ], + [ + "文档正文", + "此前部分任务含外部 PDF/DOCX 解析", + "五种浏览器各补跑完整 10 题文档专项", + "直接文档使用原生输出；嵌入文档只读取浏览器内已加载文字，无法取得内容时保留失败。" + ], + [ + "正文验收", + "最终网址或 HTML 语言标签可能影响通过", + "仅作为诊断信息，按目标正文统一重算 2,150 个结果", + "登录、验证、错误页、空正文和缺少必需内容仍失败；原题与答案不变。" + ], + [ + "类型与方法表", + "风控类型名称不清晰，接口处理说明不完整", + "风控网站正文紧接热门网站正文；公开 7×5 接口表", + "获取接口、原生脚本与 Chromium HTML 转换均明示，实际路线不合规时拒绝发布。" + ] + ], + "i18n": { + "en": { + "title": "Native content interfaces and body acceptance", + "summary": "Each browser retains 430 results: Lightpanda 238, Moli 227, Chromium 191, Obscura 189 and Kitesurf 171. External document parsers are excluded; missing final URLs do not independently fail body tasks.", + "changes": [ + [ + "Body interfaces", + "Moli risk tasks included JSON DOM conversion", + "Rerun all 100 risk tasks with native Markdown", + "Preserve prior calls. Do not repair missing body text through another interface." + ], + [ + "Document text", + "Some tasks used external PDF/DOCX parsing", + "Rerun all ten document tasks for each browser", + "Use native output or text already loaded in a browser viewer; unsupported content remains a failure." + ], + [ + "Body acceptance", + "URL and HTML-language metadata could affect acceptance", + "Rescore all 2,150 results using target-content requirements", + "Login, challenge, error and empty pages still fail. Tasks and answers remain unchanged." + ], + [ + "Categories and interfaces", + "Risk labels and processing details were unclear", + "Place risk-control content after popular sites and show the 7×5 interface matrix", + "Disclose native scripts and Chromium HTML conversion; reject noncompliant measured routes." + ] + ] + } + } + }, + +{ + "id": "2026-10-10-five-browsers-seven-families-430", + "publication_state": "published", + "scoring_revision_input_sha256": [ + "7a79357747f8a0a9bdf9d216667edfeaf3bedf0ff25a3685ad29f17abfa1d712", + "884edf8ca3f9111aea956f9635688f9c7af6492ac32a1966f4275c4d9f19dd21" + ], + "date": "2026-10-10", + "comparison_kinds": [ + "headless_markdown", + "headless_html" + ], + "title": "发布五种浏览器、七类任务的 430 题报告", + "summary": "每种浏览器完成 430 题，共 2,150 个结果。Lightpanda 通过 243 题，Moli 233 题，Chromium 199 题，Obscura 192 题，Kitesurf 171 题。新增风控网页正文专项；动态内容与热门站点各覆盖 100 题。", + "changes": [ + [ + "评测范围", + "此前未覆盖当前七类完整题集", + "普通网页、动态内容、热门站点、风控网页正文各 100 题，文档、图片、视频各 10 题", + "总分母为每种浏览器 430 题；旧报告保留。" + ], + [ + "文档和资源接口", + "部分调用将文档当网页、借用外部转换或未明确选择参评引擎", + "文档通过参评浏览器读取原始字节并共用正文解析；资源使用已加载页面与明确指定的引擎", + "替换全部受影响的调用，保留此前证据；题目、固定答案及评分规则不变。" + ], + [ + "耗时与资源", + "任务通过率无法说明获取耗时和资源开销", + "分别列出全部调用、通过任务的均值、P50、P95；保留 20 题四种本地浏览器的 80 次资源测量", + "提取与解析计入时限；托管 Kitesurf 的后端资源未公开。" + ], + [ + "原生接口与共同处理", + "正文格式和资源接口未完整区分，部分 CDP 调用混用了托管 API 的等待参数", + "展示七类任务与五种浏览器的 35 个接口组合；Kitesurf 资源使用官方 CDP 与共同 DOM 范围", + "Moli 全部 100 个热门站点和 Kitesurf 全部 19 个受影响任务重新执行，旧调用保留。后一次纠正使 Kitesurf 视频通过数从 5/10 改为 6/10，整体从 168/430 改为 169/430；分母不变，耗时采用修正后的完整调用。" + ], + [ + "共享评分器", + "Markdown 加号列表、相邻卡片和表头可能漏计已返回的内容", + "采用最新共享评分器重算全部保留输出，并用成对测试防止漏判和误判", + "四个动态任务结果由失败变为通过：Kitesurf 增加两例，Lightpanda 与 Obscura 各增加一例。未新增浏览器调用，题目、答案、输出和耗时均未改变。" + ] + ], + "i18n": { + "en": { + "title": "Five browsers across 430 tasks in seven families", + "summary": "All five browsers completed 430 tasks, yielding 2,150 results. Lightpanda passed 243, Moli 233, Chromium 199, Obscura 192, and Kitesurf 171. Risk-controlled webpage text is a new family; dynamic and popular-site tasks each cover 100 cases.", + "changes": [ + [ + "Scope", + "The previous cohort did not cover the current seven-family matrix", + "100 cases each for ordinary, dynamic, popular and risk-controlled webpage text; 10 each for documents, images and video", + "430 cases per browser; the earlier report is preserved." + ], + [ + "Document and resource routes", + "Some calls treated documents as pages, used external conversion or omitted explicit engine selection", + "Browser-native original document bytes with shared text extraction; resources from loaded pages with explicit engine binding", + "All affected calls are replaced with prior evidence retained; tasks, fixed answers and scoring remain unchanged." + ], + [ + "Latency and resources", + "Task acceptance alone does not describe retrieval cost", + "All-call and successful-call mean, P50 and P95 latency; 80 retained resource calls across 20 cases and four local browsers", + "Extraction and parsing share the deadline; hosted Kitesurf backend resources are unavailable." + ], + [ + "Native interfaces and shared processing", + "Output modes and resource scope were conflated; some CDP calls received a hosted API completion value", + "Expose all 35 browser-family interface combinations; Kitesurf resources use official CDP with the same DOM scope", + "Rerun all 100 Moli popular-site tasks and all 19 affected Kitesurf tasks, preserving prior calls. The latter correction changes Kitesurf video passes from 5/10 to 6/10 and overall passes from 168/430 to 169/430. Denominators stay fixed; latency uses the corrected complete calls." + ], + [ + "Shared scoring", + "Markdown plus lists, adjacent cards and table headers could omit returned content", + "Rescore all retained outputs with the current shared scorer and paired controls against false positives and false negatives", + "Four dynamic results change from fail to pass: two for Kitesurf and one each for Lightpanda and Obscura. No browser calls are added; tasks, answers, outputs and latency remain unchanged." + ] + ] + } + } +}, { "id": "2026-10-09-dynamic-1000-native-presentation", + "publication_state": "published", "date": "2026-10-09", "comparison_kinds": [ "uniform_native_local_measurement" @@ -1332,6 +1490,7 @@ }, { "id": "webmainbench-545-pr919-721bd2e-authority-20261008", + "publication_state": "published", "date": "2026-10-08", "comparison_kinds": [ "webmainbench_frozen_html_to_markdown" @@ -1387,6 +1546,7 @@ }, { "id": "2026-10-07-risk-control-historical-presentation", + "publication_state": "published", "comparison_kinds": [ "risk_control_100_moli" ], @@ -1442,6 +1602,7 @@ }, { "id": "2026-09-29-risk-control-review-corrections", + "publication_state": "published", "scoring_revision_input_sha256": "4dbcdbcc9232697b0420869c2dc46939e033999ec60ed0edcc4b9278a98209fb", "comparison_kinds": [ "risk_control_100_moli" @@ -1498,6 +1659,7 @@ }, { "id": "webmainbench-545-pr919-e944d066-lexmain-f7cb97d-20260929", + "publication_state": "published", "date": "2026-09-29", "comparison_kinds": ["webmainbench_frozen_html_to_markdown"], "title": "补齐渐显正文并按最新 lexmain 重算七方法结果", @@ -1519,6 +1681,7 @@ }, { "id": "webmainbench-545-lexmain-reference-and-final-pr919-20260928", + "publication_state": "published", "date": "2026-09-28", "comparison_kinds": ["webmainbench_frozen_html_to_markdown"], "title": "修正 WebMainBench 参考、专项计分范围与 TEDS", @@ -1542,6 +1705,7 @@ }, { "id": "webmainbench-545-moli-pr919-final-20260928", + "publication_state": "published", "date": "2026-09-28", "comparison_kinds": [ "webmainbench_frozen_html_to_markdown" @@ -1597,6 +1761,7 @@ }, { "id": "2026-09-24-dynamic-100-moli-119-local-audit", + "publication_state": "published", "comparison_kinds": ["single_provider_dynamic_native_fetch"], "date": "2026-09-24", "title": "完成100例动态网站 Moli 1.1.9 本地审核版", @@ -1618,6 +1783,7 @@ }, { "id": "webmainbench-545-table-controls-and-moli-provenance-20260928", + "publication_state": "published", "date": "2026-09-28", "comparison_kinds": [ "webmainbench_frozen_html_to_markdown" @@ -1661,6 +1827,7 @@ }, { "id": "webmainbench-545-seven-method-moli-lineage-20260925", + "publication_state": "published", "date": "2026-09-25", "comparison_kinds": [ "webmainbench_frozen_html_to_markdown" @@ -1704,6 +1871,7 @@ }, { "id": "webmainbench-545-moli-110-content-audit-20260924", + "publication_state": "published", "date": "2026-09-24", "comparison_kinds": [ "webmainbench_frozen_html_to_markdown" @@ -1759,6 +1927,7 @@ }, { "id": "webmainbench-545-six-method-origin-correction-20260924", + "publication_state": "published", "date": "2026-09-24", "comparison_kinds": [ "webmainbench_frozen_html_to_markdown" @@ -1802,6 +1971,7 @@ }, { "id": "2026-09-24-webmainbench-target-body-population", + "publication_state": "published", "comparison_kinds": ["webmainbench_frozen_html_to_markdown"], "date": "2026-09-24", "title": "区分请求成功与目标正文", @@ -1811,6 +1981,7 @@ }, { "id": "2026-09-24-webmainbench-six-versions-certified", + "publication_state": "published", "comparison_kinds": ["webmainbench_frozen_html_to_markdown"], "date": "2026-09-24", "title": "将修复版 Moli 纳入完整六方法比较", @@ -1820,6 +1991,7 @@ }, { "id": "2026-09-24-webmainbench-moli-repair-545-followup", + "publication_state": "published", "comparison_kinds": ["webmainbench_frozen_html_to_markdown"], "date": "2026-09-24", "title": "补充 Moli 修复后的 545 页复测", @@ -1829,6 +2001,7 @@ }, { "id": "2026-09-23-webmainbench-error-attribution-and-case-comparison", + "publication_state": "published", "comparison_kinds": ["webmainbench_frozen_html_to_markdown"], "date": "2026-09-23", "title": "按方法定位机器问题并展示同页完整输出", @@ -1838,6 +2011,7 @@ }, { "id": "2026-09-23-webmainbench-complete-545-five-methods", + "publication_state": "published", "comparison_kinds": ["webmainbench_frozen_html_to_markdown"], "date": "2026-09-23", "title": "发布 WebMainBench 完整 545 题五方法对照", @@ -1847,6 +2021,7 @@ }, { "id": "2026-09-20-wcxb-measured-mode-labels", + "publication_state": "published", "comparison_kinds": ["wcxb_frozen_html_to_markdown"], "date": "2026-09-20", "title": "明确可选清理模式与实测版本", @@ -1858,6 +2033,7 @@ }, { "id": "2026-09-18-wcxb-adjacent-code-scope-correction", + "publication_state": "published", "comparison_kinds": ["wcxb_frozen_html_to_markdown"], "date": "2026-09-18", "title": "纠正相邻代码修复的含义与评分解释", @@ -1869,6 +2045,7 @@ }, { "id": "2026-09-18-wcxb-moli-118-current-pr-comparison", + "publication_state": "published", "comparison_kinds": ["wcxb_frozen_html_to_markdown"], "date": "2026-09-18", "title": "更新 Moli 稳定版与当前修复版对照", @@ -1880,6 +2057,7 @@ }, { "id": "2026-09-18-wcxb-moli-repair-figure-provenance", + "publication_state": "published", "comparison_kinds": ["wcxb_frozen_html_to_markdown"], "date": "2026-09-18", "title": "标清 Moli 修复图的版本与证据范围", @@ -1893,6 +2071,7 @@ }, { "id": "2026-09-18-wcxb-moli-repair-visual-guide", + "publication_state": "published", "comparison_kinds": ["wcxb_frozen_html_to_markdown"], "date": "2026-09-18", "title": "用图和源码样本说明 Moli 的内容修复", @@ -1906,6 +2085,7 @@ }, { "id": "2026-09-18-wcxb-moli-repair-explanation", + "publication_state": "published", "comparison_kinds": ["wcxb_frozen_html_to_markdown"], "date": "2026-09-18", "title": "补充 Moli 修复的输入与输出对照", @@ -1919,6 +2099,7 @@ }, { "id": "2026-09-21-moli-seven-linux-release-resources", + "publication_state": "published", "date": "2026-09-21", "date_timezone": "Asia/Shanghai", "comparison_kinds": [ @@ -1970,6 +2151,7 @@ }, { "id": "2026-09-21-moli-seven-method-release-resource-calibration", + "publication_state": "published", "date": "2026-09-21", "date_timezone": "Asia/Shanghai", "comparison_kinds": [ @@ -2033,6 +2215,7 @@ }, { "id": "2026-09-21-moli-six-configuration-report", + "publication_state": "published", "date": "2026-09-21", "date_timezone": "Asia/Shanghai", "comparison_kinds": [ @@ -2084,6 +2267,7 @@ }, { "id": "2026-09-21-moli-119-fixed-layout-comparison", + "publication_state": "published", "date_timezone": "Asia/Shanghai", "comparison_kinds": [ "moli_historical_failure_retest", @@ -2147,6 +2331,7 @@ }, { "id": "2026-09-20-moli-version-layout-resource-summary", + "publication_state": "published", "date_timezone": "Asia/Shanghai", "publication": { "service": "Artifact Site", @@ -2210,6 +2395,7 @@ }, { "id": "2026-09-18-wcxb-frozen-html-reader-clarification", + "publication_state": "published", "comparison_kinds": ["wcxb_frozen_html_to_markdown"], "date": "2026-09-18", "title": "澄清冻结 HTML 阅读视图", @@ -2220,6 +2406,7 @@ }, { "id": "2026-09-18-wcxb-moli-complete-crossover-latency-correction", + "publication_state": "published", "comparison_kinds": [ "wcxb_frozen_html_to_markdown" ], @@ -2249,6 +2436,7 @@ }, { "id": "2026-09-18-wcxb-standalone-interpretation", + "publication_state": "published", "comparison_kinds": [ "wcxb_frozen_html_to_markdown" ], @@ -2278,6 +2466,7 @@ }, { "id": "2026-09-18-wcxb-obscura-and-paired-latency", + "publication_state": "published", "comparison_kinds": [ "wcxb_frozen_html_to_markdown" ], @@ -2307,6 +2496,7 @@ }, { "id": "2026-09-17-wcxb-expanded-representative-review", + "publication_state": "published", "comparison_kinds": [ "wcxb_frozen_html_to_markdown" ], @@ -2336,6 +2526,7 @@ }, { "id": "2026-09-17-wcxb-representative-audit-and-dataset-contract", + "publication_state": "published", "comparison_kinds": [ "wcxb_frozen_html_to_markdown" ], @@ -2365,6 +2556,7 @@ }, { "id": "2026-09-17-wcxb-qualified-test-results-and-preview", + "publication_state": "published", "comparison_kinds": [ "wcxb_frozen_html_to_markdown" ], @@ -2394,6 +2586,7 @@ }, { "id": "2026-09-18-wcxb-moli-repaired-separate-method", + "publication_state": "published", "comparison_kinds": [ "wcxb_frozen_html_to_markdown" ], @@ -2429,6 +2622,7 @@ }, { "id": "2026-09-14-public-benchmark-research", + "publication_state": "published", "comparison_kinds": ["public_benchmark_research"], "date": "2026-09-14", "title": "完成公开评测集调研与选型", @@ -2450,6 +2644,7 @@ }, { "id": "2026-09-17-public-webmainbench-moli-balanced-latency-correction", + "publication_state": "published", "comparison_kinds": ["webmainbench_frozen_html_to_markdown"], "date": "2026-09-17", "title": "纠正 Moli 两版的正式耗时", @@ -2473,6 +2668,7 @@ }, { "id": "2026-09-17-public-webmainbench-moli-latency-rerun", + "publication_state": "published", "comparison_kinds": [ "webmainbench_frozen_html_to_markdown" ], @@ -2507,6 +2703,7 @@ }, { "id": "2026-09-17-public-webmainbench-detail-drilldown", + "publication_state": "published", "comparison_kinds": [ "webmainbench_frozen_html_to_markdown" ], @@ -2535,6 +2732,7 @@ }, { "id": "2026-09-17-public-webmainbench-final-success-latency", + "publication_state": "published", "comparison_kinds": [ "webmainbench_frozen_html_to_markdown" ], @@ -2563,6 +2761,7 @@ }, { "id": "2026-09-17-public-webmainbench-whole-report-interpretation", + "publication_state": "published", "comparison_kinds": [ "webmainbench_frozen_html_to_markdown" ], @@ -2585,6 +2784,7 @@ }, { "id": "2026-09-17-public-webmainbench-denominators-and-version-examples", + "publication_state": "published", "comparison_kinds": ["webmainbench_frozen_html_to_markdown"], "date": "2026-09-17", "title": "用公式分实例解释计分页，并核对 Moli 版本差异", @@ -2601,6 +2801,7 @@ }, { "id": "2026-09-17-public-webmainbench-moli-1-1-7", + "publication_state": "published", "comparison_kinds": [ "webmainbench_frozen_html_to_markdown" ], @@ -2635,6 +2836,7 @@ }, { "id": "2026-09-16-public-webmainbench-reading-view", + "publication_state": "published", "comparison_kinds": ["webmainbench_frozen_html_to_markdown"], "date": "2026-09-16", "title": "明确主内容评测范围并提供网页阅读视图", @@ -2653,6 +2855,7 @@ }, { "id": "2026-09-16-public-webmainbench-decision-guide", + "publication_state": "published", "comparison_kinds": ["webmainbench_frozen_html_to_markdown"], "date": "2026-09-16", "title": "用真实案例重写评测解读", @@ -2672,6 +2875,7 @@ }, { "id": "2026-09-16-public-webmainbench-score-reading-and-layout", + "publication_state": "published", "comparison_kinds": ["webmainbench_frozen_html_to_markdown"], "date": "2026-09-16", "title": "澄清计分分支、复算身份与核心表排序", @@ -2691,6 +2895,7 @@ }, { "id": "2026-09-16-public-webmainbench-causal-interpretation", + "publication_state": "published", "comparison_kinds": ["webmainbench_frozen_html_to_markdown"], "date": "2026-09-16", "title": "用全局证据重写公开基准解读", @@ -2710,6 +2915,7 @@ }, { "id": "2026-09-16-public-webmainbench-anomaly-root-causes", + "publication_state": "published", "comparison_kinds": ["webmainbench_frozen_html_to_markdown"], "date": "2026-09-16", "title": "补齐异常结果根因与对照证据", @@ -2728,6 +2934,7 @@ }, { "id": "2026-09-16-public-webmainbench-report-information-architecture", + "publication_state": "published", "comparison_kinds": ["webmainbench_frozen_html_to_markdown"], "date": "2026-09-16", "title": "重整公开基准报告的信息架构与逐页诊断", @@ -2747,6 +2954,7 @@ }, { "id": "2026-09-15-public-webmainbench-dataset-and-result-clarity", + "publication_state": "published", "comparison_kinds": ["webmainbench_frozen_html_to_markdown"], "date": "2026-09-15", "title": "公开基准页面与方法结果分开展示", @@ -2765,6 +2973,7 @@ }, { "id": "2026-09-15-public-webmainbench-reading-and-inspection", + "publication_state": "published", "comparison_kinds": ["webmainbench_frozen_html_to_markdown"], "date": "2026-09-15", "title": "公开网页基准的逐页对照与分类筛选", @@ -2782,6 +2991,7 @@ }, { "id": "2026-09-15-resource-winner-presentation", + "publication_state": "published", "comparison_kinds": ["headless_html", "headless_markdown"], "date": "2026-09-15", "title": "资源对比按指标分组并明确领先者", @@ -2806,6 +3016,7 @@ }, { "id": "2026-09-14-five-method-resource-profile", + "publication_state": "published", "comparison_kinds": ["headless_html", "headless_markdown"], "date": "2026-09-14", "scoring_revision_input_sha256": "8e8edd5c6e327010b480d98df95e949589491e5b5d91963641d58303bc735f98", @@ -2833,6 +3044,7 @@ }, { "id": "2026-09-15-official-route-and-call-drilldown", + "publication_state": "published", "date": "2026-09-15", "comparison_kinds": ["web_fetch_methods"], "scoring_revision_input_sha256": [ @@ -2867,6 +3079,7 @@ }, { "id": "2026-09-11-full-matrix-root-cause-audit", + "publication_state": "published", "date": "2026-09-14", "comparison_kinds": ["web_fetch_methods"], "scoring_revision_input_sha256": [ @@ -2930,6 +3143,7 @@ }, { "id": "2026-09-10-twelve-method-refresh-contract", + "publication_state": "published", "date": "2026-09-10", "comparison_kinds": ["web_fetch_methods"], "scoring_revision_input_sha256": [ @@ -2969,6 +3183,7 @@ }, { "id": "2026-09-10-editorial-illustrations", + "publication_state": "published", "date": "2026-09-10", "title": "解读突出耗时减少比例并增加三张信息图", "summary": "首屏以 Chromium 为基准展示均值、P50、P95 的耗时减少比例；以结果总览和两张流程图替代重复总结与部分流程文字，标题改为直接的内容名称。", @@ -2990,6 +3205,7 @@ }, { "id": "2026-09-10-editorial-reading-order", + "publication_state": "published", "date": "2026-09-10", "title": "解读开篇增加综合通过率与六类结果", "summary": "五浏览器解读先展示综合通过率与六类任务成绩，正文解释差异，结尾归纳结论。浏览器配置改为对照表，章节统一编号。", @@ -3013,6 +3229,7 @@ }, { "id": "2026-09-10-five-browser-editorial", + "publication_state": "published", "date": "2026-09-10", "title": "增加五浏览器 Web Fetch 图文解读", "summary": "文章对比 Moli 1.1.1 与另外四款浏览器，明确 150 题中通过 135 题，完整展示均值、P50、P95，并用六个评测集的可切换案例并列标准答案与真实输出。原始调用、评分及榜单分母不变。", @@ -3060,6 +3277,7 @@ }, { "id": "2026-09-09-publication-architecture-and-privacy", + "publication_state": "published", "date": "2026-09-09", "title": "说明按需发布架构，补全安装清单与公开安全门禁", "summary": "报告明确区分逻辑路由与线上物理文件，补齐双语入口、七种方法的安装清单和公开安全门禁；同时按固定最终轮纠正四题、三个 Moli 版本的 12 个异常耗时。任务输出、判定和分数不变，延迟汇总已重算。", @@ -3119,6 +3337,7 @@ }, { "id": "2026-09-09-unified-method-comparison", + "publication_state": "published", "date": "2026-09-09", "title": "统一方法组合对比，补充 Moli 时延诊断", "summary": "结果分析页改为一个可同时处理双方法和多方法的对比器，并重新排版任务与结果；issue #66 的回退线索经公网顺序轮换与本地固定页对照后，保留为未能单因归因的长尾现象。正式任务、分数和耗时不变。", @@ -3164,6 +3383,7 @@ }, { "id": "2026-09-09-method-comparison-and-large-evidence", + "publication_state": "published", "date": "2026-09-09", "title": "增加同题方法组合对比，修复大输出卡顿", "summary": "结果分析页新增两方法差异和多方法交集；超过 256 KiB 的完整证据不再自动插入页面，改为按需加载或直接下载。正式任务、调用、判定、分数和耗时均未改变。", @@ -3202,6 +3422,7 @@ }, { "id": "2026-09-09-sites-and-report-skill", + "publication_state": "published", "date": "2026-09-09", "title": "迁移到 Sites，并固化评测报告工作流", "summary": "报告迁移到新的 Sites 域名并保留原站点；仓库新增跨评测类型的报告 Skill，覆盖评测设计、证据、独立核验、影响范围重跑和发布门禁。任务、输出、分数和耗时不变。", @@ -3240,6 +3461,7 @@ }, { "id": "2026-09-08-cross-session-rebuild-hardening", + "publication_state": "published", "date": "2026-09-08", "title": "固定默认报告证据与实测路线", "scoring_revision_input_sha256": "ea7b9d530566874af1b8e6b13796a29975d752564d6b143a9e540902d4ef8406", @@ -3279,6 +3501,7 @@ }, { "id": "2026-09-08-seven-method-final-audit", + "publication_state": "published", "date": "2026-09-08", "scoring_revision_input_sha256": [ "7ae9fbf06965f3fdddb8c2a6fef36ee6dc44f6f92ac2e592780bc5bca8f6d4bc", @@ -3352,6 +3575,7 @@ }, { "id": "2026-09-07-quote-field-boundary-explanation", + "publication_state": "published", "date": "2026-09-07", "title": "区分文字存在与字段边界完整", "summary": "复核 Quotes 异步列表的两版 Moli 输出：10 条引文和作者均已取得，但相邻字段及多标签之间没有可读分隔。任务结果保持未通过，详情页改为按引文、作者、标签逐字段呈现，避免把边界损失笼统写成文字缺失。", @@ -3390,6 +3614,7 @@ }, { "id": "2026-09-07-document-reference-and-coverage-explanation", + "publication_state": "published", "scoring_revision_input_sha256": "7c6fa386ec26720b5395bc790fa50fbffd607f38da383f2d45f03a3b35f2da7a", "date": "2026-09-07", "title": "核对文档参考，纠正等价表格误判", @@ -3443,6 +3668,7 @@ }, { "id": "2026-09-07-six-method-audited-correction", + "publication_state": "published", "scoring_revision_input_sha256": [ "42ab6a774b647a4999a13e2ec13907b2d71830b77d529be0f2814af8b0731d43", "9d5469dc7070fc70866646f8a460e63b56cef332038df743de0ac67ffab84df2", @@ -3522,6 +3748,7 @@ }, { "id": "2026-09-04-five-browser-fastest-markdown", + "publication_state": "published", "date": "2026-09-04", "title": "完成五种浏览器最快 Markdown 路线的 750 次实测", "summary": "Kitesurf、Lightpanda、Obscura、Moli 使用原生 Markdown，Chromium 使用渲染 DOM 加 node-html-markdown 2.0.0；150 题 × 5 种方法全部执行、独立参考生命周期与离线复评分全部通过。", @@ -3588,6 +3815,7 @@ }, { "id": "2026-09-04-complete-file-evaluation", + "publication_state": "published", "date": "2026-09-04", "title": "完成文件任务评测，建立报告完整性发布门槛", "summary": "36 次文件任务全部实际调用并判定：Moli 与 Crawl4AI 各通过 3 次，共 6 次通过、30 次未通过、0 次未测。整份报告的 1,800 个计划调用均有唯一结果和明确判定。", @@ -3647,6 +3875,7 @@ }, { "id": "2026-09-03-fairness-correction", + "publication_state": "published", "scoring_revision_input_sha256": "f3381fca684f6d453ef0ce2d10b0ccabfff45f64b64609901bd75b7f98fca28a", "date": "2026-09-03", "title": "统一验收口径，纠正参考、成绩与报告结论", diff --git a/scripts/build_report.py b/scripts/build_report.py index 484c365b7..3d7c36497 100644 --- a/scripts/build_report.py +++ b/scripts/build_report.py @@ -106,7 +106,7 @@ def requested_language(output, supported=SUPPORTED_REPORT_LANGUAGES): language=requested_language(args.output or ROOT/'reports/wcxb'))) raise SystemExit(0) evidence=require_evidence(args.evidence or current_evidence_dir()) - if profile.get('comparison_kind')!='webmainbench_frozen_html_to_markdown': + if profile.get('comparison_kind')!='webmainbench_frozen_html_to_markdown' and not profile.get('frozen_inputs'): canonical=(profile_path.parent if (profile_path.parent/'dataset.json').is_file() else ROOT/'evaluation') require_canonical_inputs(evidence,canonical) diff --git a/tests/package_smoke.py b/tests/package_smoke.py index 187fb7b2d..00a76bef6 100644 --- a/tests/package_smoke.py +++ b/tests/package_smoke.py @@ -1536,7 +1536,7 @@ def log_message(self, *args): 'category': 'web', 'track': 'web', 'task_kind': 'body', 'expected_any': ['Independent fact 0'], 'ground_truth_main_content': reference, - 'source_verification': {'method': 'independent HTTP capture'}} + 'source_verification': {'reference_mode': 'live_exact', 'method': 'independent HTTP capture'}} (stage / 'tasks.json').write_text(json.dumps({'schema_version': 1, 'id': 'certification-control', 'name': 'Certification control', 'tool': 'web_fetch', 'cases': [case], 'coverage_plan': {'web': ['fixture']}})) diff --git a/tests/report_check_fixture.py b/tests/report_check_fixture.py index af0453e72..02874d093 100644 --- a/tests/report_check_fixture.py +++ b/tests/report_check_fixture.py @@ -78,10 +78,10 @@ def create_report(base: Path, *, english=False, headless=False): path.write_bytes(b"fixture") pages = { "index.html": ( - "

1 类任务概览

本类最高： 判分完整性

" - "

综合通过率总榜

1 类分榜

", - "

1 task family

Highest score. Scoring completeness.

" - "

Overall ranking

1 task family ranking

", + '

综合通过率总榜

Overview
' + "

1 类任务概览

本类最高： 判分完整性

1 类分榜

", + '

Overall ranking

Overview
' + "

1 task family

Highest score. Scoring completeness.

1 task family ranking

", ), "methodology/index.html": ( "

计入耗时 不计入耗时 供应商上报的服务端耗时只作附加证据 " @@ -113,7 +113,7 @@ def create_revision_report(base: Path): home = root / "index.html" home.write_text( home.read_text() - + """

Overview
+ + """
Guide
Meaning
  • Input
  • Fetch
  • Score
  • Result
diff --git a/tests/test_apt_uri_download.py b/tests/test_apt_uri_download.py index 4cc45096f..59f62207e 100644 --- a/tests/test_apt_uri_download.py +++ b/tests/test_apt_uri_download.py @@ -50,3 +50,27 @@ def test_apt_plan_rejects_path_escape_and_missing_checksum(tmp_path: Path): plan.write_text("'https://example.invalid/fixture.deb' fixture.deb 12\n") with pytest.raises(ValueError, match="invalid apt package plan entry"): parse_plan(plan, tmp_path / "archives") + + +def test_sha256_plan_preserves_encoded_url_and_checks_package_bytes(tmp_path: Path): + payload = b"package bytes" + plan = tmp_path / "apt-uris.txt" + url = "https://example.invalid/fixture_1%2bdeb13_arm64.deb" + name = "fixture_1+deb13_arm64.deb" + digest = hashlib.sha256(payload).hexdigest() + plan.write_text(f"'{url}' {name} {len(payload)} SHA256:{digest}\n") + archives = tmp_path / "archives" + archives.mkdir() + item, = parse_plan(plan, archives) + assert item.url == url + assert item.destination.name == name + with patch("examples.web_fetch.docker.download_apt_uris.urllib.request.urlretrieve", + side_effect=lambda _url, target: Path(target).write_bytes(payload)): + download(item) + assert item.destination.read_bytes() == payload + item.destination.unlink() + with patch("examples.web_fetch.docker.download_apt_uris.urllib.request.urlretrieve", + side_effect=lambda _url, target: Path(target).write_bytes(b"altered bytes")): + with pytest.raises(ValueError, match="checksum mismatch"): + download(item) + assert not item.destination.exists() diff --git a/tests/test_assembly_evidence.py b/tests/test_assembly_evidence.py index 43ca5cc68..c657844dd 100644 --- a/tests/test_assembly_evidence.py +++ b/tests/test_assembly_evidence.py @@ -141,7 +141,7 @@ def fetch(self, case, timeout): text = 'Expected independent article' case = EvalCase('one', 'One', 'https://example.invalid/', 'synthetic', ['Expected'], ground_truth_main_content=text, - source_verification={'state': 'valid', 'method': 'independent source capture', + source_verification={'reference_mode': 'live_exact', 'state': 'valid', 'method': 'independent source capture', 'checked_at': datetime.now(timezone.utc).isoformat(), 'reference_sha256': hashlib.sha256(text.encode()).hexdigest(), 'reference_scope': 'Complete article'}) diff --git a/tests/test_dynamic_100_moli_119.py b/tests/test_dynamic_100_moli_119.py index 0c3963382..844b4411a 100644 --- a/tests/test_dynamic_100_moli_119.py +++ b/tests/test_dynamic_100_moli_119.py @@ -65,11 +65,19 @@ def test_dynamic_100_dataset_matches_the_frozen_local_audit(): assert len(revision["dynamic_dataset"]["changes"]) == 9 assert hashlib.sha256(historical.encode()).hexdigest() == receipt["bindings"]["dataset_sha256"] scorer_revision = revision["shared_scorer"] + known_sources = {scorer_revision['revised_sha256']} for followup in revision.get('shared_scorer_followups', []): assert followup['predecessor_source_sha256'] == scorer_revision['revised_sha256'] assert followup['provider_calls'] == 0 assert set(scorer_revision['regression_ids']) <= set(followup['regression_ids']) scorer_revision = {**scorer_revision, **followup} + known_sources.add(followup['revised_sha256']) + for parallel in revision.get('shared_scorer_parallel_followups', []): + assert parallel['predecessor_source_sha256'] in known_sources + assert parallel['provider_calls'] == 0 + assert any(row.get('parallel_predecessor_source_sha256') == parallel['revised_sha256'] + for row in revision['shared_scorer_followups']) + assert set(parallel['regression_ids']) <= set(scorer_revision['regression_ids']) assert scorer_revision["main_baseline_sha256"] == compatibility["bindings"]["scoring_source_sha256"] assert scorer_revision["revised_sha256"] == SCORING_SOURCE_SHA256 assert scorer_revision["scoring_regressions_sha256"] == hashlib.sha256( diff --git a/tests/test_dynamic_provider_waits.py b/tests/test_dynamic_provider_waits.py index 482142cd5..e3ee92adc 100644 --- a/tests/test_dynamic_provider_waits.py +++ b/tests/test_dynamic_provider_waits.py @@ -451,3 +451,20 @@ def test_dynamic_providers_reject_answer_bearing_hint_fields(cloudflare_request, assert firecrawl.error_type == "configuration" cloudflare_request.assert_not_called() firecrawl_request.assert_not_called() + + +@patch("harness.web_fetch.providers.cloudflare.request") +def test_kitesurf_native_dynamic_completion_uses_ordinary_url_route(mock_request): + from dataclasses import replace + mock_request.return_value = HttpResponse(200, "u", {}, b'{"success":true,"result":"Example"}') + case = replace(dynamic_case(), retrieval_hints={"version": 3, "mode": "native"}) + result = CloudflareMarkdownProvider("kitesurf", { + "api_token": "x", "account_id": "a", "browser": "kitesurf" + }).fetch(case, 45) + assert result.ok + body = mock_request.call_args.kwargs["json_body"] + assert body["url"] == case.url + assert body["gotoOptions"]["waitUntil"] == "networkidle2" + assert "waitForSelector" not in body + assert "waitForTimeout" not in body + assert result.metadata["route"] == "browser_rendering_markdown" diff --git a/tests/test_fixed_native_policy.py b/tests/test_fixed_native_policy.py index b123835fa..3176f26e1 100644 --- a/tests/test_fixed_native_policy.py +++ b/tests/test_fixed_native_policy.py @@ -59,3 +59,26 @@ def test_runner_rejects_extra_attempts_before_preflight(tmp_path, runs, retries) cases=[EvalCase("one", "one", "https://example.com/", "control", [])], providers=[provider], provider_config_path=tmp_path / "absent-providers.json", runs=runs, timeout=45, delay=0, max_failure_retries=retries) assert not (tmp_path / "run").exists() + + +def test_native_dataset_marker_reaches_real_runner_without_per_case_assistance(tmp_path): + from harness.web_fetch.retrieval_hints import native_fetch_dataset + from harness.web_fetch.runner import run_evaluation + from harness.web_fetch.config import load_dataset + source = {"id": "native-dynamic", "tool": "web_fetch", "cases": [{"id": "dynamic", "name": "Dynamic", + "url": "https://example.com/", "category": "dynamic", "track": "dynamic", "expected_any": ["fixture"]}]} + dataset = native_fetch_dataset(source) + assert native_fetch_dataset(dataset) == dataset + path = tmp_path / "dataset.json" + path.write_text(json.dumps(dataset)) + _, cases = load_dataset(path) + provider = build_provider("native", {"type": "fixture", "uniform_native_parameters": True}) + config = tmp_path / "providers.json" + config.write_text(json.dumps({"providers": {"native": {"type": "fixture", "uniform_native_parameters": True}}})) + rows, _ = run_evaluation(output_dir=tmp_path / "run", dataset_meta=dataset, dataset_path=path, + cases=cases, providers=[provider], provider_config_path=config, runs=1, timeout=45, delay=0) + assert len(rows) == 1 and rows[0]["ok"] + for extra in ({"selector": ".answer"}, {"delay_ms": 10}, {"completion": {}}, {"mode": "guided"}): + changed = replace(cases[0], retrieval_hints={**cases[0].retrieval_hints, **extra}) + with pytest.raises(ValueError): + provider.check_native_call(changed) diff --git a/tests/test_freeze_headless_references.py b/tests/test_freeze_headless_references.py index 05863cd63..f063022e3 100644 --- a/tests/test_freeze_headless_references.py +++ b/tests/test_freeze_headless_references.py @@ -8,6 +8,7 @@ from harness.web_fetch.body_coverage import body_requirements from harness.web_fetch.reference_freeze import ( _digest_json, + _change_for_case, _require_promotion, compute_reference_update, main, @@ -37,6 +38,32 @@ def setUp(self): 'revision':'oldid:123','source_snapshot_sha256':'c'*64}, } + def test_freezing_preserves_explicit_result_review_scope(self): + # Reference capture must not opt a cohort into a separate result audit. + for policy in ({}, {'require_independent_result_review': False}, + {'require_independent_result_review': True}): + with self.subTest(policy=policy), tempfile.TemporaryDirectory() as directory: + root=Path(directory) + captures=root/'captures';captures.mkdir() + (captures/'audit.json').write_text('[]\n') + source=root/'input.json';output=root/'frozen' + dataset={'comparison': policy, 'cases': [{ + 'id': 'synthetic-stable', 'track': 'web', + 'acceptance_contract': {'version': 1, 'schema': 'body_text', + 'minimum_records': 0, + 'stable_text_anchors': [{'id': 'body', + 'alternatives': ['Source prose']}]}}]} + source.write_text(json.dumps(dataset)) + main(['--dataset', str(source), '--captures', str(captures), + '--output', str(output)]) + frozen=json.loads((output/'dataset.json').read_text()) + self.assertEqual(frozen['comparison'], policy) + self.assertEqual(json.loads(source.read_text()), dataset) + + def test_promotion_reports_missing_computed_change_as_validation_error(self): + with self.assertRaisesRegex(ValueError,'omit promoted case: missing'): + _change_for_case([{'case_id':'present'}],'missing') + def test_changed_reference_fails_closed_without_explicit_promotion(self): with self.assertRaisesRegex(ValueError,'explicit --reference-promotion'): _require_promotion(None,dataset_sha256='d'*64, @@ -121,7 +148,7 @@ def test_unavailable_body_reference_recovers_from_complete_clean_capture(self): 'url':'https://example.com/docs','track':'web','task_kind':'body', 'ground_truth_main_content':'','body_requirements':{}, 'required_facts':[{'id':'fact','alternatives':['recovery fact']}], - 'source_verification':{'state':'unavailable','checked_at':'2026-09-01T00:00:00Z', + 'source_verification':{'reference_mode':'live_exact','state':'unavailable','checked_at':'2026-09-01T00:00:00Z', 'method':'Independent HTTP response'}} audit={'case_id':case['id'],'status':200,'state':'captured', 'checked_at':'2026-09-14T00:00:00Z','selector':'main', @@ -145,7 +172,7 @@ def test_unavailable_body_reference_stays_unavailable_when_capture_is_blocked(se 'url':'https://example.com/blocked','track':'web','task_kind':'body', 'ground_truth_main_content':'','body_requirements':{}, 'required_facts':[], - 'source_verification':{'state':'unavailable','checked_at':'2026-09-01T00:00:00Z', + 'source_verification':{'reference_mode':'live_exact','state':'unavailable','checked_at':'2026-09-01T00:00:00Z', 'method':'Independent HTTP response'}} audit={'case_id':case['id'],'status':403,'state':'unavailable', 'checked_at':'2026-09-14T00:00:00Z','selector':'main', @@ -166,7 +193,7 @@ def test_freeze_writes_nothing_until_changed_scope_is_promoted(self): 'url':'https://en.wikipedia.org/w/index.php?title=Fixture&oldid=123', 'track':'web','task_kind':'body','ground_truth_main_content':old, 'body_requirements':body_requirements(old),'required_facts':[], - 'source_verification':{'state':'valid','checked_at':checked, + 'source_verification':{'reference_mode':'live_exact','state':'valid','checked_at':checked, 'method':'fixture','reference_sha256': hashlib.sha256(old.encode()).hexdigest()}} dataset={'cases':[case],'coverage_plan':{'web':['p']}} diff --git a/tests/test_headless.py b/tests/test_headless.py index db963faf9..f3419f581 100644 --- a/tests/test_headless.py +++ b/tests/test_headless.py @@ -29,6 +29,138 @@ from harness.web_fetch.runner import run_evaluation +class NativeMetadataTests(unittest.TestCase): + def test_kitesurf_page_tasks_share_native_projection_and_cdp_completion(self): + from harness.web_fetch.task_outputs import resource_expression, DOCUMENT_PAGE_EXPRESSION, MEDIA_METADATA_EXPRESSION + for track,kind,url,expression in ( + ('image','resource','https://example.com/',resource_expression('image')), + ('video','resource','https://example.com/',resource_expression('video')), + ('document','body','https://example.com/viewer/',DOCUMENT_PAGE_EXPRESSION), + ('video','body','https://example.com/watch',MEDIA_METADATA_EXPRESSION), + ): + with self.subTest(track=track,kind=kind): + provider=HeadlessProvider('kitesurf',{'type':'headless_markdown','engine':'kitesurf', + 'account_id':'account','api_token':'token','wait_until':'networkidle2'}) + case=EvalCase('case','Case',url,'public',[],track=track,task_kind=kind) + raw=json.dumps({'ok':True,'content':'public output','metadata':{}}).encode() + with patch.object(provider,'_run_local_worker',return_value=(raw,'',None,0,len(raw),0)) as worker, \ + patch('harness.web_fetch.providers.cloudflare.CloudflareMarkdownProvider.fetch',side_effect=AssertionError('Top-level scrape loses frame content')): + result=provider.fetch(case,45) + request=worker.call_args.args[1] + self.assertEqual(request['wait_until'],'load') + self.assertFalse(request['remote_markdown_api']) + self.assertIn('browser=kitesurf',request['endpoint']) + self.assertEqual(result.metadata['output_projection_sha256'],hashlib.sha256(expression.encode()).hexdigest()) + + def test_native_envelope_preserves_content_and_actual_destination(self): + from types import SimpleNamespace + for engine, envelope in ( + ('lightpanda', {'content': '# Body', 'url': 'https://example.com/final', 'http_status': 200}), + ('obscura', {'content': '# Body', 'final_url': 'https://example.com/final', 'title': 'Body'}), + ): + with self.subTest(engine=engine), tempfile.TemporaryDirectory() as directory: + script = Path(directory) / 'native.js' + script.write_text('"# Body"') + metadata = {'format': 'lightpanda_json'} if engine == 'lightpanda' else { + 'format': 'obscura_eval', 'script_path': str(script), + 'script_sha256': hashlib.sha256(script.read_bytes()).hexdigest()} + request = {'engine': engine, 'binary': '/browser', 'url': 'https://example.com/start', + 'timeout': 5, 'output_mode': 'markdown', 'native_metadata': metadata} + response = SimpleNamespace(returncode=0, stdout=json.dumps(envelope).encode(), stderr=b'') + with patch('harness.web_fetch.providers.headless_worker.subprocess.run', return_value=response) as run: + result = fetch(request) + self.assertTrue(result['ok'], result) + self.assertEqual(result['content'], '# Body') + self.assertEqual(result['final_url'], 'https://example.com/final') + self.assertEqual(base64.b64decode(result['metadata']['_raw_stdout_b64']), response.stdout) + command = run.call_args.args[0] + if engine == 'lightpanda': + self.assertIn('--json', command) + else: + self.assertNotIn('--dump', command) + self.assertIn('location.href', command[command.index('--eval')+1]) + + def test_missing_destination_does_not_invent_requested_url(self): + from types import SimpleNamespace + request = {'engine': 'lightpanda', 'binary': '/browser', 'url': 'https://example.com/start', + 'timeout': 5, 'output_mode': 'markdown', + 'native_metadata': {'format': 'lightpanda_json'}} + response = SimpleNamespace(returncode=0, stdout=b'{"content":"Body"}', stderr=b'') + with patch('harness.web_fetch.providers.headless_worker.subprocess.run', return_value=response): + result = fetch(request) + self.assertTrue(result['ok']) + self.assertIsNone(result['final_url']) + response.stdout = b'{"url":"https://example.com/final"}' + with patch('harness.web_fetch.providers.headless_worker.subprocess.run', return_value=response): + result = fetch(request) + self.assertFalse(result['ok']) + self.assertIn('string content', result['error_message']) + + def test_modified_vendor_script_is_rejected_before_browser_call(self): + with tempfile.TemporaryDirectory() as directory: + script = Path(directory) / 'native.js' + script.write_text('"changed"') + with self.assertRaisesRegex(ValueError, 'hash mismatch'): + cli_command('obscura', '/browser', 'https://example.com/', 5, 'markdown', + native_metadata={'format': 'obscura_eval', 'script_path': str(script), + 'script_sha256': '0'*64}) + + + def test_isolated_resource_worker_needs_no_host_script_path(self): + from types import SimpleNamespace + with tempfile.TemporaryDirectory() as directory: + script = Path(directory) / 'native.js' + script.write_text('"# Isolated body"') + digest = hashlib.sha256(script.read_bytes()).hexdigest() + provider = HeadlessProvider('obscura', { + 'type': 'headless_markdown', 'engine': 'obscura', + 'resource_sample_interval_ms': 50, + 'resource_container_image': 'fixture:local', 'resource_binary': '/browser', + 'native_metadata': {'format': 'obscura_eval', 'script_path': str(script), + 'script_sha256': digest}, + }) + captured = [] + class IsolatedWorker: + def run(self, request, timeout): + captured.append(request) + # Model the container boundary: the host source file is + # unavailable when the worker constructs its browser call. + script.unlink() + response = SimpleNamespace(returncode=0, stderr=b'', stdout=json.dumps({ + 'content': '# Isolated body', 'final_url': 'https://example.com/final'}).encode()) + with patch('harness.web_fetch.providers.headless_worker.subprocess.run', return_value=response) as command: + result = fetch(request) + if result['ok']: + self.expression = command.call_args.args[0][command.call_args.args[0].index('--eval') + 1] + return ContainerCall(stdout=json.dumps(result).encode(), stderr=b'', exit_code=0, + duration_ms=1, resource={'cpu_ms': 1, 'memory_peak_bytes': 2, 'network_rx_bytes': 3}, + environment={'backend': 'docker_cgroup_v2'}) + runner = IsolatedWorker() + identity = {'binary_sha256': 'a'*64, 'reported_version': 'obscura 0.2.4', + 'version_probe_error': None} + case = EvalCase('case', 'Case', 'https://example.com/start', 'web', []) + with patch.object(provider, '_resource_runner', return_value=runner), \ + patch.object(provider, 'implementation_identity', return_value=identity): + result = provider.fetch(case, 1) + self.assertTrue(result.ok, result.error_message) + self.assertEqual(result.content, '# Isolated body') + self.assertEqual(result.final_url, 'https://example.com/final') + self.assertEqual(result.metadata['native_metadata']['script_sha256'], digest) + self.assertNotIn('script_path', captured[0]['native_metadata']) + self.assertIn('location.href', runner.expression) + self.assertIn('"# Isolated body"', runner.expression) + + def test_altered_in_transit_native_script_is_rejected_before_browser_launch(self): + request = {'engine': 'obscura', 'binary': '/browser', 'url': 'https://example.com/', + 'timeout': 1, 'output_mode': 'markdown', + 'native_metadata': {'format': 'obscura_eval', 'script_source': '"changed"', + 'script_sha256': hashlib.sha256(b'"original"').hexdigest()}} + with patch('harness.web_fetch.providers.headless_worker.subprocess.run') as command: + result = fetch(request) + self.assertFalse(result['ok']) + self.assertIn('hash mismatch', result['error_message']) + command.assert_not_called() + class HeadlessTests(unittest.TestCase): def test_resource_mode_routes_remote_client_through_isolated_container(self): provider = HeadlessProvider('kitesurf', { @@ -419,53 +551,10 @@ def test_every_measured_moli_version_uses_raw_route_for_direct_files(self): for name in ('moli-1.0.6', 'moli', 'moli-1.1.3'): with self.subTest(provider=name): self.assertEqual(output_mode_for_case('moli', 'headless_markdown', pdf), - 'document') + 'markdown') self.assertEqual(output_mode_for_case('moli', 'headless_markdown', image), 'file') - def test_moli_raw_pdf_is_extracted_and_preserved(self): - from types import SimpleNamespace - raw = b'%PDF-fixture' - request = dict(engine='moli', binary='/moli', url='https://example.com/paper.pdf', - timeout=1, output_mode='document') - with patch('harness.web_fetch.providers.headless_worker.subprocess.run', return_value=SimpleNamespace( - returncode=0, stdout=raw, stderr=b'')), patch( - 'harness.web_fetch.providers.headless_worker.extract_document_text', return_value=('Paper body', 'pypdf')): - result = fetch(request) - self.assertTrue(result['ok']) - self.assertEqual(result['content'], 'Paper body') - self.assertEqual(result['metadata']['document_extractor'], 'pypdf') - self.assertEqual(result['metadata']['raw_stdout_sha256'], hashlib.sha256(raw).hexdigest()) - self.assertIn('_raw_stdout_b64', result['metadata']) - - def test_moli_raw_file_is_returned_without_text_decoding(self): - from types import SimpleNamespace - raw = b'\x89PNG\r\n\x1a\n' + b'payload' - request = dict(engine='moli', binary='/moli', url='https://example.com/image.png', - timeout=1, output_mode='file') - with patch('harness.web_fetch.providers.headless_worker.subprocess.run', return_value=SimpleNamespace( - returncode=0, stdout=raw, stderr=b'')): - result = fetch(request) - self.assertTrue(result['ok']) - self.assertEqual(result['content'], '') - self.assertEqual(result['files_b64'], [base64.b64encode(raw).decode('ascii')]) - - def test_moli_document_html_fallback_remains_a_scored_text_answer(self): - from types import SimpleNamespace - request = dict(engine='moli', binary='/moli', url='https://example.com/file.docx', - timeout=1, output_mode='document') - completed = [ - SimpleNamespace(returncode=0, stdout=b'

Unsupported document

', stderr=b''), - SimpleNamespace(returncode=0, stdout='v1'.encode(), stderr=b''), - SimpleNamespace(returncode=0, stdout=b'# Unsupported document\n', stderr=b''), - ] - with patch('harness.web_fetch.providers.headless_worker.subprocess.run', side_effect=completed): - result = fetch(request) - self.assertTrue(result['ok']) - self.assertEqual(result['content'], '# Unsupported document\n') - self.assertEqual(result['metadata']['document_response'], - 'serialized_html_not_raw_document') - def test_unknown_cli_rejected(self): with self.assertRaises(ValueError): cli_command('unknown', '/binary', 'https://example.com', 45) diff --git a/tests/test_headless_access.py b/tests/test_headless_access.py index 08be2b2f5..2ee994183 100644 --- a/tests/test_headless_access.py +++ b/tests/test_headless_access.py @@ -13,7 +13,11 @@ def test_all_measured_engines_have_account_and_official_sources(self): with tempfile.TemporaryDirectory() as directory: root = Path(directory) (root / 'adoption').mkdir() - body = render_interfaces(root, []) + (root/'body.txt').write_text('Public body') + row={'provider':'kitesurf','case_id':'static-simple','attempt_id':'kitesurf-example', + 'artifact_path':'body.txt','provider_metadata':{'api_token':'credential-canary', + 'headers':{'Authorization':'Bearer credential-canary'},'engine':'kitesurf'}} + body = render_interfaces(root, [row]) self.assertEqual(body.count('

账号与获取入口

'), len(set(SPECS) & set(HEADLESS_ACCESS))) for name, access in HEADLESS_ACCESS.items(): if name in SPECS: @@ -23,7 +27,36 @@ def test_all_measured_engines_have_account_and_official_sources(self): self.assertTrue(access['links']) self.assertTrue(all(url.startswith('https://') for _, url in access['links'])) self.assertIn('<API_TOKEN>', body) - self.assertIn('不公开真实凭据', body) + self.assertNotIn('credential-canary', body) + self.assertNotIn('credential-canary',(root/'adoption/interfaces/kitesurf.json').read_text()) + + + def test_exported_chromium_interface_uses_observed_completion(self): + for wait in ('load', 'domcontentloaded', 'networkidle'): + with self.subTest(wait=wait), tempfile.TemporaryDirectory() as directory: + root = Path(directory) + (root / 'adoption').mkdir() + (root / 'body.txt').write_text('Body') + record = {'provider': 'chromium', 'case_id': 'static-simple', + 'attempt_id': 'chromium__static-simple__r001', + 'artifact_path': 'body.txt', + 'provider_metadata': {'navigation_wait_until': wait}} + render_interfaces(root, [record]) + interface = json.loads((root / 'adoption/interfaces/chromium.json').read_text())['interface'] + self.assertEqual(interface['request']['wait_until'], wait) + self.assertEqual(next(row[2] for row in interface['params'] if row[0] == 'wait_until'), wait) + self.assertEqual(SPECS['chromium']['request']['wait_until'], 'load') + + def test_executable_version_is_not_replaced_by_cdp_compatibility_version(self): + from harness.web_fetch.report_interfaces import method_version + for name,version,compat in [('moli','1.1.15','145.0.0.0'),('lightpanda','1.0.0','124.0.6367.29'),('obscura','0.2.4','145.0.0.0')]: + row={'provider':name,'provider_metadata':{'binary_sha256':'hash','browser_version':compat}} + self.assertEqual(method_version(row,{'hash':{'version':version}}),version) + self.assertIsNone(method_version(row)) + row['provider_metadata']['reported_version']=version + self.assertEqual(method_version(row),version) + self.assertEqual(method_version({'provider':'chromium','provider_metadata':{'browser_version':'155.0.8059.39'}}),'155.0.8059.39') + self.assertEqual(method_version({'provider':'kitesurf','provider_metadata':{'browser_version':'145.0.0.0'}}),'Beta · 托管后端版本未公开') if __name__ == '__main__': diff --git a/tests/test_headless_cancellation.py b/tests/test_headless_cancellation.py new file mode 100644 index 000000000..86ab38cfd --- /dev/null +++ b/tests/test_headless_cancellation.py @@ -0,0 +1,199 @@ +"""Local browser deadlines stop physical work without dropping later tasks.""" +import json +import os +import signal +import subprocess +import sys +import tempfile +import threading +import time +import unittest +from pathlib import Path +from unittest.mock import patch + +from harness.web_fetch.models import EvalCase, ProviderResult +from harness.web_fetch.providers.headless import HeadlessProvider +from harness.web_fetch.runner import run_evaluation + + +@unittest.skipUnless(os.name == 'posix', 'process-group worker contract') +class HeadlessCancellationTests(unittest.TestCase): + def worker(self, root): + worker = root / 'worker.py' + worker.write_text('''import json, os, subprocess, sys, time +from pathlib import Path +request = json.load(sys.stdin) +if request['url'].endswith('/slow'): + Path(request['pid_file']).write_text(str(os.getpid())) + child = subprocess.Popen([sys.executable, '-c', + "import time; from pathlib import Path; time.sleep(2); Path(" + repr(request['marker']) + ").write_text('orphan')"], start_new_session=True) + Path(request['pid_file'] + '.child').write_text(str(child.pid)) + print('browser still waiting', file=sys.stderr, flush=True) + print(json.dumps({'content': 'late answer'}), flush=True) + time.sleep(10) +else: + print(json.dumps({'content': 'on-time answer'}), flush=True) +''') + return worker + + def cleanup(self, pid_file): + for owned in (pid_file, Path(str(pid_file) + '.child')): + if owned.exists(): + try: + os.killpg(int(owned.read_text()), signal.SIGKILL) + except ProcessLookupError: + pass + + def test_timeout_stops_browser_children_retains_logs_and_runs_next_case(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + worker = self.worker(root) + pid_file, marker = root / 'pid', root / 'orphan' + + class LocalWorker(HeadlessProvider): + def fetch(self, case, timeout): + # The outer budget includes preparation/launch and must + # cancel even when the inner worker deadline is later. + raw, stderr, stopped, code, _, _ = self._run_local_worker( + worker, {'url': case.url, 'pid_file': str(pid_file), + 'marker': str(marker)}, 10, 100_000) + return ProviderResult(not stopped, + '' if stopped else json.loads(raw)['content'], + error_type=stopped, metadata={'stderr': stderr, 'exit_code': code}) + + provider = LocalWorker('local', {'type': 'headless_markdown', 'engine': 'obscura', 'binary': sys.executable}) + cases = [EvalCase(name, name, 'https://example.test/' + name, 'web', []) + for name in ('slow', 'fast')] + try: + records, _ = run_evaluation(output_dir=root / 'run', + dataset_meta={'id': 'cancellation', 'cases': []}, + dataset_path=root / 'dataset.json', cases=cases, + providers=[provider], provider_config_path=root / 'providers.json', + runs=1, timeout=.3, delay=0) + self.assertEqual([row['case_id'] for row in records], ['slow', 'fast']) + self.assertEqual(records[0]['error_type'], 'timeout') + self.assertLess(records[0]['elapsed_ms'], 500) + self.assertIn('browser still waiting', records[0]['provider_metadata']['stderr']) + self.assertLess(records[0]['provider_metadata']['exit_code'], 0) + self.assertEqual((root / 'run' / records[0]['artifact_path']).read_text(), '') + self.assertEqual((root / 'run' / records[1]['artifact_path']).read_text(), 'on-time answer') + time.sleep(2.1) + self.assertFalse(marker.exists(), 'timed-out browser child continued physical work') + finally: + self.cleanup(pid_file) + + def test_cancellation_during_process_creation_kills_the_late_child(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + worker = self.worker(root) + pid_file, marker = root / 'pid', root / 'orphan' + provider = HeadlessProvider('local', {'type': 'headless_markdown', 'engine': 'lightpanda'}) + created, release = threading.Event(), threading.Event() + real_popen = subprocess.Popen + children, results = [], [] + + def slow_popen(*args, **kwargs): + child = real_popen(*args, **kwargs) + if args and str(worker) in args[0]: + children.append(child) + created.set() + release.wait(1) + return child + + provider.begin_call() + def run(): + results.append(provider._run_local_worker(worker, + {'url': 'https://example.test/slow', 'pid_file': str(pid_file), + 'marker': str(marker)}, 10, 100_000)) + thread = threading.Thread(target=run, daemon=True) + try: + with patch('harness.web_fetch.providers.headless.subprocess.Popen', side_effect=slow_popen): + thread.start() + self.assertTrue(created.wait(1)) + provider.cancel() + release.set() + thread.join(1) + self.assertFalse(thread.is_alive()) + self.assertEqual(results[0][2], 'timeout') + self.assertIsNotNone(children[0].poll()) + finally: + release.set() + for child in children: + if child.poll() is None: + os.killpg(child.pid, signal.SIGKILL) + child.wait() + self.cleanup(pid_file) + thread.join(1) + + @unittest.skipUnless(os.environ.get('CHROMIUM_BINARY'), 'set CHROMIUM_BINARY for real browser cancellation') + def test_cancellation_stops_actual_chromium_detached_session(self): + from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer + arrived = threading.Event() + release = threading.Event() + + class Handler(BaseHTTPRequestHandler): + def log_message(self, *args): + pass + + def do_GET(self): + arrived.set() + release.wait(10) + body = b'Completed body' + try: + self.send_response(200) + self.send_header('Content-Length', str(len(body))) + self.end_headers() + self.wfile.write(body) + except (BrokenPipeError, ConnectionResetError): + pass + + def process_rows(): + output = subprocess.check_output(['ps', '-axo', 'pid=,ppid=,pgid=,stat='], text=True) + return {int(parts[0]): (int(parts[1]), int(parts[2]), parts[3]) + for line in output.splitlines() if len(parts := line.split()) == 4} + + server = ThreadingHTTPServer(('127.0.0.1', 0), Handler) + server_thread = threading.Thread(target=server.serve_forever, daemon=True) + server_thread.start() + provider = HeadlessProvider('chromium', {'engine': 'chromium', + 'binary': os.environ['CHROMIUM_BINARY'], 'allow_private_network': True}) + case = EvalCase('slow-navigation', 'slow-navigation', + 'http://127.0.0.1:' + str(server.server_port), 'web', []) + results = [] + owned_groups = set() + provider.begin_call() + thread = threading.Thread(target=lambda: results.append(provider.fetch(case, 10)), daemon=True) + try: + thread.start() + self.assertTrue(arrived.wait(8), 'real Chromium did not navigate to the controlled page') + process = provider._active_process + self.assertIsNotNone(process) + rows = process_rows() + owned = {process.pid} + while (expanded := owned | {pid for pid, row in rows.items() if row[0] in owned}) != owned: + owned = expanded + owned_groups = {rows[pid][1] for pid in owned} + self.assertTrue(owned_groups - {process.pid}, 'control did not exercise a detached browser session') + provider.cancel() + thread.join(2) + self.assertFalse(thread.is_alive(), 'cancelled Chromium call did not finish') + self.assertEqual(results[0].error_type, 'timeout') + survivors = process_rows() + self.assertFalse({pid for pid in owned if pid in survivors and not survivors[pid][2].startswith('Z')}, + 'physical Chromium work survived cancellation') + finally: + release.set() + for group in owned_groups: + try: + os.killpg(group, signal.SIGKILL) + except ProcessLookupError: + pass + thread.join(2) + server.shutdown() + server.server_close() + server_thread.join(1) + + def test_cleanup_grace_requires_an_actively_cancellable_local_route(self): + for config in ({'engine': 'kitesurf'}, {'engine': 'chromium', 'resource_sample_interval_ms': 50}): + self.assertEqual(HeadlessProvider('remote', config).cancellation_cleanup_seconds, 0) + self.assertGreater(HeadlessProvider('local', {'engine': 'chromium'}).cancellation_cleanup_seconds, 0) diff --git a/tests/test_headless_completion.py b/tests/test_headless_completion.py new file mode 100644 index 000000000..7a530d7cd --- /dev/null +++ b/tests/test_headless_completion.py @@ -0,0 +1,65 @@ +"""Browser integration controls for native whole-page completion.""" +import os +import threading +import time +import unittest +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer + +from harness.web_fetch.providers.headless_worker import fetch + +BINARY = os.environ.get('CHROMIUM_BINARY') + + +@unittest.skipUnless(BINARY, 'set CHROMIUM_BINARY to run real browser controls') +class NativePageCompletionTests(unittest.TestCase): + def test_native_completion_observes_load_body_and_preserves_explicit_early_snapshot(self): + class Handler(BaseHTTPRequestHandler): + def log_message(self, *args): + pass + + def do_GET(self): + if self.path == '/slow.gif': + time.sleep(.8) + body = bytes.fromhex('47494638396101000100800000000000ffffff21f90401000000002c00000000010001000002024401003b') + kind = 'image/gif' + else: + delayed = self.path == '/delayed' + body = ('''

Initial body

''' + + ('''''' if delayed else '

Completed page body

') + + '').encode() + kind = 'text/html; charset=utf-8' + self.send_response(200) + self.send_header('Content-Type', kind) + self.send_header('Content-Length', str(len(body))) + self.end_headers() + try: + self.wfile.write(body) + except BrokenPipeError: + pass + + server = ThreadingHTTPServer(('127.0.0.1', 0), Handler) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + origin = 'http://127.0.0.1:' + str(server.server_port) + request = {'engine': 'chromium', 'binary': BINARY, 'timeout': 10, + 'output_mode': 'markdown', 'task_kind': 'body', 'track': 'dynamic', + 'retrieval_hints': {'version': 3, 'mode': 'native'}, + 'allow_private_network': True} + try: + early = fetch({**request, 'url': origin + '/delayed', 'wait_until': 'domcontentloaded'}) + self.assertTrue(early['ok'], early) + self.assertNotIn('Completed page body', early['content']) + self.assertEqual(early['metadata']['navigation_wait_until'],'domcontentloaded') + native = fetch({**request, 'url': origin + '/delayed'}) + self.assertTrue(native['ok'], native) + self.assertIn('Completed page body', native['content']) + self.assertEqual(native['metadata']['navigation_wait_until'],'load') + fast = fetch({**request, 'url': origin + '/fast'}) + self.assertTrue(fast['ok'], fast) + self.assertIn('Completed page body', fast['content']) + finally: + server.shutdown() + server.server_close() + thread.join(1) diff --git a/tests/test_headless_files.py b/tests/test_headless_files.py new file mode 100644 index 000000000..0ce0f119b --- /dev/null +++ b/tests/test_headless_files.py @@ -0,0 +1,290 @@ +"""Actual browser controls for native binary delivery, independent of websites.""" +import base64 +import os +import io +import zipfile +import threading +import unittest +from pathlib import Path +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from harness.web_fetch.models import EvalCase +from harness.web_fetch.providers.headless import output_mode_for_case +from harness.web_fetch.providers.headless_worker import fetch + +BINARY = os.environ.get("CHROMIUM_BINARY") +PNG = base64.b64decode("iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwMCAO+a3e8AAAAASUVORK5CYII=") + + +class NativeFileRouteTests(unittest.TestCase): + def test_direct_document_route_preserves_html_viewer_and_plain_text_tasks(self): + for url in ('https://example.test/paper.pdf','https://example.test/paper.pdf#page=2','https://example.test/file.docx?download=1','https://example.test/pdf/1706.03762'): + case=EvalCase('doc','Document',url,'document',[],track='document') + for engine in ('moli','chromium','obscura','lightpanda','kitesurf'): + self.assertEqual(output_mode_for_case(engine,'headless_markdown',case),'markdown') + for url in ('https://example.test/plain.txt','https://example.test/documentation'): + case=EvalCase('doc','Document',url,'document',[],track='document') + self.assertEqual(output_mode_for_case('chromium','headless_markdown',case),'markdown') + + def test_file_contract_selects_browser_bytes_for_supported_engines(self): + case = EvalCase("file", "File", "https://example.test/download", "image", [], task_kind="file") + for engine in ("moli", "chromium", "obscura", "lightpanda", "kitesurf"): + self.assertEqual(output_mode_for_case(engine, "headless_markdown", case), "file") + + +@unittest.skipUnless(BINARY, "set CHROMIUM_BINARY for actual browser controls") +class ChromiumFileDeliveryTests(unittest.TestCase): + def test_inline_redirect_attachment_and_http_error_use_native_navigation(self): + calls = [] + class Handler(BaseHTTPRequestHandler): + def log_message(self, *args): pass + def do_GET(self): + calls.append(self.path) + if self.path == "/redirect": + self.send_response(302); self.send_header("Location", "/inline"); self.end_headers(); return + body = b"Access denied" if self.path == "/denied" else PNG + self.send_response(403 if self.path == "/denied" else 200) + self.send_header("Content-Type", "text/plain" if self.path == "/denied" else "image/png") + self.send_header("Content-Length", str(len(body))) + if self.path == "/attachment": self.send_header("Content-Disposition", 'attachment; filename="file.png"') + self.end_headers(); self.wfile.write(body) + server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + thread = threading.Thread(target=server.serve_forever, daemon=True); thread.start() + origin = "http://127.0.0.1:" + str(server.server_port) + try: + for path in ("/inline", "/redirect", "/attachment", "/denied"): + with self.subTest(path=path): + before = len(calls) + result = fetch({"engine": "chromium", "binary": BINARY, + "url": origin + path, "timeout": 10, + "output_mode": "file", "task_kind": "file", "track": "image"}) + if path == "/denied": + self.assertFalse(result["ok"], result) + self.assertEqual(result["status_code"], 403) + else: + self.assertTrue(result["ok"], result) + self.assertEqual([base64.b64decode(value) for value in result["files_b64"]], [PNG]) + self.assertEqual(result["final_url"], origin + ("/inline" if path == "/redirect" else path)) + self.assertEqual(calls[before:].count(path), 1 + result.get("metadata", {}).get("native_attachment_fetch_count", 0), "only the declared native attachment interface may add a fetch") + finally: + server.shutdown(); server.server_close(); thread.join(1) + + +@unittest.skipUnless(BINARY, "set CHROMIUM_BINARY for actual browser controls") +class ChromiumDocumentDeliveryTests(unittest.TestCase): + def test_native_document_bytes_use_the_shared_text_extractor(self): + from pypdf import PdfWriter + from pypdf.generic import DecodedStreamObject, DictionaryObject, NameObject + writer = PdfWriter(); page = writer.add_blank_page(width=200, height=100) + font = DictionaryObject({NameObject("/Type"): NameObject("/Font"), + NameObject("/Subtype"): NameObject("/Type1"), + NameObject("/BaseFont"): NameObject("/Helvetica")}) + page[NameObject("/Resources")] = DictionaryObject({NameObject("/Font"): DictionaryObject({NameObject("/F1"): writer._add_object(font)})}) + stream = DecodedStreamObject(); stream.set_data(b"BT /F1 12 Tf 20 50 Td (Frozen PDF body) Tj ET") + page[NameObject("/Contents")] = writer._add_object(stream) + pdf = io.BytesIO(); writer.write(pdf) + docx = io.BytesIO() + with zipfile.ZipFile(docx, "w") as archive: + archive.writestr("word/document.xml", 'Frozen DOCX body') + files = {"/fixture.pdf": (pdf.getvalue(), "application/pdf", "Frozen PDF body"), + "/fixture.docx": (docx.getvalue(), "application/vnd.openxmlformats-officedocument.wordprocessingml.document", "Frozen DOCX body")} + files.update({"/inline.docx": files["/fixture.docx"], + "/wrong-mime.docx": (docx.getvalue(), "text/html", "Frozen DOCX body"), + "/not-a-document.pdf": (b"Access challenge", "text/html", None)}) + calls = [] + class Handler(BaseHTTPRequestHandler): + def log_message(self, *args): pass + def do_GET(self): + calls.append(self.path) + raw, mime, _ = files.get(self.path, (b"missing", "text/plain", "")) + self.send_response(200 if self.path in files else 404) + self.send_header("Content-Type", mime); self.send_header("Content-Length", str(len(raw))) + if self.path == "/fixture.docx": self.send_header("Content-Disposition", 'attachment; filename="fixture.docx"') + self.end_headers(); self.wfile.write(raw) + server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + thread = threading.Thread(target=server.serve_forever, daemon=True); thread.start() + origin = "http://127.0.0.1:" + str(server.server_port) + try: + for path, (raw, _, expected) in files.items(): + with self.subTest(path=path): + result = fetch({"engine": "chromium", "binary": BINARY, + "url": origin + path, "timeout": 10, + "output_mode": "document", "task_kind": "body", "track": "document"}) + if expected is None: + self.assertFalse(result["ok"], result) + self.assertEqual(calls.count(path), 1, "invalid document bytes must not trigger another request") + continue + self.assertTrue(result["ok"], result) + self.assertIn(expected, result["content"]) + self.assertEqual(base64.b64decode(result["metadata"]["_raw_stdout_b64"]), raw) + self.assertEqual(calls.count(path), 1 + result.get("metadata", {}).get("native_attachment_fetch_count", 0), "only the declared native attachment interface may add a fetch") + finally: + server.shutdown(); server.server_close(); thread.join(1) + + +@unittest.skipUnless(BINARY, "set CHROMIUM_BINARY for actual browser controls") +class ChromiumFileTrustTests(unittest.TestCase): + def test_fixture_tls_exception_is_limited_to_declared_loopback_fixture(self): + import ssl + import subprocess + import tempfile + with tempfile.TemporaryDirectory() as temporary: + certificate = Path(temporary) / "certificate.pem" + key = Path(temporary) / "key.pem" + subprocess.run(["/usr/bin/openssl", "req", "-x509", "-newkey", "rsa:2048", + "-nodes", "-keyout", str(key), "-out", str(certificate), + "-days", "1", "-subj", "/CN=localhost"], + check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + class Handler(BaseHTTPRequestHandler): + def log_message(self, *args): pass + def do_GET(self): + self.send_response(200) + self.send_header("Content-Type", "image/png") + self.send_header("Content-Length", str(len(PNG))) + self.end_headers(); self.wfile.write(PNG) + server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + tls = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + tls.load_cert_chain(certificate, key) + server.socket = tls.wrap_socket(server.socket, server_side=True) + thread = threading.Thread(target=server.serve_forever, daemon=True); thread.start() + request = {"engine": "chromium", "binary": BINARY, + "url": f"https://127.0.0.1:{server.server_port}/file.png", + "timeout": 10, "output_mode": "file", "task_kind": "file", "track": "image"} + try: + for declared in ({}, {"ca_cert": str(certificate)}, + {"network_policy": "loopback_only"}): + result = fetch(dict(request, **declared)) + self.assertFalse(result["ok"], result) + result = fetch(dict(request, ca_cert=str(certificate), network_policy="loopback_only")) + self.assertTrue(result["ok"], result) + self.assertEqual(base64.b64decode(result["files_b64"][0]), PNG) + finally: + server.shutdown(); server.server_close(); thread.join(1) + + +@unittest.skipUnless(os.environ.get("OBSCURA_BINARY"), "set OBSCURA_BINARY for actual browser controls") +class ObscuraDocumentDeliveryTests(unittest.TestCase): + def test_original_document_bytes_use_shared_extraction(self): + # The same independent PDF/DOCX fixture and byte-identity assertions + # apply to each native route; only the supported invocation differs. + from unittest.mock import patch + original_fetch = fetch + def obscura_fetch(request): + return original_fetch({**request, "engine": "obscura", "allow_private_network": True}) + with patch(__name__ + ".BINARY", os.environ["OBSCURA_BINARY"]), patch(__name__ + ".fetch", obscura_fetch): + ChromiumDocumentDeliveryTests().test_native_document_bytes_use_the_shared_text_extractor() + + +@unittest.skipUnless(os.environ.get("LIGHTPANDA_BINARY"), "set LIGHTPANDA_BINARY for actual browser controls") +class LightpandaDocumentDeliveryTests(unittest.TestCase): + def test_original_document_bytes_use_shared_extraction(self): + from unittest.mock import patch + original_fetch = fetch + def native_fetch(request): + return original_fetch({**request, "engine": "lightpanda"}) + with patch(__name__ + ".BINARY", os.environ["LIGHTPANDA_BINARY"]), patch(__name__ + ".fetch", native_fetch): + ChromiumDocumentDeliveryTests().test_native_document_bytes_use_the_shared_text_extractor() + + +@unittest.skipUnless(os.environ.get("MOLI_BINARY"), "set MOLI_BINARY for actual browser controls") +class MoliNativeDeliveryTests(unittest.TestCase): + def test_inline_attachment_redirect_and_denial_deliver_only_native_bytes(self): + self._run_shared(ChromiumFileDeliveryTests().test_inline_redirect_attachment_and_http_error_use_native_navigation) + + def test_document_body_matches_native_markdown_without_external_parsing(self): + import hashlib, subprocess + from unittest.mock import patch + from pypdf import PdfWriter + from harness.web_fetch.providers.moli import MoliWebFetchProvider + pdf = io.BytesIO() + writer = PdfWriter(); writer.add_blank_page(width=200, height=100); writer.write(pdf) + docx = io.BytesIO() + with zipfile.ZipFile(docx, "w") as archive: + archive.writestr("word/document.xml", 'Document body') + files = {"/body.html": (b"
Native document body
", "text/html"), + "/plain.txt": (b"Native document body", "text/plain"), + "/fixture.pdf": (pdf.getvalue(), "application/pdf"), + "/fixture.docx": (docx.getvalue(), "application/vnd.openxmlformats-officedocument.wordprocessingml.document")} + class Handler(BaseHTTPRequestHandler): + def log_message(self, *args): pass + def do_GET(self): + raw, mime = files[self.path] + self.send_response(200); self.send_header("Content-Type", mime) + self.send_header("Content-Length", str(len(raw))); self.end_headers(); self.wfile.write(raw) + server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + thread = threading.Thread(target=server.serve_forever, daemon=True); thread.start() + binary = os.environ["MOLI_BINARY"] + provider = MoliWebFetchProvider("moli", {"binary": binary, "block_private_networks": False, + "expected_reported_version": subprocess.check_output([binary, "--version"], text=True).strip(), + "expected_binary_sha256": hashlib.sha256(Path(binary).read_bytes()).hexdigest()}) + try: + for path in files: + with self.subTest(path=path): + url = "http://127.0.0.1:" + str(server.server_port) + path + native = subprocess.run([binary, "fetch", "--dump", "markdown", "--wait-until", "done", + "--timeout", "10000", url], capture_output=True, timeout=12) + with patch("harness.web_fetch.providers.headless_worker.extract_document_text", + side_effect=AssertionError("external document conversion is forbidden")) as external: + result = provider.fetch(EvalCase("native-document", "Document", url, "document", [], track="document"), 10) + external.assert_not_called() + if path == "/body.html": + self.assertEqual(result.metadata["route"], "document_page_runtime") + else: + self.assertEqual(result.metadata["route"], "ordinary_markdown") + self.assertEqual(result.metadata["output"], "markdown") + if native.returncode == 0: + self.assertTrue(result.ok, result.error_message) + self.assertEqual(result.content.strip(), native.stdout.decode("utf-8", errors="replace").strip()) + else: + self.assertFalse(result.ok, result) + self.assertEqual(result.error_type, "tool_error") + if path in {"/body.html", "/plain.txt"}: + self.assertTrue(result.ok, result) + self.assertIn("Native document body", result.content) + finally: + server.shutdown(); server.server_close(); thread.join(1) + + def test_original_file_byte_budget_is_preserved(self): + import hashlib, subprocess + from harness.web_fetch.providers.moli import MoliWebFetchProvider + calls=[] + class Handler(BaseHTTPRequestHandler): + def log_message(self,*args): pass + def do_GET(self): + calls.append(self.path) + self.send_response(200);self.send_header('Content-Type','image/png') + self.send_header('Content-Length',str(len(PNG)));self.end_headers();self.wfile.write(PNG) + server=ThreadingHTTPServer(('127.0.0.1',0),Handler) + thread=threading.Thread(target=server.serve_forever,daemon=True);thread.start() + try: + binary=os.environ['MOLI_BINARY'] + provider=MoliWebFetchProvider('moli',{'binary':binary,'block_private_networks':False,'max_bytes':50, + 'expected_reported_version':subprocess.check_output([binary,'--version'],text=True).strip(), + 'expected_binary_sha256':hashlib.sha256(Path(binary).read_bytes()).hexdigest()}) + case=EvalCase('original-byte-budget','Fixture','http://127.0.0.1:'+str(server.server_port)+'/file.png', + 'image',[],track='image',task_kind='file') + result=provider.fetch(case,5) + self.assertFalse(result.ok,result) + self.assertEqual(result.error_type,'output_too_large') + self.assertEqual(result.metadata['original_file_bytes'],len(PNG)) + self.assertEqual(calls,['/file.png']) + finally: + server.shutdown();server.server_close();thread.join(1) + + def _run_shared(self, control): + from unittest.mock import patch + import hashlib, subprocess + from harness.web_fetch.providers.moli import MoliWebFetchProvider + binary = os.environ["MOLI_BINARY"] + config = {"binary":binary,"block_private_networks":False, + "expected_reported_version":subprocess.check_output([binary,"--version"],text=True).strip(), + "expected_binary_sha256":hashlib.sha256(Path(binary).read_bytes()).hexdigest()} + def native_fetch(request): + provider=MoliWebFetchProvider('moli',config) + case=EvalCase('independent-native-delivery','Fixture',request['url'],'fixture',[], + track=request['track'],task_kind=request['task_kind']) + result=provider.fetch(case,request['timeout']) + return dict(ok=result.ok,content=result.content,status_code=result.status_code,final_url=result.final_url, + error_type=result.error_type,error_message=result.error_message,metadata=result.metadata, + files_b64=[base64.b64encode(file['data']).decode() for file in result.files]) + with patch(__name__ + ".BINARY", binary), patch(__name__ + ".fetch", native_fetch): + control() diff --git a/tests/test_headless_projection.py b/tests/test_headless_projection.py index cf1e0c3b3..d07d55d42 100644 --- a/tests/test_headless_projection.py +++ b/tests/test_headless_projection.py @@ -30,10 +30,11 @@ def test_projects_only_a_complete_provider_without_changing_artifacts(self): path.write_bytes(data) records.append({"attempt_id": f"{provider}__{identifier}__r001", "provider": provider, "case_id": identifier, "run": 1, + "error_message": "正文\u2028包含\u2029合法 Unicode 分隔符", "started_at": "2026-01-01T00:00:00+00:00", "completed_at": "2026-01-01T00:00:01+00:00", "elapsed_ms": 1000, "artifact_path": relative, "metrics": {"content_sha256": hashlib.sha256(data).hexdigest()}}) - (source / "results.jsonl").write_text("".join(json.dumps(row) + "\n" for row in records)) + (source / "results.jsonl").write_text("".join(json.dumps(row, ensure_ascii=False) + "\n" for row in records)) manifest = {"started_at": "2026-01-01T00:00:00+00:00", "completed_at": "2026-01-01T00:00:02+00:00", "dataset": {"id": "d", "case_count": 2}, diff --git a/tests/test_headless_report.py b/tests/test_headless_report.py index 88514aa99..e067f0c32 100644 --- a/tests/test_headless_report.py +++ b/tests/test_headless_report.py @@ -41,6 +41,30 @@ from harness.web_fetch.speed_cohort import build_summary as build_speed_summary +class BrowserCohortRegistryTests(unittest.TestCase): + def test_generic_method_name_uses_observed_version_without_an_old_release_label(self): + from harness.web_fetch.methods import method_page_model + from harness.web_fetch.report_methods import display_name + for version in ('1.1.15','1.2.0'): + records=[{'provider_type':'moli_webfetch','provider_metadata':{'reported_version':'moli '+version}}] + model=method_page_model('moli',display_name('moli'),records) + self.assertIn('Moli '+version,model['title']) + self.assertNotIn('1.1.5',model['title']) + + + def test_task_specific_moli_adapter_keeps_browser_report_routes(self): + from harness.web_fetch.report import is_headless_cohort + from harness.web_fetch.report_explainer import _is_headless_comparison, supports_authored_explainer + records = [{'provider_type': 'moli_webfetch'}, {'provider_type': 'headless_markdown'}] + self.assertTrue(is_headless_cohort(records)) + self.assertTrue(_is_headless_comparison(records)) + self.assertFalse(supports_authored_explainer({'risk': {'track': 'risk_control'}})) + for unrelated in ([], [{'provider_type': 'moli_webfetch'}], + records + [{'provider_type': 'firecrawl'}]): + self.assertFalse(is_headless_cohort(unrelated)) + self.assertFalse(_is_headless_comparison(unrelated)) + + class HeadlessReportTests(unittest.TestCase): def test_resource_cards_name_each_actual_metric_winner(self): providers = ['moli', 'lightpanda', 'obscura', 'chromium'] @@ -78,7 +102,7 @@ def test_resource_co_leaders_are_named_in_both_languages(self): self.assertTrue(result['winner_tied']) english = localize_html(page, PurePosixPath('guide/index.html')) self.assertIn('Resource-efficiency co-leaders in this run', english) - self.assertIn('Moli 1.1.1, Lightpanda, Obscura, Chromium', english) + self.assertIn('Moli, Lightpanda, Obscura, Chromium', english) self.assertNotIn('资源效率并列领先', english) def test_resource_table_excludes_unobservable_remote_provider(self): @@ -265,6 +289,9 @@ def binding(cs, row): annotated = record | {'diagnosis': {'independent_source_check': 'available'}, 'title_quality': {'score': 0.5}, 'failure_category': 'label'} self.assertEqual(binding(cases, annotated), expected) + local = record | {'provider_metadata': record['provider_metadata'] | { + 'local_report_complete_diagnostic_links': True}} + self.assertEqual(binding(cases, local), expected) mutations = [record | {'ok': False}, record | {'elapsed_ms': 11}, record | {'metrics': record['metrics'] | {'content_pass': False}}, record | {'metrics': record['metrics'] | {'content_sha256': 'changed'}}, @@ -525,7 +552,7 @@ def prepare(self,root): now=datetime.now(timezone.utc) case={'id':'body','name':'Body','url':'https://example.test/body','track':'web', 'task_kind':'body','ground_truth_main_content':'Reference body', - 'source_verification':{'state':'valid','method':'independent fixture', + 'source_verification':{'state':'valid','reference_mode':'live_exact','method':'independent fixture', 'checked_at':(now-timedelta(seconds=10)).isoformat(), 'reference_sha256':sha256('Reference body')}} dataset={'cases':[case],'coverage_plan':{'web':['p']}} @@ -594,3 +621,216 @@ def test_diagnostic_assembly_does_not_bypass_publication_gate(self): with self.assertRaises(ReferenceLifecycleError): build_report(root/'diagnostic',root/'report') self.assertFalse((root/'report').exists()) + +class CurrentBrowserArticleTests(unittest.TestCase): + def example(self, root): + from harness.web_fetch.report_explainer import _digest, _headless_article_input + cases={cid:{'id':cid,'name':cid,'track':track,'url':'https://example.com/'+cid, + 'ground_truth_main_content':'Expected complete body'} + for cid,track in [('web-case','web'),('risk-case','risk_control')]} + rows=[] + for name,values in [('moli',[(True,10),(False,30)]),('lightpanda',[(True,20),(True,40)])]: + for (cid,case),(passed,ms) in zip(cases.items(),values): + attempt=name+'__'+cid + artifact='artifacts/'+attempt+'.txt' + (root/artifact).parent.mkdir(exist_ok=True) + (root/artifact).write_text('Expected complete body' if passed else 'Access denied') + (root/'attempts').mkdir(exist_ok=True) + (root/'attempts'/f'{attempt}.html').write_text('

Canonical result and error

') + rows.append({'provider':name,'provider_type':'headless_markdown','case_id':cid, + 'attempt_id':attempt,'artifact_path':artifact,'ok':passed,'elapsed_ms':ms, + 'started_at':'2026-10-09T00:00:00Z','provider_metadata':{}, + 'error_type':None if passed else 'http_status', + 'error_message':None if passed else 'Origin returned HTTP 403', + 'metrics':{'content_pass':passed}}) + content={'schema':'webfetch_browser_interpretation/1', + 'evidence_sha256':_digest(_headless_article_input(cases,rows,['moli','lightpanda'])), + 'title':'浏览器结果解读','intro':'相同任务和规则下的测量。', + 'scoring':'正文和最终页面地址都属于验收范围。','conclusions':'结合正文与交付结果选择方法。', + 'providers':{name:{'explanation':'这一节解释实际返回的内容。', + 'specimen_case_id':'risk-case'} for name in ['moli','lightpanda']}} + p=root/'report-assets/content.json';p.parent.mkdir();p.write_text(json.dumps(content,ensure_ascii=False)) + binding={'path':'report-assets/content.json','sha256':hashlib.sha256(p.read_bytes()).hexdigest()} + return cases,rows,binding + + def test_new_cohort_has_dynamic_families_populations_and_evidence(self): + from harness.web_fetch.report_explainer import render_explainer, check_current_browser_article + with tempfile.TemporaryDirectory() as temp: + root=Path(temp);cases,rows,binding=self.example(root) + body=render_explainer(root,cases,rows,content_binding=binding) + (root/'guide/index.html').write_text(body) + model=check_current_browser_article(root,cases,rows,binding) + self.assertEqual(model['cases'],2) + self.assertEqual(model['stats']['moli']['pass_rate'],0.5) + self.assertEqual(model['stats']['moli']['all']['mean'],20) + self.assertEqual(model['stats']['moli']['successful']['mean'],10) + self.assertEqual(model['stats']['lightpanda']['successful']['n'],2) + self.assertIn('风控网站正文',body) + self.assertIn('Origin returned HTTP 403',body) + self.assertIn('独立结果审查：0/4',body) + self.assertNotIn('150 道',body) + self.assertNotIn('1.1.5',body) + + def test_release_rejects_wrong_highlights_values_and_missing_evidence(self): + from harness.web_fetch.report_explainer import render_explainer, check_current_browser_article + with tempfile.TemporaryDirectory() as temp: + root=Path(temp);cases,rows,binding=self.example(root) + body=render_explainer(root,cases,rows,content_binding=binding) + page=root/'guide/index.html';page.write_text(body) + changes=[body.replace(' data-winner="true"','',1), + body.replace('data-value="0.5"','data-value="0.9"',1), + body.replace('data-browser-error="moli"','data-removed="moli"',1), + body.replace('../attempts/moli__risk-case.html','../attempts/absent.html',1), + body.replace('data-count="1"','data-count="9"',1), + body.replace('data-browser-interface="moli"','data-removed-interface="moli"',1), + body.replace('data-case-count="1"','data-case-count="9"',1), + body.replace('CLI fetch --dump markdown → 内置转换 → Markdown','共同转换 → Markdown',1), + body.replace('等待：CLI 等待 done','等待：不等待',1)] + for changed in changes: + with self.subTest(change=changes.index(changed)): + page.write_text(changed) + with self.assertRaises(ValueError):check_current_browser_article(root,cases,rows,binding) + page.write_text(body) + (root/'attempts/moli__risk-case.html').unlink() + with self.assertRaisesRegex(ValueError,'target is broken'): + check_current_browser_article(root,cases,rows,binding) + + def test_new_content_cannot_hide_incomplete_matrix_or_changed_measurements(self): + from harness.web_fetch.report_explainer import render_explainer, _digest, _headless_article_input + with tempfile.TemporaryDirectory() as temp: + root=Path(temp);cases,rows,binding=self.example(root) + changed=deepcopy(rows);changed[0]['elapsed_ms']=100 + with self.assertRaisesRegex(ValueError,'evidence changed'): + render_explainer(root,cases,changed,content_binding=binding) + p=root/binding['path'];content=json.loads(p.read_text()) + content['evidence_sha256']=_digest(_headless_article_input(cases,rows[:-1],['moli','lightpanda'])) + p.write_text(json.dumps(content));binding['sha256']=hashlib.sha256(p.read_bytes()).hexdigest() + with self.assertRaisesRegex(ValueError,'complete declared matrix'): + render_explainer(root,cases,rows[:-1],content_binding=binding) + + def test_public_builder_materializes_bound_prose_and_preserves_scores(self): + from harness.web_fetch.config import load_dataset + from harness.web_fetch.providers.fixture import FixtureProvider + from harness.web_fetch.runner import run_evaluation + from harness.web_fetch.report_build import build_report + from harness.web_fetch.report_explainer import _digest, _headless_article_input, check_current_browser_article + with tempfile.TemporaryDirectory() as temp: + root=Path(temp);source=root/'evidence';report=root/'report' + dataset={'schema_version':1,'id':'synthetic-browser-cohort','name':'Synthetic cohort', + 'tool':'web_fetch','cases':[ + {'id':cid,'name':cid+'\u2028正文\u0085保留','category':'test','track':track, + 'url':'https://example.com/'+cid,'expected_any':['Expected body'], + 'required_facts':[{'id':'body','label':'正文','alternatives':['Expected body']}], + 'source_verification':{'selector':'main'}, + 'source_benchmark':{'stable_annotation':{ + 'anchor_evidence':'sources/retired-qualification.txt', + 'anchor_evidence_sha256':'0'*64}}} + for cid,track in [('web-case','web'),('risk-case','risk_control')]], + 'coverage_plan':{'web':['moli','lightpanda'],'risk_control':['moli','lightpanda']}, + 'comparison':{'kind':'synthetic_browser_report','require_independent_result_review':False, + 'timeout_seconds':5,'runs_per_case':1,'concurrency':1,'conversion':'offline fixture', + 'reference_binding_mode':'frozen_task_contracts','publication_state':'verified', + 'unverified_reference_cases':[]}} + input_path=root/'dataset.json';input_path.write_text(json.dumps(dataset)) + configuration=root/'providers.json';configuration.write_text('{}') + metadata,cases=load_dataset(input_path) + rows,_=run_evaluation(output_dir=source,dataset_meta=metadata,dataset_path=input_path, + cases=cases,providers=[FixtureProvider('moli',{'type':'fixture','fail_cases':['risk-case']}), + FixtureProvider('lightpanda',{'type':'fixture'})], + provider_config_path=configuration,runs=1,timeout=5,delay=0) + measured=json.loads((source/'dataset.json').read_text());case_map={c['id']:c for c in measured['cases']} + content={'schema':'webfetch_browser_interpretation/1', + 'evidence_sha256':_digest(_headless_article_input(case_map,rows,['moli','lightpanda'])), + 'title':'新一轮浏览器结果','intro':'统一规则的合成测试。', + 'scoring':'正文必须满足题目要求。','conclusions':'完整比较所有调用。', + 'providers':{name:{'specimen_case_id':'risk-case','explanation':'展示实际调用及程序验收结果。'} for name in ['moli','lightpanda']}} + content_path=source/'report-assets/content.json';content_path.parent.mkdir();content_path.write_text(json.dumps(content,ensure_ascii=False)) + binding={'path':'report-assets/content.json','sha256':hashlib.sha256(content_path.read_bytes()).hexdigest()} + provenance=source/'implementation-provenance.json' + provenance.write_text(json.dumps({'schema_version':1,'implementations':[ + {'provider':'moli','binary_sha256':'synthetic','version':'synthetic'}]})) + measured['comparison']['implementation_provenance']={ + 'path':'implementation-provenance.json','sha256':hashlib.sha256(provenance.read_bytes()).hexdigest()} + measured['comparison']['report_interpretation']=binding + (source/'dataset.json').write_text(json.dumps(measured)) + before=(source/'results.jsonl').read_bytes() + build_report(source,report) + self.assertEqual((report/'results.jsonl').read_bytes(),before) + analysis=(report/'analysis/index.html').read_text() + self.assertNotIn('已核验判定',analysis, + 'A machine-only cohort must not claim independently reviewed verdicts') + self.assertIn('程序判定',analysis) + self.assertEqual((report/binding['path']).read_bytes(),content_path.read_bytes()) + self.assertEqual((report/'implementation-provenance.json').read_bytes(),provenance.read_bytes()) + check_current_browser_article(report,case_map,rows,binding) + from bs4 import BeautifulSoup + offline=BeautifulSoup((report/'guide/article.html').read_text(),'html.parser') + self.assertFalse(offline.select('script,link[rel="stylesheet"]')) + for anchor in offline.select('a[href]'): + self.assertTrue(anchor['href'].startswith(('http://','https://','#'))) + if anchor['href'].startswith('#'): + self.assertIsNotNone(offline.find(id=anchor['href'][1:])) + for row in rows: + if row['case_id']=='risk-case': + detail=offline.find(id='offline-result-'+row['attempt_id']) + self.assertIsNotNone(detail) + self.assertIn(row['attempt_id'],detail.get_text()) + self.assertIn('新一轮浏览器结果',(report/'index.html').read_text()) + home=(report/'index.html').read_text() + self.assertLess(home.index('id="overall-ranking"'),home.index('id="overview"')) + self.assertNotIn('达标率',(report/'datasets/index.html').read_text()) + self.assertNotIn('全部方法执行结果',(report/'datasets/risk-case.html').read_text()) + self.assertIn('../analysis/index.html#inspect?query=risk-case',(report/'datasets/risk-case.html').read_text()) + self.assertIn('查看各方法执行结果',(report/'datasets/risk-case.html').read_text()) + self.assertNotIn('其余六种方法',(report/'accounts/index.html').read_text()) + self.assertTrue('Moli 1.1.1 的普通网页' not in (report/'adoption/index.html').read_text(),'Measured-method introduction retained an old generic version') + + # Exercise the published CLI and release gate with a compact profile, + # without duplicating complete task/reference content into Git. + import subprocess, sys + from harness.web_fetch.storage import frozen_input_bindings + manifest=json.loads((source/'manifest.json').read_text()) + (source/'execution.json').write_text(json.dumps({'batches':[manifest]})) + profile={'schema_version':1,'dataset_id':measured['id'], + 'comparison_kind':measured['comparison']['kind'],'case_count':2, + 'case_ids_sha256':hashlib.sha256(json.dumps([c['id'] for c in measured['cases']], + separators=(',',':'),ensure_ascii=True).encode()).hexdigest(), + 'providers':['moli','lightpanda'], + **{k:v for k,v in measured['comparison'].items() if k!='kind'}, + 'case_inventory':{track:[c['id'] for c in measured['cases'] if c['track']==track] + for track in ['web','risk_control']}, + 'frozen_inputs':frozen_input_bindings(source)} + repository=Path(__file__).resolve().parents[1] + with tempfile.TemporaryDirectory(prefix='report-profile-test-',dir=repository/'evaluation') as profile_dir: + profile_path=Path(profile_dir)/'current-report.json' + measured['comparison']['reproduction_profile']=profile_path.relative_to(repository).as_posix() + measured['comparison']['report_output']='reports/synthetic-browser' + (source/'dataset.json').write_text(json.dumps(measured)) + profile['frozen_inputs']=frozen_input_bindings(source) + profile_path.write_text(json.dumps(profile)) + cli_report=root/'cli-report' + build=subprocess.run([sys.executable,'scripts/build_report.py','--profile',str(profile_path), + '--evidence',str(source),'--output',str(cli_report)], + cwd=repository,capture_output=True,text=True) + self.assertEqual(build.returncode,0,build.stderr) + gate=subprocess.run([sys.executable,'scripts/check_report.py',str(cli_report), + '--profile',str(profile_path)],cwd=repository,capture_output=True,text=True) + self.assertEqual(gate.returncode,0,gate.stderr+gate.stdout) + revision_gate=subprocess.run([sys.executable,'scripts/check_report_revision.py',str(cli_report)], + cwd=repository,capture_output=True,text=True) + self.assertEqual(revision_gate.returncode,0,revision_gate.stderr+revision_gate.stdout) + self.assertIn(measured['comparison']['reproduction_profile'],(cli_report/'reproduce/index.html').read_text()) + manifest_path=cli_report/'report-manifest.json';original_manifest=manifest_path.read_bytes() + for mutation in ('renderer','dataset_sha256'): + manifest=json.loads(original_manifest) + if mutation=='renderer':manifest['template']['renderer']='copied_renderer.py' + else:manifest['dataset_sha256']='0'*64 + manifest_path.write_text(json.dumps(manifest)) + rejected=subprocess.run([sys.executable,'scripts/check_report.py',str(cli_report), + '--profile',str(profile_path)],cwd=repository,capture_output=True,text=True) + self.assertNotEqual(rejected.returncode,0) + manifest_path.write_bytes(original_manifest) + content_path.write_text('{}') + with self.assertRaisesRegex(ValueError,'differs from binding'): + build_report(source,root/'tampered-report') + self.assertFalse((root/'tampered-report').exists()) diff --git a/tests/test_interrupted_evidence.py b/tests/test_interrupted_evidence.py new file mode 100644 index 000000000..06758af8a --- /dev/null +++ b/tests/test_interrupted_evidence.py @@ -0,0 +1,143 @@ +from dataclasses import asdict +import json +from pathlib import Path +import tempfile +import unittest + +from harness.web_fetch.models import EvalCase +from harness.web_fetch.providers.fixture import FixtureProvider +from harness.web_fetch.runner import run_evaluation +from harness.web_fetch.evidence import project_provider_run + + +class InterruptedEvidenceTests(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory() + self.addCleanup(self.temp.cleanup) + self.root = Path(self.temp.name) + self.source = self.root / "interrupted" + cases = [EvalCase(id=name, name=name, url="https://example.test/" + name, + category="web", expected_any=["anchor"]) + for name in ("pass", "fail", "pending")] + dataset = {"tool": "web_fetch", "cases": [asdict(case) for case in cases], + "coverage_plan": {"web": ["fixture"]}} + path = self.root / "input.json" + path.write_text(json.dumps(dataset)) + config = self.root / "providers.json" + config.write_text(json.dumps({"providers": {"fixture": {"type": "fixture"}}})) + run_evaluation(output_dir=self.source, dataset_meta=dataset, dataset_path=path, + cases=cases, providers=[FixtureProvider("fixture", {"type": "fixture", "fail_cases": ["fail"]})], + provider_config_path=config, runs=1, timeout=1, delay=0) + self.rows = [json.loads(line) for line in (self.source / "results.jsonl").read_text().split("\n") if line] + self.rows = self.rows[:2] + (self.source / "results.jsonl").write_text("".join(json.dumps(row) + "\n" for row in self.rows)) + manifest = json.loads((self.source / "manifest.json").read_text()) + for key in ("completed_at", "attempt_ids", "attempt_count"): + manifest.pop(key) + (self.source / "manifest.json").write_text(json.dumps(manifest)) + self.original = {str(p.relative_to(self.source)): p.read_bytes() + for p in self.source.rglob("*") if p.is_file()} + + def test_recovers_every_saved_pass_and_failure_without_calls_or_source_mutation(self): + output = self.root / "retained" + project_provider_run(self.source, output, "fixture", retain_completed=True) + rows = [json.loads(line) for line in (output / "results.jsonl").read_text().split("\n") if line] + self.assertEqual(rows, self.rows) + self.assertEqual([row["ok"] for row in rows], [True, False]) + self.assertEqual({str(p.relative_to(self.source)): p.read_bytes() + for p in self.source.rglob("*") if p.is_file()}, self.original) + batch = json.loads((output / "execution.json").read_text())["batches"][0] + self.assertEqual(batch["attempt_ids"], [row["attempt_id"] for row in rows]) + self.assertEqual(batch["recovery"]["original_batch_completed"], False) + self.assertEqual(batch["recovery"]["pending_logical_results"], 1) + self.assertEqual(batch["completed_at"], rows[-1]["completed_at"]) + + def test_rejects_changed_input_configuration_log_or_output(self): + for kind in ("input", "config", "log", "output", "identity"): + with self.subTest(kind=kind): + manifest = json.loads((self.source / "manifest.json").read_text()) + if kind == "input": target = Path(manifest["dataset"]["source"]) + elif kind == "config": target = Path(manifest["provider_config_source"]) + elif kind == "output": target = self.source / self.rows[0]["artifact_path"] + elif kind == "log": target = self.source / self.rows[0]["provider_metadata"]["call_log"]["path"] + else: target = self.source / "results.jsonl" + before = target.read_bytes() + if kind == "identity": + changed = json.loads(json.dumps(self.rows)) + changed[0]["attempt_id"] = "fixture__other__r001" + target.write_text("".join(json.dumps(row) + "\n" for row in changed)) + else: target.write_bytes(before + b" ") + try: + with self.assertRaises(ValueError): + project_provider_run(self.source, self.root / kind, "fixture", retain_completed=True) + finally: target.write_bytes(before) + + def test_recovery_cannot_select_only_passing_cases(self): + with self.assertRaises(ValueError): + project_provider_run(self.source, self.root / "selected", "fixture", case_ids={"pass"}, retain_completed=True) + + +class ProviderAssemblyReviewPolicyTests(unittest.TestCase): + def test_adding_a_provider_preserves_the_declared_review_policy(self): + from harness.web_fetch.assembly import assemble + for required in (False, True): + with self.subTest(required=required), tempfile.TemporaryDirectory() as directory: + root = Path(directory) + case = EvalCase(id="a", name="A", url="https://example.test/a", + category="web", expected_any=["anchor"]) + runs = [] + for provider in ("fixture", "peer"): + dataset = {"tool": "web_fetch", "cases": [asdict(case)], + "comparison": {"require_independent_result_review": required}, + "coverage_plan": {"web": [provider]}} + path = root / (provider + ".json") + path.write_text(json.dumps(dataset)) + config = root / (provider + "-config.json") + config.write_text(json.dumps({"providers": {provider: {"type": "fixture"}}})) + output = root / provider + run_evaluation(output_dir=output, dataset_meta=dataset, dataset_path=path, + cases=[case], providers=[FixtureProvider(provider, {"type": "fixture"})], + provider_config_path=config, runs=1, timeout=1, delay=0) + (output / "sources").mkdir() + (output / "list-references").mkdir() + runs.append(output) + target = root / "assembled" + assemble(runs[0], root / "unused", [], target, provider_additions=[runs[1]]) + from harness.web_fetch.artifacts import verify_record_public_call_logs + from harness.record_io import parse_jsonl + for row in parse_jsonl((target / 'results.jsonl').read_bytes()): + verify_record_public_call_logs(target, row) + saved = json.loads((target / "dataset.json").read_text()) + self.assertEqual(saved["comparison"]["require_independent_result_review"], required) + self.assertEqual(saved["coverage_plan"], {"web": ["fixture", "peer"]}) + + +class CaseAssemblyExecutionBatchTests(unittest.TestCase): + def test_case_addition_retains_every_certified_batch_and_call_log(self): + from harness.web_fetch.assembly import assemble + from harness.record_io import parse_jsonl + from harness.web_fetch.artifacts import verify_record_public_call_logs + with tempfile.TemporaryDirectory() as directory: + root=Path(directory);runs=[] + for cid in ('first','second'): + case=EvalCase(id=cid,name=cid,url='https://example.test/'+cid,category='web',expected_any=['anchor']) + dataset={'tool':'web_fetch','cases':[asdict(case)], + 'comparison':{'require_independent_result_review':False}, + 'coverage_plan':{'web':['fixture','peer']}} + path=root/(cid+'.json');path.write_text(json.dumps(dataset)) + config=root/(cid+'-config.json');config.write_text('{}') + output=root/cid + run_evaluation(output_dir=output,dataset_meta=dataset,dataset_path=path,cases=[case], + providers=[FixtureProvider(n,{'type':'fixture'}) for n in ['fixture','peer']], + provider_config_path=config,runs=1,timeout=1,delay=0) + manifest=json.loads((output/'manifest.json').read_text()) + (output/'execution.json').write_text(json.dumps({'batches':[manifest]})) + (output/'manifest.json').unlink() + (output/'sources').mkdir();(output/'list-references').mkdir();runs.append(output) + target=root/'assembled' + assemble(runs[0],root/'unused',[],target,additions=[runs[1]]) + rows=parse_jsonl((target/'results.jsonl').read_bytes()) + self.assertEqual({(r['case_id'],r['provider']) for r in rows}, + {(cid,p) for cid in ('first','second') for p in ('fixture','peer')}) + self.assertEqual(len(json.loads((target/'execution.json').read_text())['batches']),2) + for row in rows:verify_record_public_call_logs(target,row) diff --git a/tests/test_jsonl_evidence.py b/tests/test_jsonl_evidence.py new file mode 100644 index 000000000..4c59d8fa5 --- /dev/null +++ b/tests/test_jsonl_evidence.py @@ -0,0 +1,18 @@ +import json +import unittest + +from harness.record_io import parse_jsonl + + +class JsonlEvidenceTests(unittest.TestCase): + def test_unicode_content_survives_physical_line_boundaries(self): + rows = [{"text": "正文" + separator + "保留", "index": index} + for index, separator in enumerate(("\u0085", "\u2028", "\u2029", "\v", "\f"))] + raw = "\n".join(json.dumps(row, ensure_ascii=False) for row in rows) + "\n" + for payload in (raw, raw.encode("utf-8"), raw.replace("\n", "\r\n")): + with self.subTest(kind=type(payload).__name__): + self.assertEqual(parse_jsonl(payload), rows) + + def test_malformed_record_is_rejected_without_discarding_it(self): + with self.assertRaises(json.JSONDecodeError): + parse_jsonl('{"case":"a"}\ninvalid\n') diff --git a/tests/test_moli_provider.py b/tests/test_moli_provider.py index 20ad08893..4f5b0f308 100644 --- a/tests/test_moli_provider.py +++ b/tests/test_moli_provider.py @@ -1,6 +1,5 @@ import base64 import hashlib -import json import os import sys import tempfile @@ -147,11 +146,11 @@ def test_committed_nonredirect_post_and_get_preserve_same_document_identity(self self.assertTrue(task_passed(True,metrics,'body')) def test_resource_route_enumerates_bounded_same_origin_frame_documents(self): - from harness.web_fetch.providers.moli import _RESOURCE_PAGE_EXPRESSION + from harness.web_fetch.task_outputs import RESOURCE_PAGE_EXPRESSION - self.assertIn("frame.contentDocument", _RESOURCE_PAGE_EXPRESSION) - self.assertIn("roots.length < 16", _RESOURCE_PAGE_EXPRESSION) - self.assertNotIn("pic_trulli", _RESOURCE_PAGE_EXPRESSION) + self.assertIn("frame.contentDocument", RESOURCE_PAGE_EXPRESSION) + self.assertIn("roots.length < 16", RESOURCE_PAGE_EXPRESSION) + self.assertNotIn("pic_trulli", RESOURCE_PAGE_EXPRESSION) def test_cancellation_between_popen_and_handle_registration_is_latched(self): import subprocess @@ -163,8 +162,9 @@ def test_cancellation_between_popen_and_handle_registration_is_latched(self): def launch(*args, **kwargs): process = original(*args, **kwargs) - children.append(process) - provider.cancel() # Real child exists but fetch has not received its handle. + if args and args[0][0] == str(self.binary): + children.append(process) + provider.cancel() # Real child exists but fetch has not received its handle. return process provider.begin_call() @@ -406,80 +406,49 @@ def test_non_video_body_keeps_ordinary_markdown_route(self): self.assertTrue(result.ok, repr(result)) self.assertEqual(result.metadata["route"], "ordinary_markdown") - def test_challenge_route_uses_one_json_trace_and_converts_final_dom(self): - envelope = { - "final_url": "https://example.com/final", - "status": 200, - "title": "Catalog", - "headers": [{"name": "content-type", "value": "text/html"}], - "redirect_chain": [ - { - "from_url": "https://example.com/start", - "to_url": "https://example.com/final", - "status": 302, - } - ], - "html": '
Real product
', - "network": { - "main_document": {"status": 200, "redirected": True, "redirect_count": 1}, - "subresources": [{"ok": False, "error_text": "blocked"}], - }, - } + def test_popular_site_uses_native_markdown_without_local_conversion(self): provider = self.provider( - "import json,sys\n" - 'assert "--dump" in sys.argv and sys.argv[sys.argv.index("--dump")+1]=="json"\n' - 'assert "--trace-network" in sys.argv and "--eval" not in sys.argv\n' - f"print({json.dumps(envelope)!r})" - ) - result = provider.fetch(self.case_with(track="challenge", task_kind="structure"), 10) + 'import sys\n' + 'assert sys.argv[sys.argv.index("--dump")+1] == "markdown"\n' + 'assert "--trace-network" not in sys.argv and "--eval" not in sys.argv\n' + 'print("# Native catalog\\n\\n[Real product](/item/1)")') + self.assertEqual(provider.validate(), []) + with patch("harness.web_fetch.providers.moli.subprocess.run", side_effect=AssertionError("Local converter must not run")): + result = provider.fetch(self.case_with(track="challenge", task_kind="structure"), 10) self.assertTrue(result.ok, repr(result)) - self.assertIn("[Real product]", result.content) - self.assertEqual(result.status_code, 200) - self.assertEqual(result.final_url, "https://example.com/final") - self.assertEqual(result.title, "Catalog") - self.assertEqual(result.metadata["route"], "challenge_json_trace") - self.assertEqual(result.metadata["network_trace"]["failed_subresource_count"], 1) - self.assertEqual( - result.metadata["network_trace"]["response_header_names"], ["content-type"] - ) - self.assertIn("_raw_stdout_b64", result.metadata) + self.assertEqual(result.content, "# Native catalog\n\n[Real product](/item/1)\n") + self.assertEqual(result.metadata["route"], "ordinary_markdown") + self.assertEqual(result.metadata["output"], "markdown") + self.assertFalse(result.metadata["network_trace_requested"]) - def test_page_identity_uses_same_call_json_for_single_and_multihop_redirects(self): + def test_required_final_url_never_switches_native_markdown_to_json(self): from harness.web_fetch.scoring import score_attempt from harness.web_fetch.metrics import task_passed - for hops in (1, 2): - with self.subTest(hops=hops): + for supplied_url in (None, 'https://example.com/document'): + with self.subTest(final_url=supplied_url): final='https://example.com/document' - envelope={'final_url':final,'status':200,'title':'Article', - 'html':'
Complete required body
', - 'redirect_chain':[{'from_url':'https://example.com/redirect', - 'to_url':final,'status':302}]*hops} + log = (f'DEBUG fetch request complete method=GET url={self.case.url} final_url={final} status=200' if supplied_url else '') provider=self.provider( 'import sys\n' - 'assert sys.argv[sys.argv.index("--dump")+1]=="json"\n' + 'assert sys.argv[sys.argv.index("--dump")+1]=="markdown"\n' 'assert "--trace-network" not in sys.argv\n' - 'print("DEBUG fetch request complete method=GET url=https://cdn.example.com/app.js final_url=https://cdn.example.com/app.js status=200",file=sys.stderr)\n' - f'print({json.dumps(envelope)!r})') + f'print({log!r},file=sys.stderr)\n' + 'print("Complete required body")') case=self.case_with(track='risk_control',task_kind='body', source_benchmark={'risk_control':{'navigation_contract':{'requested_url':self.case.url,'observed_final_url':final,'target_language':'en'}}}) result=provider.fetch(case,10) self.assertTrue(result.ok,repr(result)) - self.assertEqual(result.final_url,final) - self.assertEqual(result.metadata['final_url_source'],'same_call_json') - self.assertEqual(result.metadata['output'],'final_dom_html+local_markdown') - self.assertFalse(result.metadata['network_trace_requested']) + self.assertEqual(result.final_url,supplied_url) + self.assertEqual(result.content.strip(),'Complete required body') + self.assertEqual(result.metadata['route'],'ordinary_markdown') + self.assertEqual(result.metadata['output'],'markdown') config={'id':'article','url':case.url,'track':'risk_control','task_kind':'body', 'source_benchmark':case.source_benchmark, 'acceptance_contract':{'version':1,'schema':'body_text','minimum_records':0, 'stable_text_anchors':[{'id':'body','alternatives':['Complete required body']}]}} metrics=score_attempt(config,result.content.encode(),ok=result.ok, status_code=result.status_code,final_url=result.final_url) - self.assertEqual(metrics['page_identity_status'],'verified') self.assertTrue(task_passed(result.ok,metrics,'body')) - wrong=score_attempt(config,result.content.encode(),ok=True,status_code=200, - final_url='https://cdn.example.com/app.js') - self.assertEqual(wrong['page_identity_status'],'mismatch') - self.assertFalse(task_passed(True,wrong,'body')) def test_legacy_multihop_log_does_not_select_interleaved_resource(self): log=( @@ -493,37 +462,21 @@ def test_legacy_multihop_log_does_not_select_interleaved_resource(self): 'DEBUG published standalone replacement Document page_id=1\n') self.assertIsNone(_diagnostic_main_document_url(log,'https://example.com/start')) - def test_challenge_http_error_keeps_body_but_request_fails(self): - envelope = { - "final_url": "https://example.com/gate", - "status": 403, - "title": "Access denied", - "headers": [], - "redirect_chain": [], - "html": "

Access denied

", - "network": {}, - } - provider = self.provider(f"print({json.dumps(envelope)!r})") + def test_popular_site_native_http_error_keeps_body_but_request_fails(self): + provider = self.provider('print("**Error:** Forbidden\\n\\n**403**: Forbidden")') result = provider.fetch(self.case_with(track="challenge", task_kind="structure"), 10) self.assertFalse(result.ok) self.assertEqual(result.error_type, "api_error") self.assertEqual(result.status_code, 403) - self.assertIn("Access denied", result.content) + self.assertIn("Forbidden", result.content) - def test_page_json_runtime_error_is_not_a_serialization_error(self): - for exit_code in (0, 1): - with self.subTest(exit_code=exit_code): - provider = self.provider( - 'import sys\n' - 'print("SSL connect error: connection reset", file=sys.stderr)\n' - f'sys.exit({exit_code})\n') - result = provider.fetch(self.case_with(track="challenge", task_kind="structure"), 10) - self.assertFalse(result.ok) - self.assertEqual(result.error_type, 'tool_error' if exit_code else 'invalid_output') - if exit_code: - self.assertIn('SSL connect error', result.error_message) - self.assertEqual(result.metadata['exit_code'], exit_code) - self.assertEqual(result.metadata['raw_stdout_sha256'], hashlib.sha256(b'').hexdigest()) + def test_native_markdown_runtime_error_keeps_official_stderr(self): + provider = self.provider('import sys\nprint("SSL connect error: connection reset", file=sys.stderr)\nsys.exit(1)\n') + result = provider.fetch(self.case_with(track="risk_control", task_kind="body", source_benchmark={"risk_control":{"navigation_contract":{"requested_url":self.case.url,"observed_final_url":"https://example.com/final"}}}), 10) + self.assertFalse(result.ok) + self.assertEqual(result.error_type, 'tool_error') + self.assertIn('SSL connect error', result.error_message) + self.assertEqual(result.metadata['route'], 'ordinary_markdown') def test_dynamic_task_without_hints_fails_closed(self): provider = self.provider('raise AssertionError("should not run")') @@ -531,6 +484,19 @@ def test_dynamic_task_without_hints_fails_closed(self): self.assertFalse(result.ok) self.assertEqual(result.error_type, "configuration") + def test_dynamic_native_completion_keeps_native_markdown_route(self): + provider = self.provider( + 'import sys\n' + 'assert "--eval" not in sys.argv and "--wait-selector" not in sys.argv\n' + 'assert sys.argv[sys.argv.index("--dump")+1] == "markdown"\n' + 'print("# Native body")' + ) + result = provider.fetch(self.case_with(track="dynamic", retrieval_hints={ + "version": 3, "mode": "native"}), 10) + self.assertTrue(result.ok, result.error_message) + self.assertEqual(result.content, "# Native body\n") + self.assertEqual(result.metadata["output"], "markdown") + def test_dynamic_task_prefers_predeclared_selector_without_answer_leakage(self): provider = self.provider( "import sys\n" @@ -832,8 +798,7 @@ def test_resource_page_uses_bounded_runtime_url_discovery(self): 'assert "--dump" not in sys.argv\n' 'assert "--eval" in sys.argv\n' 'expression=sys.argv[sys.argv.index("--eval")+1]\n' - 'assert "[src],[href],[data],[poster],[srcset]" in expression\n' - 'assert "astro-island[component-url]" in expression\n' + 'assert "fetch(" not in expression\n' 'assert not ({"--image","--video","--media","--resource"} & set(sys.argv))\n' 'print("{\\"links\\":[\\"https://example.com/movie.mp4\\"]}")' ) @@ -843,316 +808,19 @@ def test_resource_page_uses_bounded_runtime_url_discovery(self): self.assertEqual(result.metadata["output"], "structured_resource_urls") self.assertIn("movie.mp4", result.content) - def test_resource_contract_takes_priority_over_page_family(self): - for track, path in (("challenge", "/page"), ("document", "/document.pdf"), - ("document", "/document.docx"), ("document", "/viewer.html"), - ("video", "/page"), ("image", "/page")): - with self.subTest(track=track, path=path): - provider = self.provider( - 'import sys\n' - 'assert "--eval" in sys.argv and "--dump" not in sys.argv\n' - 'assert "--trace-network" not in sys.argv\n' - 'expression=sys.argv[sys.argv.index("--eval")+1]\n' - 'assert "[src],[href],[data],[poster],[srcset]" in expression\n' - 'print("{\\"links\\":[\\"https://example.com/movie.mp4\\"]}")' - ) - with patch("harness.web_fetch.providers.moli.require_converter", - side_effect=AssertionError("resource discovery must not convert Markdown")): - result = provider.fetch(self.case_with( - track=track, url="https://example.com" + path, task_kind="resource"), 10) - self.assertTrue(result.ok, repr(result)) - self.assertEqual(result.metadata["route"], "resource_page_runtime") - self.assertEqual(result.metadata["output"], "structured_resource_urls") - self.assertFalse(result.metadata["network_trace_requested"]) - self.assertEqual(json.loads(result.content)["links"], ["https://example.com/movie.mp4"]) - - def test_file_contract_never_initializes_page_conversion(self): - for track in ("challenge", "document", "video", "image"): - with self.subTest(track=track): + def test_document_urls_use_native_text_without_external_parsing(self): + for url in ("https://example.com/paper.pdf", "https://example.com/file.docx"): + with self.subTest(url=url): provider = self.provider( - 'import sys\n' - 'assert "--dump" not in sys.argv and "--eval" not in sys.argv\n' - 'assert "--trace-network" not in sys.argv\n' - 'sys.stdout.buffer.write(b"fixture-file")' - ) - with patch("harness.web_fetch.providers.moli.require_converter", - side_effect=AssertionError("file delivery must not convert Markdown")): - result = provider.fetch(self.case_with(track=track, task_kind="file"), 10) + "import sys\n" + 'assert sys.argv[sys.argv.index("--dump")+1] == "markdown"\n' + 'assert "--eval" not in sys.argv\n' + 'print("Native document text")') + result = provider.fetch(self.case_with(url=url, track="document"), 2) self.assertTrue(result.ok, repr(result)) - self.assertEqual(result.files[0]["data"], b"fixture-file") - self.assertEqual(result.metadata["route"], "raw_file") - self.assertEqual(result.metadata["output"], "raw_bytes") - self.assertFalse(result.metadata["network_trace_requested"]) - - def test_dynamic_resource_honors_declared_completion(self): - provider = self.provider( - 'import sys\n' - 'assert sys.argv[sys.argv.index("--wait-selector")+1]=="main article"\n' - 'assert sys.argv.count("--wait-until")==1\n' - 'assert sys.argv[sys.argv.index("--wait-until")+1]=="domcontentloaded"\n' - 'assert sys.argv[sys.argv.index("--delay-ms")+1]=="999"\n' - 'assert sys.argv[sys.argv.index("--timeout")+1]=="3000"\n' - 'assert "[src],[href],[data],[poster],[srcset]" in sys.argv[sys.argv.index("--eval")+1]\n' - 'import json\nprint(json.dumps(dict(links=["https://example.com/image.png"])))' - ) - result = provider.fetch(self.case_with( - track="dynamic", task_kind="resource", retrieval_hints={ - "version": 1, "wait_until": "domcontentloaded", "completion": { - "selector": "main article", "minimum_count": 1, - "timeout_ms": 3000, "delay_ms": 999, - }, - }), 10) - self.assertTrue(result.ok, repr(result)) - self.assertEqual(result.metadata["route"], "resource_page_runtime") - self.assertEqual(result.metadata["wait"]["selector"], "main article") - self.assertEqual(result.metadata["wait"]["completion_support"], "exact") - - def test_dynamic_file_cannot_discard_valid_dom_completion(self): - marker = Path(self.directory.name) / "fetch-executed" - provider = self.provider( - f'from pathlib import Path\nPath({str(marker)!r}).write_text("executed")\n' - 'print("unexpected output")' - ) - result = provider.fetch(self.case_with( - track="dynamic", task_kind="file", retrieval_hints={ - "version": 1, "wait_until": "load", "completion": { - "selector": "main", "minimum_count": 1, - "timeout_ms": 3000, "delay_ms": 0, - }, - }), 10) - self.assertFalse(marker.exists()) - self.assertFalse(result.ok) - self.assertEqual(result.error_type, "configuration") - self.assertIn("DOM completion", result.error_message) - - def test_resource_route_cannot_discard_declared_text_extraction(self): - marker = Path(self.directory.name) / "fetch-executed" - provider = self.provider( - f'from pathlib import Path\nPath({str(marker)!r}).write_text("executed")\n' - 'print("unexpected output")' - ) - result = provider.fetch(self.case_with( - track="dynamic", task_kind="resource", retrieval_hints={ - "version": 2, "wait_until": "load", "completion": { - "selector": "main", "minimum_count": 1, - "timeout_ms": 3000, "delay_ms": 0, - }, - "extraction": {"selector": "article", "maximum_count": 2, - "exclude_selectors": [], "fallback": "empty"}, - }), 10) - self.assertFalse(marker.exists()) - self.assertFalse(result.ok) - self.assertEqual(result.error_type, "configuration") - self.assertIn("Declared text extraction", result.error_message) - - def test_scoring_answers_do_not_change_execution(self): - provider = self.provider('import json, sys\nprint(json.dumps(sys.argv[1:]))') - original = provider.fetch(self.case, 2) - changed = provider.fetch(self.case_with( - id="unrelated-id", name="Unrelated label", expected_any=["secret-answer"], - reference_answer="secret-reference", required_facts=["secret-fact"], - ground_truth_main_content="secret-body", expected_resources=["secret-resource"], - ), 2) - self.assertTrue(original.ok, repr(original)) - self.assertTrue(changed.ok, repr(changed)) - self.assertEqual(original.content, changed.content) - self.assertEqual(original.metadata["route"], changed.metadata["route"]) - - def test_file_task_preserves_binary_stdout(self): - from dataclasses import replace - - provider = self.provider( - "import sys\n" - 'assert "--dump" not in sys.argv\n' - 'assert "--wait-until" not in sys.argv\n' - 'assert sys.argv[1:6]==["fetch","--timeout","2000","--http-timeout","2000"]\n' - 'sys.stdout.buffer.write(b"\\x89PNG\\r\\n\\x1a\\n" + b"x"*40)' - ) - result = provider.fetch(replace(self.case, task_kind="file"), 2) - self.assertTrue(result.ok) - self.assertEqual(result.files[0]["data"], b"\x89PNG\r\n\x1a\n" + b"x" * 40) - self.assertEqual(result.content, "") - self.assertIsNone(result.metadata["dump"]) - self.assertIsNone(result.metadata["wait_until"]) - self.assertIsNone(result.metadata["wait"]) - self.assertEqual(result.metadata["route"], "raw_file") - self.assertEqual(result.metadata["output"], "raw_bytes") - - def test_direct_pdf_body_uses_raw_fetch_and_extracts_inside_call(self): - import io - from dataclasses import replace - - from pypdf import PdfWriter - - output = io.BytesIO() - writer = PdfWriter() - writer.add_blank_page(width=72, height=72) - writer.write(output) - encoded = repr(output.getvalue()) - provider = self.provider( - "import sys\n" - 'assert "--dump" not in sys.argv\n' - 'assert "--wait-until" not in sys.argv\n' - 'assert "--http-timeout" in sys.argv\n' - f"sys.stdout.buffer.write({encoded})" - ) - document = replace( - self.case, - url="https://example.com/fixed.PDF?download=1#page=2", - track="document", - ) - result = provider.fetch(document, 2) - self.assertTrue(result.ok) # The request completed; the blank PDF still fails task scoring. - self.assertIsNone(result.error_type) - self.assertEqual(result.metadata["document_extractor"], "pypdf") - self.assertEqual(result.metadata["dump"], None) - self.assertEqual(result.metadata["route"], "raw_document") - self.assertEqual(result.metadata["output"], "raw_document_bytes+local_text") - self.assertIn("_raw_stdout_b64", result.metadata) - - def test_docx_runtime_fetch_extracts_valid_ooxml_and_discloses_second_request(self): - from dataclasses import replace - - document_bytes = b"PK\x03\x04fixture" - encoded = base64.b64encode(document_bytes).decode("ascii") - provider = self.provider( - "import json,sys\n" - 'assert "--eval" in sys.argv and "--dump" not in sys.argv\n' - 'expression=sys.argv[sys.argv.index("--eval")+1]\n' - 'assert "fetch(location.href" in expression and "arrayBuffer" in expression\n' - f'print(json.dumps({{"kind":"document_base64","source":' - f'"https://example.com/demo.docx","status":200,' - f'"content_type":"application/vnd.openxmlformats-officedocument.' - f'wordprocessingml.document","data":"{encoded}"}}))' - ) - document = replace( - self.case, - url="https://example.com/demo.docx", - track="document", - ) - with patch( - "harness.web_fetch.providers.moli.extract_document_text", - return_value=("DOCX body", "python-docx"), - ): - result = provider.fetch(document, 2) - self.assertTrue(result.ok, repr(result)) - self.assertEqual(result.content, "DOCX body") - self.assertEqual(result.status_code, 200) - self.assertEqual(result.metadata["route"], "document_runtime_fetch") - self.assertEqual(result.metadata["output"], "runtime_document_base64+local_text") - self.assertTrue(result.metadata["document_runtime_second_http_fetch"]) - self.assertEqual(base64.b64decode(result.metadata["_raw_stdout_b64"]), document_bytes) - - def test_docx_runtime_fetch_rejects_non_ooxml_envelope_and_preserves_evidence(self): - from dataclasses import replace - - raw = b'{"kind":"document_base64","data":"bm90LW9veG1s"}' - provider = self.provider(f"import sys\nsys.stdout.buffer.write({raw!r})") - document = replace( - self.case, - url="https://example.com/demo.docx", - track="document", - ) - result = provider.fetch(document, 2) - self.assertFalse(result.ok) - self.assertEqual(result.error_type, "invalid_output") - self.assertEqual(result.content, "") - self.assertEqual(result.metadata["raw_stdout_sha256"], hashlib.sha256(raw).hexdigest()) - self.assertEqual(base64.b64decode(result.metadata["_raw_stdout_b64"]), raw) - - def test_docx_runtime_local_extraction_must_finish_inside_call_budget(self): - import time - - encoded = base64.b64encode(b"PK\x03\x04fixture").decode("ascii") - provider = self.provider( - f'import json\nprint(json.dumps({{"kind":"document_base64","data":"{encoded}"}}))' - ) - document = self.case_with(url="https://example.com/demo.docx", track="document") - - real_clock = time.monotonic - for extraction_seconds, expected_ok in ((0.1, True), (11, False)): - with self.subTest(extraction_seconds=extraction_seconds): - elapsed = [0.0] - - def extract(_raw): - elapsed[0] += extraction_seconds - return "extracted content", "fixture" - - clock = SimpleNamespace(monotonic=lambda: real_clock() + elapsed[0]) - with ( - patch("harness.web_fetch.providers.moli.time", clock), - patch("harness.web_fetch.providers.moli.extract_document_text", - side_effect=extract) as extractor, - ): - result = provider.fetch(document, 10) - extractor.assert_called_once() - self.assertEqual(result.ok, expected_ok) - if expected_ok: - self.assertEqual(result.content, "extracted content") - else: - self.assertEqual(result.error_type, "timeout") - self.assertIn("runtime extraction", result.error_message) - - def test_document_extraction_must_finish_inside_the_call_budget(self): - import time - - provider = self.provider('import sys\nsys.stdout.buffer.write(b"%PDF-fixture")') - document = self.case_with(url="https://example.com/fixed.pdf", track="document") - - def slow_extract(_raw): - time.sleep(3.2) - return "late content", "fixture" - - with patch( - "harness.web_fetch.providers.moli.extract_document_text", - side_effect=slow_extract, - ): - result = provider.fetch(document, 3) - self.assertFalse(result.ok) - self.assertEqual(result.error_type, "timeout") - self.assertIn("extraction", result.error_message) - - def test_pdfjs_viewer_uses_runtime_document_text(self): - provider = self.provider( - "import json,sys\n" - 'assert "--dump" not in sys.argv and "--eval" in sys.argv\n' - 'expression=sys.argv[sys.argv.index("--eval")+1]\n' - 'assert "PDFViewerApplication" in expression and "getTextContent" in expression\n' - 'print(json.dumps({"kind":"text","source":"https://example.com/paper.pdf",' - '"text":"TraceMonkey nested trace trees"}))' - ) - result = provider.fetch( - self.case_with( - url="https://mozilla.github.io/pdf.js/web/viewer.html", track="document" - ), - 2, - ) - self.assertTrue(result.ok, repr(result)) - self.assertEqual(result.content, "TraceMonkey nested trace trees") - self.assertEqual(result.metadata["route"], "document_page_runtime") - self.assertEqual(result.metadata["document_extractor"], "pdfjs_runtime") - - def test_embedded_pdf_page_returns_bytes_for_local_extraction(self): - encoded = base64.b64encode(b"%PDF-fixture").decode("ascii") - provider = self.provider( - "import json,sys\n" - 'assert "--dump" not in sys.argv and "--eval" in sys.argv\n' - 'expression=sys.argv[sys.argv.index("--eval")+1]\n' - 'assert "embed[src],iframe[src],object[data],a[href]" in expression\n' - f'print(json.dumps({{"kind":"pdf_base64","source":"https://example.com/sample.pdf",' - f'"data":"{encoded}"}}))' - ) - with patch( - "harness.web_fetch.providers.moli.extract_document_text", - return_value=("embedded PDF text", "fixture"), - ): - result = provider.fetch( - self.case_with(url="https://pdfobject.com/static/", track="document"), 2 - ) - self.assertTrue(result.ok, repr(result)) - self.assertEqual(result.content, "embedded PDF text") - self.assertEqual(result.metadata["document_extractor"], "fixture") - self.assertEqual(result.metadata["embedded_document_url"], "https://example.com/sample.pdf") + self.assertEqual(result.content.strip(), "Native document text") + self.assertEqual(result.metadata["route"], "ordinary_markdown") + self.assertNotIn("document_extractor", result.metadata) def test_invalid_dynamic_hints_fail_before_process(self): for hints in ( @@ -1195,7 +863,9 @@ def test_failure_binary_and_oversized_output(self): "diagnostic_log_too_large", True), ]: with self.subTest(expected=expected): - result = self.provider(script, **config).fetch(self.case, 2) + provider = self.provider(script, **config) + self.assertEqual(provider.validate(), []) + result = provider.fetch(self.case, 2) self.assertFalse(result.ok) self.assertEqual(result.error_type, expected) self.assertEqual("_diagnostic_log_b64" in result.metadata, retained) @@ -1271,47 +941,77 @@ def test_outer_deadline_preserves_live_cli_log_and_rejects_late_answer(self): self.assertIn(b"DEBUG actual fixture CLI started", raw) self.assertNotIn(b"late answer", raw) self.assertIn(b"native_cli_start", raw) + def test_timeout_stops_a_browser_child_in_an_independent_session(self): + import signal, time + pid_file = Path(self.directory.name) / 'detached-child.pid' + marker = Path(self.directory.name) / 'orphan-work' + child_source = ('import time; from pathlib import Path; time.sleep(2); ' + + 'Path(' + repr(str(marker)) + ').write_text("orphan")') + script = ('import subprocess, time\nfrom pathlib import Path\n' + + 'child = subprocess.Popen([sys.executable, "-c", ' + repr(child_source) + + '], start_new_session=True)\n' + + 'Path(' + repr(str(pid_file)) + ').write_text(str(child.pid))\n' + + 'time.sleep(10)\n') + try: + provider = self.provider(script) + self.assertEqual(provider.validate(), []) + result = provider.fetch(self.case, 1) + self.assertEqual(result.error_type, 'timeout') + self.assertTrue(pid_file.exists(), 'fixture did not start the independent browser child') + time.sleep(2.1) + self.assertFalse(marker.exists(), 'browser child continued physical work after timeout') + finally: + if pid_file.exists(): + try: + os.killpg(int(pid_file.read_text()), signal.SIGKILL) + except ProcessLookupError: + pass def test_timeout_is_failure(self): result = self.provider("import time\ntime.sleep(10)").fetch(self.case, 0.05) self.assertEqual(result.error_type, "timeout") - def test_process_group_permission_error_falls_back_to_direct_kill(self): + def test_cancel_does_not_abandon_descendants_when_group_signalling_is_forbidden(self): process = Mock(pid=123) process.poll.return_value = None - with patch( - "harness.web_fetch.providers.moli.os.killpg", - side_effect=PermissionError("group signal forbidden"), - ): - from harness.web_fetch.providers.moli import MoliWebFetchProvider - - MoliWebFetchProvider._kill(process) - process.kill.assert_called_once_with() + provider = self.provider('print("unused")') + provider._active_process = process + with patch('harness.web_fetch.providers.base.subprocess.run', + return_value=SimpleNamespace(stdout='123 1 123\n124 123 124\n')), patch( + 'harness.web_fetch.providers.base.os.killpg', + side_effect=PermissionError('group signal forbidden')): + provider.cancel() + process.kill.assert_not_called() + self.assertIsNone(process.poll(), 'unresolved physical work must remain visible to the deadline boundary') - def test_cancel_swallows_direct_kill_oserror(self): + def test_cancel_does_not_signal_an_unobservable_process_tree(self): process = Mock(pid=123) process.poll.return_value = None - process.kill.side_effect = OSError("direct signal forbidden") provider = self.provider('print("unused")') provider._active_process = process - with patch( - "harness.web_fetch.providers.moli.os.killpg", - side_effect=PermissionError("group signal forbidden"), - ): + with patch('harness.web_fetch.providers.base.subprocess.run', + side_effect=OSError('process inventory unavailable')), patch( + 'harness.web_fetch.providers.base.os.killpg') as signal_group: provider.cancel() + signal_group.assert_not_called() + process.kill.assert_not_called() - def test_cancel_permission_error_does_not_replace_deadline_timeout(self): + def test_cancel_permission_error_preserves_timeout_and_unresolved_worker(self): from harness.web_fetch.call_boundary import fetch_with_deadline - - provider = self.provider("import time\ntime.sleep(10)") - with patch( - "harness.web_fetch.providers.moli.os.killpg", - side_effect=PermissionError("group signal forbidden"), - ): - result, worker = fetch_with_deadline(provider, self.case, 0.05) - self.assertFalse(result.ok) - self.assertEqual(result.error_type, "timeout") - worker.join(1) + provider = self.provider('import time\ntime.sleep(10)') + worker = None + try: + with patch('harness.web_fetch.providers.base.os.killpg', + side_effect=PermissionError('group signal forbidden')): + result, worker = fetch_with_deadline(provider, self.case, .1) + self.assertFalse(result.ok) + self.assertEqual(result.error_type, 'timeout') + self.assertTrue(worker.is_alive(), 'failed cleanup must not hide remaining physical work') + finally: + provider.cancel() + if worker is not None: + worker.join(2) + self.assertFalse(worker.is_alive()) def test_url_and_config_validation_before_process(self): provider = self.provider('raise AssertionError("should not run")') diff --git a/tests/test_native_browser_variants.py b/tests/test_native_browser_variants.py new file mode 100644 index 000000000..6a25d202e --- /dev/null +++ b/tests/test_native_browser_variants.py @@ -0,0 +1,97 @@ +import unittest +import hashlib +import json +import tempfile +from pathlib import Path + +from harness.web_fetch.task_outputs import require_native_task_routes +from harness.web_fetch.report_interfaces import browser_task_interface_matrix, method_access, method_spec + + +class NativeBrowserVariantTests(unittest.TestCase): + candidate = "moli-pr919" + cases = [{"id": "body", "url": "https://example.com/", "track": "web", "task_kind": "body"}] + + def row(self, metadata, **fields): + return {"provider": self.candidate, "provider_type": "moli_webfetch", "case_id": "body", "ok": True, + "provider_metadata": metadata, **fields} + + def test_branch_build_has_same_native_body_acceptance_as_release(self): + require_native_task_routes(self.cases, [self.row({"route": "ordinary_markdown", "output": "markdown"})]) + with self.assertRaises(ValueError): + require_native_task_routes(self.cases, [self.row({"route": "json_body", "extractor": "node-html-markdown"})]) + + def test_branch_configuration_failure_is_not_browser_capability(self): + with self.assertRaises(ValueError): + require_native_task_routes(self.cases, [self.row({"route": "ordinary_markdown", "output": "markdown"}, + ok=False, error_type="configuration")]) + + def test_unknown_native_participant_cannot_bypass_route_validation(self): + with self.assertRaises(ValueError): + require_native_task_routes(self.cases, [self.row({}, provider="unregistered-browser")]) + + def test_interface_matrix_covers_every_family_for_six_configurations(self): + cases = {family: {"id": family, "url": "https://example.com/file.pdf" if family == "document" else "https://example.com/", + "track": family, "task_kind": "resource" if family in {"image", "video"} else "body"} + for family in ["web", "dynamic", "challenge", "risk_control", "document", "image", "video"]} + providers = ["moli", "lightpanda", "obscura", "chromium", "kitesurf", self.candidate] + matrix = browser_task_interface_matrix(cases, providers) + self.assertEqual(len(matrix["rows"]), 7) + self.assertEqual(sum(len(row["cells"]) for row in matrix["rows"]), 42) + for row in matrix["rows"]: + self.assertEqual(row["cells"]["moli"], row["cells"][self.candidate]) + self.assertEqual(matrix["capabilities"]["moli"], matrix["capabilities"][self.candidate]) + + def test_branch_interface_description_uses_native_moli_contract(self): + self.assertIsNotNone(method_access(self.candidate, [])) + description = method_spec(self.candidate, self.row({"reported_version": "moli 1.1.15"})) + self.assertIn("Moli", description["entry"]) + self.assertNotIn("pypdf", description["notes"]) + + def test_branch_only_report_requires_native_interface_contract(self): + from harness.web_fetch.storage import require_current_report_profile + with tempfile.TemporaryDirectory() as folder: + root = Path(folder) + comparison = {"kind": "headless_markdown", "timeout_seconds": 45, + "runs_per_case": 1, "concurrency": 1, "conversion": None, + "reference_binding_mode": "strict", "publication_state": "verified", + "unverified_reference_cases": [], "require_independent_result_review": False} + profile = {"dataset_id": "branch", "comparison_kind": comparison["kind"], + "case_count": 1, "providers": [self.candidate], + "case_ids_sha256": hashlib.sha256(b'["body"]').hexdigest(), + **{k: v for k, v in comparison.items() if k != "kind"}} + (root / "profile.json").write_text(json.dumps(profile)) + (root / "dataset.json").write_text(json.dumps({"id": "branch", "cases": self.cases, + "coverage_plan": {"web": [self.candidate]}, "comparison": comparison})) + row = self.row({"route": "ordinary_markdown", "output": "markdown"}, attempt_id="branch-body") + (root / "results.jsonl").write_text(json.dumps(row) + "\n") + with self.assertRaisesRegex(ValueError, "requires native_task_route_contract"): + require_current_report_profile(root, root / "profile.json") + profile["native_task_route_contract"] = 4 + (root / "profile.json").write_text(json.dumps(profile)) + require_current_report_profile(root, root / "profile.json") + + def test_interpretation_counts_engines_and_configurations_separately(self): + from harness.web_fetch.report_explainer import _render_current_browser_article, _digest, _headless_article_input + providers = ["moli", "lightpanda", "obscura", "chromium", "kitesurf", self.candidate] + families = ["web", "dynamic", "challenge", "risk_control", "document", "image", "video"] + cases = {f: {"id": f, "name": f, "url": "https://example.com/", "track": f, + "task_kind": "body", "ground_truth_main_content": "Required body"} for f in families} + records = [{"provider": p, "case_id": f, "attempt_id": p + "-" + f, + "ok": False, "metrics": {}, "elapsed_ms": 10, "started_at": "2026-10-10T00:00:00Z", + "artifact_path": "output.txt"} for p in providers for f in families] + with tempfile.TemporaryDirectory() as folder: + out = Path(folder) + (out / "output.txt").write_text("Observed body") + content = {"schema": "webfetch_browser_interpretation/1", "title": "配置对比", + "intro": "同一任务和答案。", "scoring": "按任务验收。", "conclusions": "保留实际结果。", + "evidence_sha256": _digest(_headless_article_input(cases, records, sorted(providers))), + "providers": {p: {"specimen_case_id": "web", "explanation": "未满足要求。"} for p in providers}} + path = out / "article.json" + path.write_text(json.dumps(content)) + body = _render_current_browser_article(out, cases, records, + {"path": "article.json", "sha256": hashlib.sha256(path.read_bytes()).hexdigest()}) + self.assertIn("5 种浏览器、6 种配置", body) + self.assertIn("Moli · PR #919", body) + self.assertNotIn("6 个浏览器", body) + self.assertEqual(body.count('data-browser-interface='), 42) diff --git a/tests/test_native_network_dns.py b/tests/test_native_network_dns.py index de85aba66..03b1ce6c6 100644 --- a/tests/test_native_network_dns.py +++ b/tests/test_native_network_dns.py @@ -84,7 +84,11 @@ def resolver(host, timeout): self.assertEqual(broker.traces, []) -@unittest.skipUnless(platform.system() == "Darwin" and shutil.which("cc"), "Darwin native bridge") +@unittest.skipUnless( + platform.system() == "Darwin" and shutil.which("cc") + and subprocess.run(["xcode-select", "-p"], capture_output=True).returncode == 0, + "Darwin native bridge requires installed developer tools; cc alone can be an installer shim", +) class NativeDnsInteropTests(unittest.TestCase): def test_native_addresses_flags_concurrent_allocations_and_binding(self): source = r""" diff --git a/tests/test_native_task_outputs.py b/tests/test_native_task_outputs.py new file mode 100644 index 000000000..3b6a841e9 --- /dev/null +++ b/tests/test_native_task_outputs.py @@ -0,0 +1,281 @@ +"""Browser-facing controls for shared output scope and forbidden answer repair.""" +import json +import hashlib +import os +import threading +import unittest +from collections import Counter +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer + +from harness.web_fetch.models import EvalCase +from harness.web_fetch.providers.moli import MoliWebFetchProvider +from harness.web_fetch.providers.headless_worker import fetch + + +class NativeTaskOutputTests(unittest.TestCase): + def test_resource_discovery_reads_dom_without_mining_unrendered_modules(self): + engines = {name: os.environ.get(name.upper() + '_BINARY') + for name in ('moli', 'lightpanda', 'obscura', 'chromium')} + engines = {name: binary for name, binary in engines.items() if binary} + if not engines: + self.skipTest('set native browser binary variables for actual controls') + calls = Counter() + page = b'''Resource fixture + + + + + ''' + hidden = 'https://answer-repair.invalid/unrendered.mp4' + class Handler(BaseHTTPRequestHandler): + def log_message(self, *args): pass + def do_GET(self): + calls[self.path] += 1 + body = page if self.path == '/' else ('const unrendered = ' + json.dumps(hidden) + ';').encode() + self.send_response(200) + self.send_header('Content-Type', 'text/html' if self.path == '/' else 'application/javascript') + self.send_header('Content-Length', str(len(body))) + self.end_headers(); self.wfile.write(body) + server = ThreadingHTTPServer(('127.0.0.1', 0), Handler) + thread = threading.Thread(target=server.serve_forever, daemon=True); thread.start() + origin = 'http://127.0.0.1:' + str(server.server_port) + try: + for engine, binary in engines.items(): + with self.subTest(engine=engine): + calls.clear() + if engine == 'moli': + import hashlib, subprocess + provider = MoliWebFetchProvider('moli', {'binary':binary, + 'block_private_networks':False, 'expected_reported_version':subprocess.check_output([binary, '--version'], text=True).strip(), + 'expected_binary_sha256':hashlib.sha256(__import__('pathlib').Path(binary).read_bytes()).hexdigest()}) + case = EvalCase('unseen-resource', 'Fixture', origin + '/', 'image', [], track='image', task_kind='resource') + result = provider.fetch(case, 10) + self.assertTrue(result.ok, result.error_message) + content = result.content + else: + result = fetch({'engine':engine, 'binary':binary, 'url':origin + '/', 'timeout':10, + 'output_mode':'resource', 'task_kind':'resource', 'track':'image', 'allow_private_network':True}) + self.assertTrue(result['ok'], result) + content = result['content'] + links = set(json.loads(content)['links']) + self.assertTrue({origin+'/visible.png', origin+'/small.png', origin+'/large.png', + origin+'/image/a,b.png', origin+'/image/thumb.png?crop=10,20', + origin+'/set/a,b.png', origin+'/set/thumb.png?crop=10,20'} <= links, links) + self.assertNotIn(origin+'/image/a', links) + self.assertNotIn(origin+'/20', links) + self.assertNotIn(hidden, links) + self.assertNotIn(origin+'/visible.mp4', links, 'an image task must not return video sources') + self.assertEqual(calls['/'], 1) + self.assertLessEqual(calls['/module.js'], 1, 'adapter must not refetch scripts to manufacture resource answers') + finally: + server.shutdown(); server.server_close(); thread.join(1) + + def test_native_document_projection_rejects_bytes_and_preserves_native_text(self): + from unittest.mock import patch + from harness.web_fetch.providers.headless_worker import _projection_text + from harness.web_fetch.task_outputs import DOCUMENT_PAGE_EXPRESSION + import time + self.assertNotIn('fetch(', DOCUMENT_PAGE_EXPRESSION) + for kind in ('text', 'page_text'): + with self.subTest(kind=kind): + meta={} + actual=_projection_text(json.dumps({'kind':kind,'text':'Native document content'}), + 'document_page',meta,time.monotonic()+10) + self.assertEqual(actual,'Native document content') + with patch('harness.web_fetch.providers.headless_worker.extract_document_text', + side_effect=AssertionError('external parsing is forbidden')) as parser: + with self.assertRaises(ValueError): + _projection_text(json.dumps({'kind':'pdf_base64','data':'JVBERg=='}), + 'document_page',{},time.monotonic()+10) + parser.assert_not_called() + + +class NativeRoutePublicationTests(unittest.TestCase): + def test_page_routes_prioritize_native_output_and_keep_required_identity(self): + from harness.web_fetch.task_outputs import require_native_task_routes + pages=[{'id':'page','url':'https://example.com/page','track':'challenge','task_kind':'structure'}] + accepted={'moli':{'route':'ordinary_markdown','output':'markdown'}, + 'lightpanda':{'extractor':'lightpanda_native_markdown'}, + 'obscura':{'extractor':'obscura_native_markdown'}, + 'chromium':{'extractor':'node-html-markdown','extractor_version':'2.0.0'}, + 'kitesurf':{'route':'browser_rendering_markdown','extractor':'cloudflare_native_markdown','browser':'kitesurf'}} + for provider,meta in accepted.items(): + row={'provider':provider,'case_id':'page','provider_metadata':meta} + require_native_task_routes(pages,[row]) + with self.subTest(provider=provider),self.assertRaises(ValueError): + require_native_task_routes(pages,[{**row,'provider_metadata':{}}]) + with self.assertRaises(ValueError): + require_native_task_routes(pages,[{'provider':'moli','case_id':'page','provider_metadata':{ + 'route':'challenge_json_trace','output':'final_dom_html+local_markdown+network_diagnostics'}}]) + identity=[{**pages[0],'track':'risk_control','task_kind':'body','source_benchmark':{'risk_control':{'navigation_contract':{ + 'requested_url':pages[0]['url'],'observed_final_url':'https://example.com/final'}}}}] + meta={'route':'page_identity_json','output':'final_dom_html+local_markdown','final_url_source':'same_call_json'} + row={'provider':'moli','case_id':'page','provider_metadata':meta} + with self.assertRaises(ValueError): + require_native_task_routes(identity,[row]) + require_native_task_routes(identity,[{**row,'provider_metadata':accepted['moli']}]) + for provider,meta in [('moli',{'route':'ordinary_markdown','output':'markdown'}), + ('chromium',{'extractor':'node-html-markdown'})]: + case_scope=identity if provider=='moli' else pages + stopped={'provider':provider,'case_id':'page','ok':False,'error_type':'timeout','provider_metadata':meta} + require_native_task_routes(case_scope,[stopped]) + if provider=='chromium': + with self.assertRaises(ValueError): + require_native_task_routes(case_scope,[{**stopped,'ok':True}]) + + def test_configuration_failures_cannot_be_published_as_browser_results(self): + from harness.web_fetch.task_outputs import require_native_task_routes + cases=[{'id':'pdf','url':'https://example.com/file.pdf','track':'document','task_kind':'body'}] + native={'moli':{'route':'ordinary_markdown','output':'markdown'}, + 'lightpanda':{'extractor':'lightpanda_native_markdown'}, + 'obscura':{'extractor':'obscura_native_markdown'}, + 'chromium':{'extractor':'node-html-markdown','extractor_version':'2.0.0'}, + 'kitesurf':{'route':'browser_rendering_markdown','extractor':'cloudflare_native_markdown','browser':'kitesurf'}} + for provider,meta in native.items(): + row={'provider':provider,'case_id':'pdf','provider_metadata':meta,'ok':False} + with self.subTest(provider=provider): + require_native_task_routes(cases,[{**row,'error_type':'api_error'}]) + with self.assertRaisesRegex(ValueError,'Native task route contract violated'): + require_native_task_routes(cases,[{**row,'error_type':'configuration'}]) + + def test_document_text_cannot_be_supplied_by_external_converter(self): + from harness.web_fetch.task_outputs import require_native_task_routes + cases=[{'id':'pdf','url':'https://example.com/file.pdf','track':'document','task_kind':'body'}] + native={'moli':{'route':'ordinary_markdown','output':'markdown'}, + 'lightpanda':{'extractor':'lightpanda_native_markdown'}, + 'obscura':{'extractor':'obscura_native_markdown'}, + 'chromium':{'extractor':'node-html-markdown','extractor_version':'2.0.0'}, + 'kitesurf':{'route':'browser_rendering_markdown','extractor':'cloudflare_native_markdown','browser':'kitesurf'}} + for provider,meta in native.items(): + row={'provider':provider,'case_id':'pdf','provider_metadata':meta,'ok':False} + require_native_task_routes(cases,[row]) + for bad in ({**meta,'document_extractor':'pypdf'}, + {'route':'raw_document','extractor':provider+'_cdp_response+document_text'}, + {'route':'workers_ai_tomarkdown','extractor':'external_conversion'}): + with self.subTest(provider=provider,bad=bad),self.assertRaises(ValueError): + require_native_task_routes(cases,[{**row,'provider_metadata':bad}]) + + def test_kitesurf_resource_backend_must_be_bound_explicitly(self): + from harness.web_fetch.task_outputs import require_native_task_routes + cases=[{'id':'images','url':'https://example.com/','track':'image','task_kind':'resource'}] + from harness.web_fetch.task_outputs import resource_expression + meta={'browser':'kitesurf','endpoint_route':'/browser-run/devtools/browser?browser=kitesurf', + 'output_projection_sha256':hashlib.sha256(resource_expression('image').encode()).hexdigest()} + row={'provider':'kitesurf','case_id':'images','provider_metadata':meta} + require_native_task_routes(cases,[row]) + for field in ('browser','endpoint_route','output_projection_sha256'): + bad={k:v for k,v in meta.items() if k!=field} + with self.subTest(missing=field),self.assertRaises(ValueError): + require_native_task_routes(cases,[{**row,'provider_metadata':bad}]) + + def test_network_timeout_with_only_runner_receipts_is_not_an_adapter_substitution(self): + from harness.web_fetch.task_outputs import require_native_task_routes + cases=[{'id':'images','url':'https://example.com/','track':'image','task_kind':'resource'}] + row={'provider':'kitesurf','case_id':'images','ok':False,'error_type':'timeout', + 'provider_metadata':{'call_log':{'path':'call.json','sha256':'0'*64}, + 'configured_min_interval_seconds':0,'client_throttle_wait_ms':0}} + require_native_task_routes(cases,[row]) + with self.assertRaises(ValueError): + require_native_task_routes(cases,[{**row,'provider_metadata':{**row['provider_metadata'],'browser':'chromium'}}]) + + def test_shared_projection_hash_rejects_one_provider_extra_crawler(self): + import hashlib + from harness.web_fetch.task_outputs import require_native_task_routes, resource_expression + cases=[{'id':'images','url':'https://example.com/','track':'image','task_kind':'resource'}] + digest=hashlib.sha256(resource_expression('image').encode()).hexdigest() + for provider in ('moli','lightpanda','obscura','chromium'): + row={'provider':provider,'case_id':'images','provider_metadata':{'output_projection_sha256':digest}} + require_native_task_routes(cases,[row]) + with self.subTest(provider=provider),self.assertRaises(ValueError): + require_native_task_routes(cases,[{**row,'provider_metadata':{'output_projection_sha256':'0'*64}}]) + + +class AllFamilyInterfaceTests(unittest.TestCase): + def test_reader_matrix_exposes_all_families_and_non_native_processing(self): + from copy import deepcopy + from harness.web_fetch.report_interfaces import browser_task_interface_matrix + providers=['moli','lightpanda','obscura','chromium','kitesurf'] + cases={track:{'id':track,'url':'https://example.test/page','track':track,'task_kind':'body'} + for track in ['web','dynamic','challenge','risk_control','document','image','video']} + cases['document']['url']='https://example.test/file.pdf' + cases['image']['task_kind']=cases['video']['task_kind']='resource' + cases['risk_control']['source_benchmark']={'risk_control':{'navigation_contract':{ + 'requested_url':'https://example.test/page','observed_final_url':'https://example.test/final'}}} + matrix=browser_task_interface_matrix(cases,providers) + self.assertEqual(len(matrix['rows']),7) + self.assertEqual(sum(len(row['cells']) for row in matrix['rows']),35) + rows={row['track']:row for row in matrix['rows']} + for provider in providers: + self.assertIn('不使用外部 PDF/DOCX 解析器',rows['document']['cells'][provider][0]['description']) + self.assertIn('不转 Markdown',rows['image']['cells'][provider][0]['description']) + self.assertTrue(rows['web']['cells'][provider][0]['wait']) + self.assertIn('共同转换',rows['web']['cells']['chromium'][0]['description']) + self.assertIn('内置转换',rows['challenge']['cells']['moli'][0]['description']) + self.assertIn('内置转换',rows['risk_control']['cells']['moli'][0]['description']) + self.assertNotIn('共同转换',rows['risk_control']['cells']['moli'][0]['description']) + changed=deepcopy(cases) + for case in changed.values(): + case.update(ground_truth_main_content='Different hidden answer',expected_resources=[{'url':'https://secret.test/answer'}]) + self.assertEqual(browser_task_interface_matrix(changed,providers),matrix) + + def test_wait_labels_distinguish_page_interfaces_from_document_and_media_interfaces(self): + from harness.web_fetch.report_interfaces import browser_task_interface_matrix + cases={ + 'page':{'url':'https://example.test/page','track':'web','task_kind':'body'}, + 'embedded':{'url':'https://example.test/viewer.html','track':'document','task_kind':'body'}, + 'metadata':{'url':'https://example.test/watch','track':'video','task_kind':'body'}, + 'resources':{'url':'https://example.test/gallery','track':'image','task_kind':'resource'}, + } + providers=['lightpanda','kitesurf','obscura'] + rows={row['track']:row for row in browser_task_interface_matrix(cases,providers)['rows']} + self.assertIn('done',rows['web']['cells']['lightpanda'][0]['wait']) + self.assertIn('networkidle2',rows['web']['cells']['kitesurf'][0]['wait']) + for track in ('document','video'): + for provider in providers: + self.assertTrue(rows[track]['cells'][provider][0]['wait'].startswith('CDP 页面提取等待 load')) + for provider in ('lightpanda','kitesurf'): + self.assertEqual(rows['image']['cells'][provider][0]['wait'],'CDP 页面提取等待 load') + self.assertIn('自适应静默等待',rows['image']['cells']['obscura'][0]['wait']) + + def test_all_thirty_five_combinations_follow_deliverable_without_answer_values(self): + from harness.web_fetch.providers.headless import ENGINES, output_mode_for_case + from harness.web_fetch.task_outputs import resource_detail_links, resource_expression + families = [('web','body','https://example.test/page','markdown'), + ('dynamic','body','https://example.test/app','markdown'), + ('challenge','body','https://example.test/popular','markdown'), + ('risk_control','body','https://example.test/protected','markdown'), + ('document','body','https://example.test/paper.pdf#page=2','markdown'), + ('image','resource','https://example.test/gallery','resource'), + ('video','resource','https://example.test/watch','resource')] + for engine in ENGINES: + for track, kind, url, expected_mode in families: + with self.subTest(engine=engine, track=track): + case = EvalCase('arbitrary-id','Arbitrary',url,track,[],track=track,task_kind=kind) + self.assertEqual(output_mode_for_case(engine,'headless_markdown',case),expected_mode) + for answer in ('https://secret-answer.test/video/42','https://different-answer.test/watch/99'): + case = EvalCase('arbitrary-id','Arbitrary','https://example.test/', 'video',[], + track='video',task_kind='resource',expected_resources=[{'kind':'video-link','url':answer}]) + expression = resource_expression(case.track,resource_detail_links(case)) + self.assertNotIn(answer,expression) + self.assertEqual(expression,resource_expression('video',True)) + +class ResourceAttributeParsingTests(unittest.TestCase): + def test_single_url_commas_and_srcset_candidates_preserve_address_identity(self): + import shutil, subprocess + from harness.web_fetch.task_outputs import resource_expression + node = shutil.which('node') + if not node: + self.skipTest('Node is required by the repository UI test runtime') + attributes = [{'src': '/image/a,b.png'}, {'src': '/image/thumb.png?crop=10,20'}, + {'srcset': '/set/a,b.png 1x, /set/thumb.png?crop=10,20 2x'}, + {'srcset': '/first.png, /second.png 2x'}, {'src': 'data:image/png;base64,AA=='}] + setup = '''const location={href:'https://example.test/page'}; + const nodes=ATTRS.map(attrs=>({baseURI:location.href,getAttribute:name=>attrs[name]||null})); + const document={baseURI:location.href,querySelectorAll:selector=>selector==='iframe,frame'?[]:nodes}; + process.stdout.write(eval(EXPR));''' + code = setup.replace('ATTRS', json.dumps(attributes)).replace('EXPR', json.dumps(resource_expression('image'))) + result = subprocess.run([node, '-e', code], check=True, capture_output=True, text=True) + self.assertEqual(set(json.loads(result.stdout)['links']), { + 'https://example.test/image/a,b.png', 'https://example.test/image/thumb.png?crop=10,20', + 'https://example.test/set/a,b.png', 'https://example.test/set/thumb.png?crop=10,20', + 'https://example.test/first.png', 'https://example.test/second.png'}) diff --git a/tests/test_provider_evidence_boundary.py b/tests/test_provider_evidence_boundary.py index 4406bf68f..f42aac40b 100644 --- a/tests/test_provider_evidence_boundary.py +++ b/tests/test_provider_evidence_boundary.py @@ -200,7 +200,7 @@ def dataset(fresh=False): 'synthetic', ['Expected']) for identifier in ('one', 'two')] if fresh: cases = [replace(case, ground_truth_main_content='Expected fixture', - source_verification={'state': 'valid', 'method': 'independent source capture', + source_verification={'reference_mode': 'live_exact', 'state': 'valid', 'method': 'independent source capture', 'checked_at': datetime.now(timezone.utc).isoformat(), 'reference_scope': 'Complete article', 'reference_sha256': hashlib.sha256(b'Expected fixture').hexdigest()}) for case in cases] diff --git a/tests/test_providers.py b/tests/test_providers.py index 85cc8b7d7..24e91c3f3 100644 --- a/tests/test_providers.py +++ b/tests/test_providers.py @@ -1041,6 +1041,29 @@ def test_cloudflare_image_resource_uses_scrape_and_resolves_srcset( selectors = mock_request.call_args.kwargs["json_body"]["elements"] self.assertEqual(selectors, [{"selector": "img"}, {"selector": "picture source"}]) + @patch("harness.web_fetch.providers.cloudflare.request") + @patch("harness.web_fetch.providers.cloudflare._get_document") + @patch("harness.web_fetch.providers.cloudflare._post_multipart") + def test_kitesurf_cannot_receive_external_download_and_ai_conversion_credit(self, convert, download, native_request): + case = EvalCase("pdf", "PDF", "https://example.com/paper.pdf", "document", [], track="document", task_kind="body") + native_request.return_value = HttpResponse(503, "u", {}, b'{"success":false,"errors":[]}') + result = CloudflareMarkdownProvider("kitesurf", {"api_token":"x", "account_id":"a", "browser":"kitesurf"}).fetch(case, 45) + self.assertFalse(result.ok) + self.assertEqual(result.error_type, "api_error") + native_request.assert_called_once() + self.assertIn("/browser-rendering/markdown?browser=kitesurf", native_request.call_args.args[0]) + download.assert_not_called() + convert.assert_not_called() + + @patch("harness.web_fetch.providers.cloudflare.request") + def test_kitesurf_resource_request_cannot_silently_select_chromium(self, mock_request): + from urllib.parse import urlsplit, parse_qs + mock_request.return_value = HttpResponse(200, "u", {}, b'{"success":true,"result":[]}') + for track in ("image", "video"): + case = EvalCase("resource", "Resource", "https://example.com/", track, [], track=track, task_kind="resource") + CloudflareMarkdownProvider("kitesurf", {"api_token":"x", "account_id":"a", "browser":"kitesurf"}).fetch(case, 45) + self.assertEqual(parse_qs(urlsplit(mock_request.call_args.args[0]).query).get("browser"), ["kitesurf"]) + @patch("harness.web_fetch.providers.cloudflare.request") def test_cloudflare_video_resource_scrape_filters_ordinary_links( self, mock_request @@ -1241,6 +1264,17 @@ def test_cloudflare_failed_document_download_does_not_fallback_or_convert( self.assertEqual(base64.b64decode(result.metadata["_raw_response_b64"]), body) mock_post_multipart.assert_not_called() + @patch("harness.web_fetch.providers.cloudflare.request") + def test_cloudflare_identity_uses_observed_destination_only(self, mock_request): + provider = CloudflareMarkdownProvider("cf", {"api_token": "x", "account_id": "a"}) + for meta, expected in (({}, None), ({"finalUrl": "https://example.com/final"}, + "https://example.com/final")): + with self.subTest(meta=meta): + body = json.dumps({"success": True, "result": "Body", "meta": meta}).encode() + mock_request.return_value = HttpResponse(200, "u", {}, body) + result = provider.fetch(self.case, 45) + self.assertEqual(result.final_url, expected) + @patch("harness.web_fetch.providers.cloudflare.request") def test_cloudflare_empty_target_2xx_succeeds_but_origin_error_fails(self, mock_request) -> None: examples = ( diff --git a/tests/test_publication_privacy.py b/tests/test_publication_privacy.py index dcefff46a..7ac12df10 100644 --- a/tests/test_publication_privacy.py +++ b/tests/test_publication_privacy.py @@ -18,6 +18,35 @@ class PublicationPrivacyTests(unittest.TestCase): + def test_header_at_end_of_json_text_cannot_consume_following_record_fields(self): + record={'body':'RFC example: Authorization: Basic PRIVATE_CANARY', + 'metrics':{'passed':False,'count':7},'case_id':'rfc-example'} + raw=(json.dumps(record)+'\n').encode() + projected, receipt=project_file('results.jsonl',raw) + result=json.loads(projected) + self.assertEqual(result['metrics'],record['metrics']) + self.assertEqual(result['case_id'],record['case_id']) + self.assertNotIn('PRIVATE_CANARY',result['body']) + self.assertIsNone(publication_risk(projected)) + self.assertEqual(project_file('results.jsonl',projected),(projected,None)) + self.assertEqual(receipt['replacements'],1) + + def test_entity_encoded_query_equals_in_rendered_text_and_links_are_projected(self): + for equals in ('=', '=', '='): + with self.subTest(equals=equals): + url='https://example.test/video?category'+equals+'news&x-signature'+equals+'PRIVATE_CANARY' + raw=('

'+url+'

video').encode() + self.assertEqual(publication_risk(raw), 'sensitive URL query value') + projected, receipt=project_file('attempts/case.html',raw) + self.assertNotIn(b'PRIVATE_CANARY', projected) + self.assertIsNone(publication_risk(projected)) + self.assertEqual(receipt['replacements'],2) + soup=BeautifulSoup(projected,'html.parser') + self.assertEqual(soup.a['href'], 'https://example.test/video?category=news&x-signature=[REDACTED]') + self.assertEqual(project_file('attempts/case.html',projected),(projected,None)) + clean=('

https://example.test/video?category'+equals+'news&page'+equals+'2

').encode() + self.assertEqual(project_file('attempts/case.html',clean),(clean,None)) + def test_entity_escaped_fields_inside_json_logs_preserve_json_and_mask_values(self): for quote in ('"','"','"',''',''','''): with self.subTest(quote=quote): @@ -281,6 +310,8 @@ def test_packaged_downloads_and_disclosure(self): '
{"event":"complete"}
') (source / 'attempts/browser__file-case__r001.html').write_text( '

目标文件与实际返回

' + '
DEBUG href="/not-a-report-link.css" '
+                '<link href="/another-diagnostic.css">
' '
{"field":"parsed"}
' '
{"access_token":"canary-private-123"}
' '
{"field":"response"}
') diff --git a/tests/test_real_smoke.py b/tests/test_real_smoke.py index 9324574c4..bfd32507f 100644 --- a/tests/test_real_smoke.py +++ b/tests/test_real_smoke.py @@ -54,7 +54,7 @@ def test_moli_same_call_page_identity(self) -> None: }, }}) result = self.assert_provider_smoke("moli-webfetch", case) - self.assertEqual(result.metadata["route"], "page_identity_json") + self.assertEqual(result.metadata["route"], "ordinary_markdown") self.assertEqual(result.final_url, case.url) def test_lexmount_global_real_fetch(self) -> None: diff --git a/tests/test_reference_inputs.py b/tests/test_reference_inputs.py index 86ec51add..f5df4ae66 100644 --- a/tests/test_reference_inputs.py +++ b/tests/test_reference_inputs.py @@ -168,9 +168,9 @@ def intercept(stage, arguments): assert prepared['cases'] == json.loads(original)['cases'] assert not dataset.exists() assert read_dataset_bytes(dataset) == original - from harness.web_fetch.source_capture import main as collect_sources - assert commands[0][0] is collect_sources - assert commands[0][1][0] == str(work / 'input-dataset.json') + from harness.web_fetch import reference_freeze + assert commands[0][0] is reference_freeze.main + assert str(work / 'input-dataset.json') in commands[0][1] @pytest.mark.parametrize("with_source_tree", [True, False]) diff --git a/tests/test_reference_lifecycle.py b/tests/test_reference_lifecycle.py index efd5db3aa..edc5f759f 100644 --- a/tests/test_reference_lifecycle.py +++ b/tests/test_reference_lifecycle.py @@ -32,12 +32,41 @@ def setUp(self): self.now = datetime(2026, 9, 3, 12, tzinfo=timezone.utc) self.case = EvalCase(id="body", name="Body", url="https://example.test/body", category="static", expected_any=["Price"], ground_truth_main_content="Price 10 dollars", - source_verification={"state": "valid", "method": "independent source capture", + source_verification={"state": "valid", "reference_mode": "live_exact", "method": "independent source capture", "checked_at": (self.now - timedelta(minutes=1)).isoformat(), "reference_sha256": sha256("Price 10 dollars"), "reference_scope": "Complete article"}) self.dataset_path = self.root / "dataset.json" self.run = self.root / "run" + def test_frozen_body_answer_does_not_imply_live_reference_checks(self): + from harness.web_fetch.reference_lifecycle import requires_fresh_reference + verification = {key: value for key, value in self.case.source_verification.items() + if key != 'reference_mode'} + verification['checked_at'] = '2020-01-01T00:00:00Z' + case = replace(self.case, source_verification=verification) + original = deepcopy(case.__dict__) + self.assertFalse(requires_fresh_reference(case)) + self.assertIsNone(self.checks(case)) + self.assertEqual(case.__dict__, original) + self.assertFalse((self.root / 'sources').exists()) + + def test_live_reference_checks_require_an_explicit_mode(self): + from harness.web_fetch.reference_lifecycle import requires_fresh_reference + case = replace(self.case, source_verification={ + **self.case.source_verification, 'reference_mode': 'live_exact'}) + self.assertTrue(requires_fresh_reference(case)) + self.assertIsNotNone(self.checks(case)) + stale = replace(case, source_verification={ + **case.source_verification, 'checked_at': '2020-01-01T00:00:00Z'}) + with self.assertRaises(ReferenceLifecycleError): + self.checks(stale) + + def test_misspelled_reference_mode_is_rejected(self): + case = replace(self.case, source_verification={ + **self.case.source_verification, 'reference_mode': 'live_excat'}) + with self.assertRaisesRegex(ReferenceLifecycleError, 'reference mode'): + self.checks(case) + def write_json(self, path, value): path.parent.mkdir(parents=True, exist_ok=True) path.write_text(json.dumps(value), encoding="utf-8") diff --git a/tests/test_reference_quality.py b/tests/test_reference_quality.py index 52fe43d74..f0c05c1f9 100644 --- a/tests/test_reference_quality.py +++ b/tests/test_reference_quality.py @@ -1,3 +1,5 @@ +import subprocess +import sys import unittest from harness.web_fetch.reference_quality import (audit_reference, clean_reference_text, blocked_page_hits, markdown_link_destinations, markdown_links, reference_anchor_in_text, @@ -7,6 +9,36 @@ class ReferenceQualityTests(unittest.TestCase): + def test_unmatched_markdown_brackets_preserve_body_scoring_within_budget(self): + code = """ +from harness.web_fetch.scoring import score_attempt +case = {'id': 'body', 'url': 'https://example.test/', 'track': 'web', + 'acceptance_contract': {'version': 1, 'schema': 'body_text', + 'minimum_records': 0, 'stable_text_anchors': [ + {'id': 'body', 'alternatives': ['Source prose']}]}} +for brackets in ('[' * 20000, '[x](' * 5000): + for present in (True, False): + text = ('Source prose' if present else 'Unrelated prose') + brackets + result = score_attempt(case, text.encode(), ok=True) + assert result['content_pass'] is present +""" + result = subprocess.run([sys.executable, '-c', code], capture_output=True, + text=True, timeout=5) + self.assertEqual(result.returncode, 0, result.stderr) + + def test_nested_and_escaped_link_delimiters_retain_existing_destinations(self): + samples = [ + (r'\[Docs](/guide)', 'Docs', '/guide'), + (r'[Docs \[with note](/guide)', r'Docs \[with note', '/guide'), + ('[Docs [nested]](/guide(a))', 'Docs [nested]', '/guide(a)'), + (r'[Docs](/guide\))', 'Docs', r'/guide\)'), + ] + for text, label, destination in samples: + with self.subTest(text=text): + self.assertEqual([(link.label, link.destination) + for link in markdown_links(text)], + [(label, destination)]) + def test_script_payload_shell_rejects_state_dump_but_not_readable_js_article(self): state = '\n'.join( f'window.__STATE{i}__ = {{"article":"Required readable sentence {i}",' diff --git a/tests/test_report_body_scope.py b/tests/test_report_body_scope.py index 845c6eae2..cabbe668c 100644 --- a/tests/test_report_body_scope.py +++ b/tests/test_report_body_scope.py @@ -52,6 +52,12 @@ def table_cells(rendered, identifier): class ReportBodyScopeTests(unittest.TestCase): + def test_task_family_names_and_order_share_the_report_registry(self): + from harness.web_fetch.report_explainer import TRACK_NAMES + self.assertIs(TRACK_NAMES, TRACKS) + self.assertEqual(list(TRACKS), ['web','dynamic','challenge','risk_control','document','image','video']) + self.assertEqual(TRACKS['risk_control'], '风控网站正文') + def test_all_call_latency_keeps_failed_calls_and_rejects_partial_matrix(self): c=case('body',kind='body') rows=[record(c,passed=True,elapsed=100),record(c,passed=False,elapsed=900)] @@ -412,7 +418,9 @@ def test_rendered_pages_preserve_results_and_label_only_missing_word_metrics(sel self.assertIn(phrase,detail) self.assertNotIn('正文未评测',detail) self.assertNotIn('评分方式',detail) - self.assertIn('100.0%',(out/'datasets/index.html').read_text()) + self.assertIn('100.0%',(out/'index.html').read_text()) + self.assertNotIn('
',(out/'datasets/index.html').read_text()) + self.assertIn('../analysis/index.html#inspect?query=list',(out/'datasets/list.html').read_text()) self.assertNotIn('正文未评测',(out/'datasets/list.html').read_text()) for page in ['index.html', 'datasets/list.html']: self.assertIn('稳定标题、栏目、说明', (out/page).read_text()) @@ -525,6 +533,19 @@ def test_attempt_call_record_renders_every_retry_log(self): self.assertTrue((out/'public-logs/fixture/retried/run-001-physical-001.stderr.log').is_file()) self.assertTrue((out/'public-logs/fixture/retried/run-001-physical-002.stderr.log').is_file()) + def test_failure_log_exposes_complete_runtime_error_and_actual_method_identity(self): + r=record(case());r.update(ok=False,error_type='timeout',error_message='Full runtime error '+('details '*100), + provider_metadata={'browser_version':'155.0.8059.39','binary_sha256':'actual-hash'}) + with tempfile.TemporaryDirectory() as directory: + rendered=run_log_section(Path(directory),r,'') + self.assertIn(r['error_message'],rendered) + self.assertIn('

程序错误

',rendered) + self.assertIn('timeout',rendered) + self.assertIn('155.0.8059.39',rendered) + self.assertIn('actual-hash',rendered) + self.assertNotIn('Moli 版本',rendered) + self.assertNotIn('Markdown 输出',rendered) + def test_run_log_section_uses_supplied_output_when_artifact_path_is_absent(self): c=case('unpersisted','body','web');r=record(c);r.pop('artifact_path') with tempfile.TemporaryDirectory() as directory: @@ -816,6 +837,31 @@ def test_stable_failure_categories_preserve_diagnosis_and_filters(self): self.assertIn((label,label),failure_filters([r],{'list':c})[1][2]) self.assertIn('data-failure-category="'+label+'"',attempt_row(r,c)[0]) + def test_fixed_contract_rules_survive_absent_historical_attachment(self): + from harness.web_fetch.report_components import stable_contract_details + c=case();c['acceptance_contract']={'schema':'named_link','minimum_records':3, + 'hosts':['example.com'],'entry_path':'/items/.*', + 'stable_text_anchors':[{'id':'heading','alternatives':['Fixed heading']}]} + c['source_benchmark']['stable_annotation']={'anchor_evidence':'sources/early.txt'} + with tempfile.TemporaryDirectory() as directory: + root=Path(directory) + rendered=stable_contract_details(c,evidence_root=root) + self.assertIn('Fixed heading',rendered) + self.assertNotIn('href="../sources/early.txt"',rendered) + (root/'sources').mkdir();(root/'sources/early.txt').write_text('Fixed heading') + self.assertIn('href="../sources/early.txt"',stable_contract_details(c,evidence_root=root)) + + def test_body_coverage_explanation_uses_the_declared_unit_and_region_rules(self): + c=case(kind='body',track='web');c['body_requirements']={ + 'unit':'region_compact_hexagram_recall','minimum_coverage':.8, + 'minimum_floor':.6,'minimum_region_passes':4, + 'regions':[{'id':str(i),'text':'reference'} for i in range(5)]} + rendered=task_pass_guide('web',cases={c['id']:c}) + self.assertIn('字符六元组',rendered) + self.assertIn('通过区域数',rendered) + self.assertNotIn('词语计数覆盖',rendered) + self.assertNotIn('逐区达到 80%',rendered) + def test_schema_fields_are_explained_without_internal_version_or_empty_sections(self): from harness.web_fetch.report_components import stable_contract_details, FIELD_REQUIREMENTS from harness.web_fetch.stable_tasks import SCHEMA_FIELDS, UNIQUE_FIELDS diff --git a/tests/test_report_capabilities.py b/tests/test_report_capabilities.py index 91af0038c..7c1d0165e 100644 --- a/tests/test_report_capabilities.py +++ b/tests/test_report_capabilities.py @@ -9,6 +9,26 @@ class CapabilityTests(unittest.TestCase): + def test_native_cohort_discloses_native_document_output_without_external_processing(self): + from harness.web_fetch.report_capabilities import task_capabilities + with tempfile.TemporaryDirectory() as directory: + out=Path(directory) + (out/'dataset.json').write_text(json.dumps({'cases':[ + {'id':'body','url':'https://example.com/','track':'web','task_kind':'body'}, + {'id':'pdf','url':'https://example.com/a.pdf','track':'document','task_kind':'body'}]})) + names=['moli','lightpanda','obscura','chromium','kitesurf'] + claims=task_capabilities(out,names,[{'provider_metadata':{'output_contract':'readable_document_text'}}]) + for name in names: + self.assertEqual(claims[name]['document']['status'],'原生输出') + self.assertIn('资源读取脚本由评测提供',claims[name]['notes']) + self.assertIn('不使用外部 PDF/DOCX 解析器',claims[name]['document']['detail']) + self.assertNotIn('没有 PDF / Office 解析器',claims[name]['document']['detail']) + self.assertIn('--eval',claims['obscura']['scope']) + self.assertIn('JSON',claims['moli']['scope']) + self.assertIn('browser=kitesurf',claims['kitesurf']['scope']) + self.assertEqual(CAPABILITIES['lightpanda']['document']['status'],'未声明') + self.assertIsNone(task_capabilities(out,['firecrawl'],[{'provider_metadata':{'output_contract':'markdown'}}])) + def test_source_backed_capabilities_link_to_current_pinned_revisions(self): expected = { 'claw': '12c4bdf7482f24c85b011e9d858fdec3c245101c', diff --git a/tests/test_report_changes.py b/tests/test_report_changes.py index 8d6b940f8..423d43643 100644 --- a/tests/test_report_changes.py +++ b/tests/test_report_changes.py @@ -7,6 +7,23 @@ from harness.web_fetch.report import overview_card, ranking class ReportChangesTests(unittest.TestCase): + def test_unpublished_corrections_are_not_public_history_or_revision_bindings(self): + from harness.web_fetch.report_changes import select_cohort_history, require_revision_release_bindings + public={'id':'public','publication_state':'published','comparison_kinds':['headless_markdown'],'scoring_revision_input_sha256':'published'} + draft={'id':'draft','comparison_kinds':['headless_markdown'],'publication_state':'unpublished','scoring_revision_input_sha256':'draft'} + data={'schema_version':3,'releases':[draft,public],'review_rounds':[{'id':'review','description':'Synthetic review'}]} + with patch('harness.web_fetch.report_changes.require_review_rounds',return_value=[]): + selected=select_cohort_history(data,{'comparison':{'kind':'headless_markdown'}},[]) + self.assertEqual(selected['releases'],[public]) + unmarked={k:v for k,v in public.items() if k!='publication_state'} + with patch('harness.web_fetch.report_changes.require_review_rounds',return_value=[]), self.assertRaisesRegex(ValueError,'publication state'): + select_cohort_history({**data,'releases':[unmarked]}, {'comparison':{'kind':'headless_markdown'}}, []) + require_revision_release_bindings([{'input_results_sha256':'published'}],[draft,public]) + with self.assertRaises(ValueError): + require_revision_release_bindings([{'input_results_sha256':'draft'}],[draft,public]) + with self.assertRaisesRegex(ValueError,'publication state'): + require_revision_release_bindings([{'input_results_sha256':'published'}],[{**public,'publication_state':'typo'}]) + def test_webmainbench_history_is_localized_from_one_record(self): from harness.web_fetch.report_changes import ROOT source = json.loads((ROOT / 'research/report-changes.json').read_text(encoding='utf-8')) diff --git a/tests/test_report_inventory.py b/tests/test_report_inventory.py index d9a61196f..27abdcc1d 100644 --- a/tests/test_report_inventory.py +++ b/tests/test_report_inventory.py @@ -35,7 +35,7 @@ def check(self): root/'dataset.json': json.dumps(self.dataset), root/'results.jsonl': '\n'.join(json.dumps(r, ensure_ascii=False) for r in self.records), root/'execution.json': json.dumps({'batches': [{'attempt_ids': [r['attempt_id'] for r in self.records]}]}), - root/'index.html': '

6 类任务概览

本类最高： 判分完整性

综合通过率总榜

6 类分榜

', + root/'index.html': '

综合通过率总榜

6 类任务概览

本类最高： 判分完整性

6 类分榜

', root/'methodology/index.html': '计入耗时 不计入耗时 供应商上报的服务端耗时只作附加证据 形成可送入统一评分器的最终答案', ROOT/'configs/evaluation-inventory.json': json.dumps(self.inventory), } diff --git a/tests/test_report_links.py b/tests/test_report_links.py index be9a0c541..64c6ab138 100644 --- a/tests/test_report_links.py +++ b/tests/test_report_links.py @@ -42,6 +42,18 @@ def test_rejects_missing_fragment(self) -> None: self.assertEqual(result.returncode, 1) self.assertIn("missing fragment", result.stdout) + def test_inspection_links_use_the_declared_filter_contract(self): + for attribute in ('data-filter','data-wmb-filter'): + with self.subTest(attribute=attribute), tempfile.TemporaryDirectory() as directory: + root=Path(directory) + (root/'index.html').write_text('结果') + (root/'analysis.html').write_text('') + self.assertEqual(self.run_check(root).returncode,0) + (root/'index.html').write_text('结果') + self.assertEqual(self.run_check(root).returncode,1) + (root/'index.html').write_text('结果') + self.assertEqual(self.run_check(root).returncode,1) + def test_honors_document_base(self) -> None: with tempfile.TemporaryDirectory() as directory: root = Path(directory) diff --git a/tests/test_report_presentation.py b/tests/test_report_presentation.py index da7737ff7..6fb420e38 100644 --- a/tests/test_report_presentation.py +++ b/tests/test_report_presentation.py @@ -1,4 +1,5 @@ import tempfile +import json from pathlib import Path import unittest from harness.web_fetch.report import EXTRA,active_tracks,failure_category,filter_controls,case_attrs,failure_filters,overall_ranking,overview_matched_latency,provider_configuration,review_cell,table,write @@ -30,6 +31,14 @@ def test_overview_latency_includes_every_configuration_and_same_case_mean(self): self.assertIn('调用矩阵尚不完整',overview_matched_latency(records[1:],cases)) def row(self,**kw):return dict({'ok':True,'metrics':{'content_pass':False,'length_pass':True}},**kw) + def test_provider_filter_defaults_to_the_complete_population(self): + controls=filter_controls([('provider','方法',[('one','One'),('two','Two')])]) + self.assertIn('data-filter="provider" aria-label="方法">',controls) + def test_case_search_includes_identifier_and_source_url(self): + c={'id':'case-a','name':'Readable title','url':'https://example.com/source','category':'test','task_kind':'body'} + r=self.row(provider='one',case_id='case-a') + attrs=case_attrs(r,c) + self.assertIn('data-search="case-a Readable title https://example.com/source ',attrs) def test_result_default_failure(self): controls=filter_controls([('result','结果',[('pass','任务通过'),('fail','任务失败')])]) self.assertIn('value="fail" selected',controls) @@ -159,7 +168,11 @@ def test_provider_configuration_is_method_specific(self): chromium=provider_configuration('chromium',[{'provider_type':'headless_markdown','provider_metadata':{'extractor':'node-html-markdown','output_contract':'markdown'}}],default) self.assertIn('rawBase64',firecrawl) self.assertNotIn('Exa、Tavily、Firecrawl',builtin) - self.assertIn('Chromium 渲染 DOM',chromium) + self.assertIn('渲染 DOM',chromium) + self.assertIn('node-html-markdown 2.0.0',chromium) + moli=provider_configuration('moli',[{'provider_type':'headless_markdown','provider_metadata':{'extractor':'node-html-markdown','output_contract':'markdown'}}],default) + self.assertNotIn('Chromium',moli) + self.assertIn('node-html-markdown 2.0.0',moli) def test_dynamic_cohort_copy_has_english_translations(self): from harness.web_fetch.report_i18n import translate_text self.assertEqual(translate_text('90 个案例 · 12 种方法 · 4类分榜与一个综合通过率总榜。'), @@ -280,3 +293,35 @@ def test_article_embedded_assets_and_measured_markup_stay_distinct(self): self.assertIn('
文章目录
',translated) outside=localize_html(markup,PurePosixPath('attempts/example.html')) self.assertIn('src="data:image/png;base64,YQ==" data-en-src="data:image/png;base64,Yg=="',outside) + +class MeasuredInterfaceIdentityTests(unittest.TestCase): + def render(self, records, name='moli'): + import json + from harness.web_fetch.report_interfaces import render_interfaces + with tempfile.TemporaryDirectory() as directory: + out=Path(directory);(out/'adoption').mkdir();(out/'answer.md').write_text('Body') + rows=[{'case_id':'static-simple','attempt_id':str(i),'artifact_path':'answer.md',**r} for i,r in enumerate(records)] + render_interfaces(out,rows) + return json.loads((out/'adoption/interfaces'/f'{name}.json').read_text()) + def test_generic_moli_uses_observed_release_and_task_adapter(self): + record={'provider':'moli','provider_type':'moli_webfetch','provider_metadata':{'reported_version':'moli 1.1.15'}} + output=self.render([record]);access=output['access'] + self.assertIn('/tag/v1.1.15',access['links'][0][1]) + self.assertNotIn('1.1.5',str(access)) + self.assertEqual(output['interface']['source'],'harness/web_fetch/providers/moli.py') + old={'provider':'moli-1.0.6','provider_type':'headless_markdown','provider_metadata':{'reported_version':'moli 1.0.6'}} + output=self.render([old],'moli-1.0.6') + self.assertIn('1.0.6',output['access']['links'][0][1]) + self.assertEqual(output['interface'],json.loads(json.dumps(SPECS['moli-1.0.6']))) + def test_unknown_or_mixed_generic_versions_do_not_invent_a_release(self): + rows=[{'provider':'moli','provider_metadata':{'reported_version':'moli '+v}} for v in ('1.1.14','1.1.15')] + for records in ([],rows): + self.assertNotIn('/tag/',str(self.render(records)['access'])) + def test_native_metadata_envelopes_keep_native_output_and_actual_cli_examples(self): + engines=[('lightpanda','lightpanda_json','--json'),('obscura','obscura_eval','--eval')] + for name,format_name,flag in engines: + record={'provider':name,'provider_type':'headless_markdown','provider_metadata':{'native_metadata':{'format':format_name,'script_sha256':'a'*64,'script_path':'/private/local-script.js'}}} + output=self.render([record],name) + self.assertIn(flag,output['interface']['entry']) + self.assertNotIn('/private/local-script.js',str(output)) + if name=='obscura':self.assertNotIn('--dump',str(output['interface']['request'])) diff --git a/tests/test_report_presentation_contract.py b/tests/test_report_presentation_contract.py index 1c6cb71ff..d3ff4208a 100644 --- a/tests/test_report_presentation_contract.py +++ b/tests/test_report_presentation_contract.py @@ -24,3 +24,17 @@ def test_evidence_and_negative_controls(self): check_presentation(page) for broken in [page.replace('id="result-logs"','id="missing"'),page.replace('data-kind="error"',''),page.replace('
','
flattened raw log
')]: with self.assertRaises(ValueError):check_presentation(broken) + + +class SharedRankingPresentationTests(unittest.TestCase): + def test_raw_column_extremes_zero_ties_and_unavailable_values(self): + from harness.web_fetch.report import cell, comparative_rows, table + rows=[('',cell('A')+cell('0 ms',0)+cell('50%',0.5)), + ('',cell('B')+cell('10 ms',10)+cell('50%',0.5)), + ('',cell('C')+cell('—',-1)+cell('—',-1))] + rendered=table(['Method','Mean','Pass rate'],comparative_rows(rows,{1:('mean','min'),2:('pass','max')})) + check_presentation(rendered) + self.assertEqual(rendered.count('comparison-winner'),1) + self.assertIn('comparison-winner good">0 ms',rendered) + with self.assertRaises(ValueError): + check_presentation(rendered.replace('comparison-winner good','good')) diff --git a/tests/test_report_ui.mjs b/tests/test_report_ui.mjs index 16065c4d8..ba414fdc1 100644 --- a/tests/test_report_ui.mjs +++ b/tests/test_report_ui.mjs @@ -265,3 +265,21 @@ vm.runInContext('wmbCompare()',wmbCompareContext); assert.deepEqual(wmbCompareRows.map(r=>r.hidden),[true,false,true]); assert.equal(wmbCompareChecks[0].disabled,true); console.log('PASS: WebMainBench same-page group comparison combines methods, delivery pattern and official categories'); + +// A task deep link must show both pass and fail results across providers, +// even when the inventory was previously narrowed to one provider's failures. +const taskFilters=[choices('provider',['one','two']),choices('result',['fail','pass']), + {tagName:'INPUT',dataset:{filter:'query'},value:''}]; +taskFilters.slice(0,2).forEach(f=>f.tagName='SELECT'); +const taskRows=[ + {dataset:{provider:'one',result:'fail',search:'case-a'},textContent:'First result'}, + {dataset:{provider:'two',result:'pass',search:'case-a'},textContent:'Second result'}, + {dataset:{provider:'two',result:'fail',search:'case-b'},textContent:'Other task'}]; +const taskContext=vm.createContext({URLSearchParams,window:{addEventListener(){}},document:{ + addEventListener(){},querySelectorAll(s){return s==='#rows tr'?taskRows:s==='select[data-filter]'?taskFilters.slice(0,2):s==='[data-filter]'?taskFilters:[]}, + querySelector(s){return s==='#count'?count:{scrollIntoView(){}}}}}); +vm.runInContext(js,taskContext); +assert.equal(vm.runInContext("wmbInspect('inspect?query=case-a')",taskContext),true); +assert.deepEqual(taskFilters.map(f=>f.value),['','','case-a']); +assert.deepEqual(taskRows.map(r=>r.hidden),[false,false,true]); +console.log('PASS: same-task deep link clears unrelated filters and includes every outcome'); diff --git a/tests/test_resource_build_context.py b/tests/test_resource_build_context.py new file mode 100644 index 000000000..ee5b958c6 --- /dev/null +++ b/tests/test_resource_build_context.py @@ -0,0 +1,31 @@ +"""The resource image receives runtime code without evaluation credentials.""" +from pathlib import Path +import tarfile +import tempfile +import unittest + +from docker.utils.build import tar + +ROOT = Path(__file__).resolve().parents[1] + + +class ResourceBuildContextTests(unittest.TestCase): + def test_build_context_excludes_credentials_and_host_dependencies(self): + excluded = {'.env', '.env.local', 'nested/.env', 'nested/.env.production', + 'node_modules/example/package.json', 'nested/node_modules/example/package.json'} + required = {'pyproject.toml', 'uv.lock', 'package.json', 'package-lock.json', + 'harness/provider.py', 'scripts/worker.py', + 'examples/web_fetch/docker/resource.Dockerfile'} + patterns = [line.strip() for line in (ROOT / '.dockerignore').read_text().splitlines() + if line.strip() and not line.lstrip().startswith('#')] + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + for name in excluded | required: + path = root / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text('fixture') + with tar(str(root), exclude=patterns) as context: + with tarfile.open(fileobj=context) as archive: + names = {item.name.removeprefix('./') for item in archive.getmembers() if item.isfile()} + self.assertFalse(names & excluded, names & excluded) + self.assertTrue(required <= names, required - names) diff --git a/tests/test_retrieval_hints.py b/tests/test_retrieval_hints.py index c405328be..0be45fd6b 100644 --- a/tests/test_retrieval_hints.py +++ b/tests/test_retrieval_hints.py @@ -1,4 +1,5 @@ import json +import tempfile from copy import deepcopy from pathlib import Path @@ -10,21 +11,30 @@ def test_native_execution_keeps_dynamic_scenario_and_acceptance_without_injected_hints(): from copy import deepcopy - from harness.web_fetch.retrieval_hints import native_fetch_dataset + from harness.web_fetch.models import EvalCase - from harness.web_fetch.retrieval_hints import plan_for_case + from harness.web_fetch.retrieval_hints import native_fetch_dataset, plan_for_case dataset={'tool':'web_fetch','coverage_plan':{'dynamic':['browser']},'cases':[ {'id':'native-page','name':'Dynamic site','url':'https://example.com/', 'category':'dynamic_content','track':'dynamic','expected_any':[], 'task_kind':'body','acceptance_contract':{'version':1,'schema':'body_text'}, 'min_content_chars':1000}]} - original=deepcopy(dataset);selected=native_fetch_dataset(dataset) + original = deepcopy(dataset) + selected = native_fetch_dataset(dataset) assert dataset==original + assert selected['cases'][0]['track']=='dynamic' + assert selected['coverage_plan']==original['coverage_plan'] assert selected['cases'][0]['category']=='dynamic_content' assert selected['cases'][0]['source_benchmark']['scenario_track']=='dynamic' assert selected['cases'][0]['acceptance_contract']==original['cases'][0]['acceptance_contract'] assert selected['cases'][0]['min_content_chars']==1000 assert plan_for_case(EvalCase(**selected['cases'][0])) is None + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / 'dataset.json' + path.write_text(json.dumps(selected)) + _, cases = load_dataset(path) + assert cases[0].track == 'dynamic' + assert plan_for_case(cases[0]) is None dataset['cases'][0]['retrieval_hints']={'version':1,'completion':{'selector':'.record'}} with pytest.raises(ValueError,match='cannot discard'): native_fetch_dataset(dataset) @@ -250,3 +260,36 @@ def test_native_default_contract_must_be_valid(tmp_path): case["acceptance_contract"] = {"version": 1, "schema": "unknown"} with pytest.raises(ValueError): load_dataset(_write_dataset(tmp_path, case)) +@pytest.mark.parametrize("hints", [ + {"version": 3, "mode": "native", "completion": {"selector": ".answer"}}, + {"version": 3, "mode": "native", "answer_text": "expected answer"}, + {"version": 3, "mode": "guided"}, + {"version": 3}, +]) +def test_native_completion_cannot_carry_answer_or_selector_hints(hints): + with pytest.raises(ValueError, match="native retrieval_hints"): + parse_retrieval_hints("dynamic", hints) + + +def test_native_contract_does_not_waive_missing_dynamic_execution_contract(): + assert parse_retrieval_hints("dynamic", {"version": 3, "mode": "native"}) is None + with pytest.raises(ValueError, match="require retrieval_hints"): + parse_retrieval_hints("dynamic", None) + + +@pytest.mark.parametrize("version", [True, 3.0, "3", None]) +def test_native_completion_version_requires_an_integer(version): + with pytest.raises(ValueError): + parse_retrieval_hints("dynamic", {"version": version, "mode": "native"}) + + +def test_worker_accepts_explicit_native_dynamic_contract_without_injected_plan(): + from harness.web_fetch.providers.headless_worker import require_normalized_retrieval_plan + assert require_normalized_retrieval_plan({ + "track": "dynamic", "retrieval_plan": None, + "retrieval_hints": {"version": 3, "mode": "native"}}) is None + with pytest.raises(ValueError, match="explicit native contract"): + require_normalized_retrieval_plan({"track": "dynamic", "retrieval_plan": None}) + with pytest.raises(ValueError, match="native completion"): + require_normalized_retrieval_plan({"track": "dynamic", "retrieval_plan": {"selector": "body"}, + "retrieval_hints": {"version": 3, "mode": "native"}}) diff --git a/tests/test_risk_control_dataset.py b/tests/test_risk_control_dataset.py index 5e7c4c4b0..f10fb66b6 100644 --- a/tests/test_risk_control_dataset.py +++ b/tests/test_risk_control_dataset.py @@ -13,7 +13,6 @@ from harness.web_fetch.risk_control import main as run_risk_control from harness.web_fetch.risk_control import ( measured_risk_control_dataset, - reference_proof_snapshot, require_complete_diagnostic_logs, ) from harness.web_fetch.risk_control_dataset import ( @@ -354,32 +353,24 @@ def test_direct_formal_execution_is_rejected_before_provider_validation(tmp_path run_risk_control(["--execute", "--output", str(tmp_path / "run")]) -def test_reference_proof_snapshot_requires_exact_unchanged_100_file_scope(tmp_path): - roots = {"proofs": tmp_path} - cases = [] - for index in range(100): - identifier = f"case-{index:03d}" - path = tmp_path / f"{identifier}.txt" - path.write_text(f"evidence {index}\n") - cases.append( - { - "id": identifier, - "source_benchmark": { - "risk_control": { - "reference_proof": { - "root": "proofs", - "path": path.name, - "sha256": hashlib.sha256(path.read_bytes()).hexdigest(), - } - } - }, - } - ) - before = reference_proof_snapshot(cases, roots) - assert before["case_count"] == 100 - (tmp_path / "case-042.txt").write_text("changed\n") - with pytest.raises(ValueError, match="Changed reference proof: case-042"): - reference_proof_snapshot(cases, roots) +def test_frozen_execution_inputs_do_not_require_historical_source_files(tmp_path): + from harness.web_fetch.risk_control import frozen_risk_control_input_snapshot + profile = json.loads(RUN_PROFILE.read_text()) + profile_path = tmp_path / "evaluation/risk-control/run-profile.json" + profile_path.parent.mkdir(parents=True) + profile_path.write_bytes(RUN_PROFILE.read_bytes()) + bindings = [profile["dataset"], *profile["source_bindings"].values(), + {"path": profile["provider"]["config_path"]}] + for binding in bindings: + target = tmp_path / binding["path"] + target.parent.mkdir(parents=True, exist_ok=True) + target.write_bytes(read_frozen_dataset_bytes(ROOT / binding["path"]) if binding == profile['dataset'] else (ROOT / binding["path"]).read_bytes()) + # There is no qualification data home or browser.txt tree in this fixture. + snapshot = frozen_risk_control_input_snapshot(profile_path) + assert snapshot["case_count"] == 100 + (tmp_path / profile["dataset"]["path"]).write_text('{"cases": []}') + with pytest.raises(ValueError, match="Changed frozen input"): + frozen_risk_control_input_snapshot(profile_path) def certified_report_fixture(tmp_path): @@ -1035,3 +1026,30 @@ def test_certified_report_preserves_certificate_bound_files(tmp_path, capsys): "reference-proof-verification.json", "certification.json"): assert (report / name).read_bytes() == (tmp_path / name).read_bytes() require_risk_control_certification(report, measured, records) + + +def test_new_risk_certification_uses_frozen_inputs_and_preserves_old_receipts(tmp_path): + from harness.web_fetch.risk_control import frozen_risk_control_input_snapshot + dataset, records, receipt, manifest, execution = certified_report_fixture(tmp_path) + snapshot = frozen_risk_control_input_snapshot(RUN_PROFILE) + verification = {"schema_version": 1, "kind": "risk_control_frozen_input_verification", + "before": snapshot, "after": snapshot, "unchanged": True} + path = tmp_path / "frozen-input-verification.json" + path.write_text(json.dumps(verification)) + digest = hashlib.sha256(path.read_bytes()).hexdigest() + manifest.pop("reference_proof_verification_sha256") + manifest["frozen_input_verification_sha256"] = digest + receipt.pop("reference_proof_verification_sha256") + receipt.update(kind="certified_risk_control_100_moli_v2", + frozen_input_verification_sha256=digest) + (tmp_path / "reference-proof-verification.json").unlink() + rewrite_certified_report_fixture(tmp_path, records, receipt, manifest, execution) + require_risk_control_certification(tmp_path, dataset, records) + verification["after"] = {**snapshot, "case_count": 99} + path.write_text(json.dumps(verification)) + digest = hashlib.sha256(path.read_bytes()).hexdigest() + manifest["frozen_input_verification_sha256"] = digest + receipt["frozen_input_verification_sha256"] = digest + rewrite_certified_report_fixture(tmp_path, records, receipt, manifest, execution) + with pytest.raises(ValueError, match="frozen input verification"): + require_risk_control_certification(tmp_path, dataset, records) diff --git a/tests/test_risk_control_report.py b/tests/test_risk_control_report.py index 237ba6499..c7bd04308 100644 --- a/tests/test_risk_control_report.py +++ b/tests/test_risk_control_report.py @@ -57,7 +57,7 @@ def test_full_risk_control_matrix_builds_existing_interactive_report(tmp_path, c guide = (report / 'guide/index.html').read_text() assert '本报告包含 100 个任务' in guide assert '静态网站正文：0' not in guide - assert "历史访问受限网址" in home + assert "风控网站正文" in home assert "风控网站内容" not in home assert "资格只解释样本来源" in home assert "组合规则结果" in detail diff --git a/tests/test_run_certified_benchmark.py b/tests/test_run_certified_benchmark.py index 2ae604127..9ddaec6a3 100644 --- a/tests/test_run_certified_benchmark.py +++ b/tests/test_run_certified_benchmark.py @@ -76,11 +76,11 @@ def execute(stage, arguments): self.assertFalse((root/'formal').exists()) saved=json.loads((output/'dataset.json').read_text()) self.assertEqual(saved['comparison']['evidence_state'],'diagnostic') - self.assertEqual(saved['comparison']['unqualified_reference_cases'],['stable']) + self.assertEqual(saved['comparison']['unqualified_reference_cases'],[]) batches=json.loads((output/'execution.json').read_text())['batches'] self.assertTrue(all(b['parameters']['max_failure_retries']==retries for b in batches)) self.assertTrue(all(b['evidence_state']=='diagnostic' and - b['unqualified_reference_cases']==['stable'] for b in batches)) + b['unqualified_reference_cases']==[] for b in batches)) # Dropping dataset display metadata cannot erase execution provenance. saved['comparison']={} with self.assertRaisesRegex(ReferenceLifecycleError,'[Dd]iagnostic'): @@ -90,25 +90,66 @@ def execute(stage, arguments): self.assertFalse((root/'formal-report').exists()) self.assertEqual(dataset.read_bytes(),original) - def test_stable_rules_alone_cannot_be_mistaken_for_live_source_certification(self): + def test_frozen_contract_execution_needs_no_historical_qualification_files(self): + from harness.web_fetch import reference_freeze, benchmark_runner + from harness.web_fetch.evidence import finalize_aggregate + from harness.web_fetch.reference_lifecycle import require_verified_reference_checks with tempfile.TemporaryDirectory() as directory: - root=Path(directory);dataset=root/'dataset.json' - dataset.write_text(json.dumps({'cases':[{'id':'stable','acceptance_contract':{ - 'version':1,'schema':'body_text','minimum_records':0, - 'stable_text_anchors':[{'id':'a','alternatives':['Required label']} ]}}]})) - arguments=['--dataset',str(dataset),'--output',str(root/'run'), - '--work-root',str(root/'work'),'--providers','p','--case-ids','stable'] - with patch('harness.web_fetch.certification.run') as invoked: - with self.assertRaises(SystemExit):main(arguments) - invoked.assert_not_called() - def completed(stage, arguments): - command = [stage, *arguments] - script=stage_name(command) - if script=='source_capture.main':self.write_capture(Path(command[2]),'stable') - if script=='benchmark_runner.main':self.write_output_dataset(command,dataset) - with patch('harness.web_fetch.certification.run',side_effect=completed): - main([*arguments,'--diagnostic']) - self.assertEqual(json.loads(dataset.read_text())['cases'][0]['acceptance_contract']['stable_text_anchors'][0]['alternatives'],['Required label']) + root = Path(directory) + dataset = root / 'dataset.json' + output = root / 'run' + case = {'id': 'stable', 'name': 'Stable page', + 'url': 'https://example.test/page', 'category': 'web', + 'track': 'web', 'expected_any': ['Required label'], + 'source_benchmark': {'risk_control': {'reference_proof': { + 'root': 'historical', 'path': 'missing/browser.txt', + 'sha256': 'a' * 64}}}, + 'acceptance_contract': {'version': 1, 'schema': 'body_text', + 'minimum_records': 0, 'stable_text_anchors': [ + {'id': 'a', 'alternatives': ['Required label']}]}} + import hashlib + fixed = {'id': 'fixed-answer', 'name': 'Fixed answer', + 'url': 'https://example.test/fixed', 'category': 'web', 'track': 'web', + 'expected_any': ['Required label'], 'ground_truth_main_content': 'Required label', + 'source_verification': {'state': 'valid', 'checked_at': '2020-01-01T00:00:00Z', + 'method': 'Independent source capture', + 'reference_sha256': hashlib.sha256(b'Required label').hexdigest()}} + dataset.write_text(json.dumps({'tool': 'web_fetch', 'comparison': {}, 'cases': [case, fixed]})) + original = dataset.read_bytes() + config = root / 'providers.json' + config.write_text(json.dumps({'providers': {'fixture': {'type': 'fixture'}}})) + (root / 'evaluation/list-references').mkdir(parents=True) + stages = [] + + def execute(stage, arguments): + script = stage.__module__.rsplit('.', 1)[-1] + '.' + stage.__name__ + stages.append(script) + if script == 'reference_freeze.main': + reference_freeze.main(arguments) + elif script == 'benchmark_runner.main': + with patch.object(benchmark_runner, 'load_runtime_env'): + benchmark_runner.main(arguments) + elif script == 'reference_freeze.postcheck_main': + self.fail('Frozen stable contracts must not trigger live-source recapture') + elif script == 'evidence.finalize_main': + finalize_aggregate(output, root / 'work/post-captures/observations.json') + else: + self.fail('Unexpected external work: ' + script) + + with patch('harness.web_fetch.certification.run', side_effect=execute): + main(['--dataset', str(dataset), '--output', str(output), + '--work-root', str(root / 'work'), '--providers', 'fixture', + '--provider-config', str(config), '--case-ids', 'stable,fixed-answer']) + measured = json.loads((output / 'dataset.json').read_text()) + self.assertEqual(measured['cases'], [case, fixed]) + self.assertEqual(dataset.read_bytes(), original) + self.assertEqual(measured['comparison']['source_reference_policy'], + 'frozen_benchmark_inputs_with_explicit_live_exact_checks') + batches = json.loads((output / 'execution.json').read_text())['batches'] + records = [json.loads(line) for line in (output / 'results.jsonl').read_text().splitlines()] + self.assertEqual(len(records), 2) + self.assertNotIn('collect_source_evidence.py', stages) + require_verified_reference_checks(batches, cases=measured, records=records, source=output) def test_dispatches_complete_webmainbench_certification(self) -> None: arguments = ["--output-root", "/tmp/run", "--source", "/tmp/source.jsonl"] @@ -159,7 +200,7 @@ def test_builds_mandatory_lifecycle_in_order(self) -> None: dataset.write_text(json.dumps({"cases": [{ "id": "javascript-spa", "track": "web", "task_kind": "body", "ground_truth_main_content": "reference", - "source_verification": { + "source_verification": {"reference_mode": "live_exact", "method": "Independent visible browser DOM; complete declared scope" }, }]})) @@ -205,7 +246,7 @@ def test_requires_explicit_browser_for_rendered_case(self) -> None: dataset.write_text(json.dumps({"cases": [{ "id": "javascript-spa", "track": "web", "task_kind": "body", "ground_truth_main_content": "reference", - "source_verification": { + "source_verification": {"reference_mode": "live_exact", "method": "Independent visible browser DOM; complete declared scope" }, }]})) @@ -256,7 +297,7 @@ def test_http_capture_failure_uses_browser_for_any_case_id(self) -> None: dataset.write_text(json.dumps({"cases": [{ "id": "previously-unknown-browser-fallback", "track": "web", "task_kind": "body", "ground_truth_main_content": "reference", - "source_verification": { + "source_verification": {"reference_mode": "live_exact", "method": "Independent HTTP response and reviewed article selector", "selector": "main", }, @@ -310,7 +351,7 @@ def test_post_http_failure_adds_browser_fallback_not_selected_pre_call(self) -> dataset.write_text(json.dumps({"cases": [{ "id": case_id, "track": "web", "task_kind": "body", "ground_truth_main_content": "reference", - "source_verification": { + "source_verification": {"reference_mode": "live_exact", "method": "Independent HTTP response and reviewed article selector", "selector": "main", }, @@ -358,7 +399,7 @@ def test_http_capture_failure_without_browser_stops_before_provider_calls(self) dataset.write_text(json.dumps({"cases": [{ "id": "http-unavailable", "track": "web", "task_kind": "body", "ground_truth_main_content": "reference", - "source_verification": { + "source_verification": {"reference_mode": "live_exact", "method": "Independent HTTP response and reviewed article selector" }, }]})) @@ -388,7 +429,7 @@ def test_failed_pre_browser_observation_stops_before_freeze_and_provider_calls(s "id": case_id, "url": "https://example.com/", "track": "web", "task_kind": "body", "ground_truth_main_content": "reference", - "source_verification": { + "source_verification": {"reference_mode": "live_exact", "method": "Independent rendered browser DOM; complete declared scope" }, }]})) @@ -429,7 +470,7 @@ def test_failed_post_browser_observation_cannot_reach_lifecycle_finalization(sel "id": case_id, "url": "https://example.com/", "track": "web", "task_kind": "body", "ground_truth_main_content": "reference", - "source_verification": { + "source_verification": {"reference_mode": "live_exact", "method": "Independent rendered browser DOM; complete declared scope" }, }]})) @@ -476,7 +517,7 @@ def test_explicit_browser_reference_route_is_used_even_with_usable_http_capture( "task_kind": "body", "ground_truth_main_content": "Required fact", "required_facts": [{"alternatives": ["Required fact"]}], - "source_verification": {"observation_route": "browser"}, + "source_verification": {"reference_mode": "live_exact", "observation_route": "browser"}, } (tmp_path / case["id"]).mkdir() (tmp_path / case["id"] / "reference.txt").write_text("Required fact") @@ -494,6 +535,7 @@ def test_conflicting_http_route_and_browser_provenance_is_rejected(tmp_path): "ground_truth_main_content": "Required fact", "required_facts": [{"alternatives": ["Required fact"]}], "source_verification": { + "reference_mode": "live_exact", "observation_route": "http", "method": "Independent rendered browser DOM; complete declared scope", }, @@ -514,7 +556,7 @@ def test_explicit_empty_arguments_do_not_dispatch_an_ambient_subcommand(monkeypa def test_invalid_selected_reference_route_stops_before_any_capture(tmp_path): dataset = tmp_path / "dataset.json" - dataset.write_text(json.dumps({"cases": [{"id": "one", "track": "web", "task_kind": "body", "ground_truth_main_content": "Required fact", "source_verification": {"observation_route": "http", "method": "Independent rendered browser DOM; complete declared scope"}}]})) + dataset.write_text(json.dumps({"cases": [{"id": "one", "track": "web", "task_kind": "body", "ground_truth_main_content": "Required fact", "source_verification": {"reference_mode": "live_exact", "observation_route": "http", "method": "Independent rendered browser DOM; complete declared scope"}}]})) with patch("harness.web_fetch.certification.run") as invoked: with pytest.raises(ValueError, match="contradicts browser reference provenance"): main(["--dataset", str(dataset), "--case-ids", "one", "--providers", "p", "--output", str(tmp_path / "output"), "--work-root", str(tmp_path / "work")]) diff --git a/tests/test_scoring.py b/tests/test_scoring.py index 2e778660f..2b00f63bd 100644 --- a/tests/test_scoring.py +++ b/tests/test_scoring.py @@ -63,7 +63,7 @@ def test_untagged_source_requires_explicitly_observed_empty_language(self): final_url="https://example.test/page") self.assertEqual(observed_tagged["page_identity_status"], "mismatch") - def test_risk_control_requires_declared_page_identity(self): + def test_risk_control_acceptance_uses_content_not_final_url(self): navigation = { "risk_control": { "navigation_contract": { @@ -85,14 +85,14 @@ def test_risk_control_requires_declared_page_identity(self): source_benchmark=navigation, ) wrong = score_attempt(c, b"Expected", ok=True, final_url="https://other.test/") - self.assertTrue(wrong["page_identity_required"]) + self.assertFalse(wrong["page_identity_required"]) self.assertEqual(wrong["page_identity_status"], "mismatch") self.assertTrue(task_measured(wrong)) - self.assertFalse(task_passed(True, wrong)) + self.assertTrue(task_passed(True, wrong)) missing = score_attempt(c, b"Expected", ok=True) self.assertEqual(missing["page_identity_status"], "unavailable") - self.assertFalse(task_measured(missing)) - self.assertFalse(task_passed(True, missing)) + self.assertTrue(task_measured(missing)) + self.assertTrue(task_passed(True, missing)) failed = score_attempt(c, b"", ok=False) self.assertEqual(failed["page_identity_status"], "unavailable") self.assertTrue(task_measured(failed)) @@ -191,6 +191,23 @@ def test_cloudflare_empty_success_is_request_success_but_not_task_success(self): self.assertFalse(task_passed(result.ok, measured)) self.assertEqual(measured["content_check_coverage"], 0) + def test_kitesurf_document_urls_reach_native_markdown_without_ai_conversion(self): + for url in ('https://example.test/paper.pdf', 'https://example.test/file.docx', + 'https://example.test/plain.txt'): + with self.subTest(url=url): + provider=CloudflareMarkdownProvider('kitesurf', {'account_id':'account', + 'api_token':'token','browser':'kitesurf'}) + with patch('harness.web_fetch.providers.cloudflare.request', return_value=HttpResponse( + 200,'https://example.test/api',{},b'{"success":true,"result":"Native document text"}')) as request, \ + patch.object(provider,'_fetch_direct_document',side_effect=AssertionError('external AI conversion forbidden')): + result=provider.fetch(case(url=url,track='document'),1) + self.assertTrue(result.ok,repr(result)) + self.assertEqual(result.content,'Native document text') + request.assert_called_once() + endpoint=request.call_args.args[0] + self.assertIn('/browser-rendering/markdown?browser=kitesurf',endpoint) + self.assertNotIn('/ai/',endpoint) + def test_dataclass_and_dict_inputs_produce_same_metrics_without_mutation(self): c = case(required_facts=[{"id": "a", "alternatives": ["Alpha"]}]) mapping = asdict(c) diff --git a/tests/test_scoring_regressions.py b/tests/test_scoring_regressions.py index e816ae3d7..947346953 100644 --- a/tests/test_scoring_regressions.py +++ b/tests/test_scoring_regressions.py @@ -4,7 +4,7 @@ import unittest from pathlib import Path -from harness.web_fetch.metrics import task_passed +from harness.web_fetch.metrics import task_passed, task_measured from harness.web_fetch.reference_lifecycle import ( reference_revision_cases, scoring_revision_history_sha256, @@ -12,14 +12,65 @@ ) from harness.web_fetch.scoring import score_attempt from harness.web_fetch.scoring_regressions import ( - load_scoring_regressions, - require_scoring_regressions, + active_scoring_regressions, require_scoring_regressions, valid_correction_provenance, ) from harness.web_fetch.scoring_revision import correction_provenance class ScoringRegressionTests(unittest.TestCase): + def test_policy_supersession_preserves_inputs_and_rejects_changed_body(self): + document=json.loads((Path(__file__).resolve().parents[1]/'evaluation/scoring-regressions.json').read_text()) + self.assertTrue(document['superseded_cases']) + with tempfile.TemporaryDirectory() as directory: + path=Path(directory)/'controls.json' + path.write_text(json.dumps(document)) + current=active_scoring_regressions(path) + old,new=next(iter(document['superseded_cases'].items())) + self.assertNotIn(old,current) + self.assertIn(new,current) + for field,value in (('artifact','Changed body'),('final_url','https://different.test/')): + changed=json.loads(json.dumps(document)) + next(row for row in changed['cases'] if row['id']==new)[field]=value + path.write_text(json.dumps(changed)) + with self.subTest(field=field),self.assertRaises(ValueError): + active_scoring_regressions(path) + changed=json.loads(json.dumps(document)) + changed['superseded_cases'][old]='nonexistent-control' + path.write_text(json.dumps(changed)) + with self.assertRaises(ValueError): + active_scoring_regressions(path) + + def test_report_prose_rebinding_preserves_scoring_history_and_rejects_task_edits(self): + from harness.web_fetch.scoring_revision import rebind_report_interpretation + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + raw = b'{"cases": [{"id": "a", "url": "https://example.test"}]}\n' + (root/'dataset.json').write_bytes(raw) + (root/'results.jsonl').write_bytes(b'') + receipt = {'kind': 'audited_scoring_rule_revision', 'provider_calls': 0, + 'changed_cases': [], 'task_decision_changes': [], + 'input_dataset_sha256': 'a'*64, 'input_results_sha256': 'b'*64, + 'output_dataset_sha256': hashlib.sha256(raw).hexdigest(), + 'output_results_sha256': hashlib.sha256(b'').hexdigest()} + history = json.dumps([receipt]).encode() + (root/'scoring-revisions.json').write_bytes(history) + (root/'article.json').write_bytes(b'{}') + rebind_report_interpretation(root, {'path': 'article.json', + 'sha256': hashlib.sha256(b'{}').hexdigest()}) + self.assertTrue(valid_scoring_revision(root)) + self.assertEqual((root/'scoring-revisions.json').read_bytes(), history) + self.assertEqual((root/'results.jsonl').read_bytes(), b'') + dataset = json.loads((root/'dataset.json').read_text()) + dataset['cases'][0]['url'] = 'https://different.test' + changed = json.dumps(dataset).encode() + revisions = json.loads((root/'report-binding-revisions.json').read_text()) + revisions[0]['output_dataset_sha256'] = hashlib.sha256(changed).hexdigest() + (root/revisions[0]['output_dataset_path']).write_bytes(changed) + (root/'dataset.json').write_bytes(changed) + (root/'report-binding-revisions.json').write_text(json.dumps(revisions)) + self.assertFalse(valid_scoring_revision(root)) + def test_historical_blind_review_keeps_revision_chain_readable_after_new_anomaly_rule(self): with tempfile.TemporaryDirectory() as directory: root=Path(directory) @@ -58,7 +109,7 @@ def test_historical_blind_review_keeps_revision_chain_readable_after_new_anomaly self.assertTrue(valid_scoring_revision(root)) def test_every_registered_false_verdict_control_runs_through_shared_scorer(self): - rows = load_scoring_regressions() + rows = active_scoring_regressions() root_modes = {} for identifier, row in rows.items(): with self.subTest(regression=identifier): @@ -71,6 +122,8 @@ def test_every_registered_false_verdict_control_runs_through_shared_scorer(self) row["ok"], metrics, row["case"].get("task_kind", "body") ) self.assertEqual(verdict, row["expected_task_pass"]) + if "expected_task_measured" in row: + self.assertEqual(task_measured(metrics), row["expected_task_measured"]) for key, expected in row.get("expected_metrics", {}).items(): self.assertEqual(metrics.get(key), expected, key) root_modes.setdefault(row["root_cause"], set()).add(row["failure_mode"]) diff --git a/tests/test_storage.py b/tests/test_storage.py index 1858a4ba2..e39ac5628 100644 --- a/tests/test_storage.py +++ b/tests/test_storage.py @@ -233,3 +233,103 @@ def test_current_report_profile_requires_bound_isolated_resource_cohort(self): if __name__ == "__main__": unittest.main() + +class FrozenProfileInputsTests(unittest.TestCase): + def test_report_answers_must_match_independently_bound_pre_call_input(self): + from harness.web_fetch.storage import frozen_input_bindings, require_frozen_case_source + with tempfile.TemporaryDirectory() as temp: + root=Path(temp);source=root/'pre-call.json';profile_path=root/'profile.json' + original={'cases':[{'id':'a','url':'https://example.com','ground_truth_main_content':'Fixed answer', + 'source_verification':{'reference_sha256':hashlib.sha256(b'Earlier capture').hexdigest()}}]} + source.write_text(json.dumps(original)) + current={**original,'comparison':{'report_interpretation':{'path':'report-assets/article.json'}}} + (root/'dataset.json').write_text(json.dumps(current)) + profile={'source_native_dataset_sha256':hashlib.sha256(source.read_bytes()).hexdigest(), + 'frozen_inputs':frozen_input_bindings(root)} + profile_path.write_text(json.dumps(profile)) + require_frozen_case_source(root,profile_path,source) + source.write_text(json.dumps({**original,'cases':[{'id':'a','ground_truth_main_content':'Changed answer'}]})) + with self.assertRaisesRegex(ValueError,'source binding'): + require_frozen_case_source(root,profile_path,source) + source.write_text(json.dumps(original)) + current['cases']=[{**original['cases'][0],'ground_truth_main_content':'Changed answer'}] + (root/'dataset.json').write_text(json.dumps(current)) + with self.assertRaisesRegex(ValueError,'frozen input bindings'): + require_frozen_case_source(root,profile_path,source) + # Rebinding a generated report still cannot bless an answer change. + profile['frozen_inputs']=frozen_input_bindings(root) + profile_path.write_text(json.dumps(profile)) + with self.assertRaisesRegex(ValueError,'answers differ from pre-call'): + require_frozen_case_source(root,profile_path,source) + current['cases']=original['cases'];(root/'dataset.json').write_text(json.dumps(current)) + profile['frozen_inputs']=frozen_input_bindings(root);profile.pop('source_native_dataset_sha256') + profile_path.write_text(json.dumps(profile)) + with self.assertRaisesRegex(ValueError,'both source and report input bindings'): + require_frozen_case_source(root,profile_path,source) + + def test_registered_native_browser_cannot_omit_interface_gate(self): + with tempfile.TemporaryDirectory() as folder: + root=Path(folder) + dataset={'id':'native','cases':[{'id':'a','url':'https://example.test/paper.pdf','track':'document'}], + 'coverage_plan':{'document':['moli']},'comparison':{'kind':'headless_markdown', + 'timeout_seconds':45,'runs_per_case':1,'concurrency':1,'conversion':'native', + 'reference_binding_mode':'frozen_inputs','publication_state':'verified', + 'unverified_reference_cases':[],'require_independent_result_review':False}} + (root/'dataset.json').write_text(json.dumps(dataset)) + row={'attempt_id':'moli__a','provider':'moli','case_id':'a', + 'provider_metadata':{'route':'ordinary_markdown','output':'markdown'}} + (root/'results.jsonl').write_text(json.dumps(row)+'\n') + profile={'dataset_id':'native','comparison_kind':'headless_markdown','case_count':1, + 'case_ids_sha256':hashlib.sha256(b'["a"]').hexdigest(),'providers':['moli'], + **{k:v for k,v in dataset['comparison'].items() if k!='kind'}} + path=root/'profile.json';path.write_text(json.dumps(profile)) + with self.assertRaisesRegex(ValueError,'requires native_task_route_contract'): + require_current_report_profile(root,path) + profile['native_task_route_contract']=4 + path.write_text(json.dumps(profile));require_current_report_profile(root,path) + row['provider_metadata']['document_extractor']='pypdf' + (root/'results.jsonl').write_text(json.dumps(row)+'\n') + with self.assertRaisesRegex(ValueError,'Native task route contract violated'): + require_current_report_profile(root,path) + + def test_composed_dataset_binds_all_current_inputs_without_raw_qualification_files(self): + from harness.web_fetch.storage import frozen_input_bindings, require_current_report_profile + with tempfile.TemporaryDirectory() as temp: + root=Path(temp) + dataset={'id':'composed','cases':[{'id':'a','track':'web'}], + 'coverage_plan':{'web':['p']}, + 'comparison':{'kind':'headless_markdown','timeout_seconds':45,'runs_per_case':1, + 'concurrency':1,'conversion':'native','reference_binding_mode':'frozen_inputs', + 'publication_state':'verified','unverified_reference_cases':[], + 'require_independent_result_review':False}} + (root/'dataset.json').write_text(json.dumps(dataset)) + (root/'results.jsonl').write_text(json.dumps({'attempt_id':'p__a','provider':'p','case_id':'a','content':'First\u0085middle\u2028last\u2029paragraph'}, ensure_ascii=False)+'\n') + profile={'dataset_id':'composed','comparison_kind':'headless_markdown','case_count':1, + 'case_ids_sha256':hashlib.sha256(b'["a"]').hexdigest(),'providers':['p'], + **{k:v for k,v in dataset['comparison'].items() if k!='kind'}, + 'frozen_inputs':frozen_input_bindings(root),'case_inventory':{'web':['a']}} + path=root/'profile.json';path.write_text(json.dumps(profile)) + require_current_report_profile(root,path) + profile['case_inventory']={'web':['wrong-case']} + path.write_text(json.dumps(profile)) + with self.assertRaisesRegex(ValueError,'case_inventory'): + require_current_report_profile(root,path) + profile['case_inventory']={'web':['a']} + path.write_text(json.dumps(profile)) + (root/'sources').mkdir();(root/'sources/reference.txt').write_text('Changed current input') + with self.assertRaisesRegex(ValueError,'frozen input bindings'): + require_current_report_profile(root,path) + (root/'sources/reference.txt').unlink() + dataset['cases'][0]['url']='https://different.example/' + (root/'dataset.json').write_text(json.dumps(dataset)) + with self.assertRaisesRegex(ValueError,'frozen input bindings'): + require_current_report_profile(root,path) + + def test_input_binding_rejects_symlinked_assets(self): + from harness.web_fetch.storage import frozen_input_bindings + with tempfile.TemporaryDirectory() as temp: + root=Path(temp);(root/'dataset.json').write_text('{}') + (root/'outside.txt').write_text('body');(root/'sources').mkdir() + (root/'sources/link.txt').symlink_to(root/'outside.txt') + with self.assertRaisesRegex(ValueError,'symlinks'): + frozen_input_bindings(root) From c0ee2aeadfc26b21d5ec5f4245432e48445671c2 Mon Sep 17 00:00:00 2001 From: lanyue-llk <270302213+lanyue-llk@users.noreply.github.com> Date: Sat, 10 Oct 2026 16:47:12 +0800 Subject: [PATCH 002/172] Run frozen resource targets without repeating source qualification Separate resource admission from execution of hash-bound fixed benchmark inputs. Keep source qualification mandatory for new or changed resource tasks, explicit live references and invalid sources. Exercise the public certified entry point with frozen body and resource answers whose historical raw files are absent, preserve the original task bytes, and retain strict admission controls. --- harness/web_fetch/README.md | 12 ++++++++++-- harness/web_fetch/reference_lifecycle.py | 18 ++++++++++++++---- harness/web_fetch/report_check.py | 2 +- tests/test_reference_lifecycle.py | 15 +++++++++++++++ tests/test_run_certified_benchmark.py | 18 ++++++++++++++---- 5 files changed, 54 insertions(+), 11 deletions(-) diff --git a/harness/web_fetch/README.md b/harness/web_fetch/README.md index 92c1ea201..56645ce06 100644 --- a/harness/web_fetch/README.md +++ b/harness/web_fetch/README.md @@ -141,8 +141,16 @@ rule bytes without recovering historical qualification files or rechecking admission. Source qualification is required when adding tasks or changing references or acceptance rules. Live exact-reference cases retain their pre/post reference checks. Use `--diagnostic` only when a diagnostic run is intended; its -outputs cannot be upgraded to certified evidence. For declared native completion, -add `--retrieval-mode native`: it preserves acceptance rules and rejects existing +outputs cannot be upgraded to certified evidence. + +This applies to resource targets as well as text answers. Execution and report +checks use the frozen resource target list without reopening historical source +qualification files. Adding or changing a resource task still requires independent +source qualification; an invalid source state or an explicitly live reference +cannot take the frozen-input route. + +For declared native completion, add `--retrieval-mode native`: it preserves +acceptance rules and rejects existing selector-guided retrieval hints rather than dropping them. Dynamic task families stay `dynamic`: the explicit `retrieval_hints: {"version": 3, "mode": "native"}` contract selects each diff --git a/harness/web_fetch/reference_lifecycle.py b/harness/web_fetch/reference_lifecycle.py index 13affaa39..fbe087713 100644 --- a/harness/web_fetch/reference_lifecycle.py +++ b/harness/web_fetch/reference_lifecycle.py @@ -17,7 +17,7 @@ from harness.record_io import load_jsonl, parse_jsonl from .metrics import normalize_for_match, normalize_resource_url -from .reference_inputs import input_tree +from .reference_inputs import input_tree, read_dataset_bytes from .render_evidence import validate_render_reference from .safety import contained_path, safe_identifier @@ -208,13 +208,15 @@ def _resource_target_matches(case, candidates: list[dict]) -> list[dict]: return results -def validate_resource_source_evidence(case, dataset_path: Path) -> dict | None: - """Fail closed unless every resource target has independent, hash-bound evidence. +def validate_resource_source_evidence(case, dataset_path: Path, *, execution: bool = False) -> dict | None: + """Qualify resource sources, or reuse fixed targets during declared execution. HTTP probes remain useful diagnostics, but a blocked probe cannot become a valid source merely because an unbound human-review note says the page was checked. A successful browser observation may qualify only when its bounded candidate list, completion condition and exact resource scope are retained. + Execution of frozen benchmark inputs binds the existing task and answer + bytes instead of reopening historical admission files. """ if _get(case, "task_kind", "body") != "resource": return None @@ -228,6 +230,14 @@ def validate_resource_source_evidence(case, dataset_path: Path) -> dict | None: if verification.get("state") != "valid": raise ReferenceLifecycleError( f"{identifier}: independent resource source verification is not valid") + if execution and verification.get('reference_mode', 'frozen') == 'frozen': + dataset = json.loads(read_dataset_bytes(Path(dataset_path))) + if dataset.get('comparison', {}).get('source_reference_policy') in { + 'frozen_benchmark_inputs_with_explicit_live_exact_checks', + 'frozen_acceptance_contracts_with_live_exact_reference_checks'}: + # Admission has already qualified these fixed answers. Execution + # binds the task bytes; it does not repeat historical qualification. + return None expected_path = f"sources/{identifier}/evidence.json" evidence_path = verification.get("evidence_path") evidence_hash = verification.get("evidence_sha256") @@ -540,7 +550,7 @@ def preflight_reference_checks(cases, dataset_path: Path, now=None) -> dict | No checked = _now(now) bindings = {} for case in cases: - validate_resource_source_evidence(case, dataset_path) + validate_resource_source_evidence(case, dataset_path, execution=True) if requires_fresh_reference(case): identifier = _get(case, "id") if identifier in bindings: diff --git a/harness/web_fetch/report_check.py b/harness/web_fetch/report_check.py index 3a7086b25..c9b46bd1c 100644 --- a/harness/web_fetch/report_check.py +++ b/harness/web_fetch/report_check.py @@ -279,7 +279,7 @@ def check(root: Path, *, authority_root: Path, profile: Path | None = None, pres case_by_id = {c["id"]: c for c in d["cases"]} for case in d["cases"]: safe_identifier(case["id"]) - validate_resource_source_evidence(case, canonical_dataset) + validate_resource_source_evidence(case, canonical_dataset, execution=True) reliability = require_reliable(d, rs) saved_reliability = json.loads((root / "reliability.json").read_text()) if not saved_reliability == reliability: diff --git a/tests/test_reference_lifecycle.py b/tests/test_reference_lifecycle.py index edc5f759f..81aa17d84 100644 --- a/tests/test_reference_lifecycle.py +++ b/tests/test_reference_lifecycle.py @@ -200,6 +200,21 @@ def test_resource_source_requires_hash_bound_independent_full_target_coverage(se with self.assertRaisesRegex(ReferenceLifecycleError, "SHA-256 mismatch"): validate_resource_source_evidence(case, self.dataset_path) + def test_frozen_resource_execution_does_not_repeat_admission(self): + import shutil + case = self.resource_case() + self.dataset_path.write_text(json.dumps({'comparison': { + 'source_reference_policy': 'frozen_benchmark_inputs_with_explicit_live_exact_checks'}})) + shutil.rmtree(self.root / 'sources') + self.assertIsNone(validate_resource_source_evidence(case, self.dataset_path, execution=True)) + with self.assertRaisesRegex(ReferenceLifecycleError, 'invalid resource source evidence'): + validate_resource_source_evidence(case, self.dataset_path) + live = replace(case, source_verification={**case.source_verification, 'reference_mode': 'live_exact'}) + with self.assertRaisesRegex(ReferenceLifecycleError, 'invalid resource source evidence'): + validate_resource_source_evidence(live, self.dataset_path, execution=True) + invalid = replace(case, source_verification={**case.source_verification, 'state': 'invalid'}) + with self.assertRaisesRegex(ReferenceLifecycleError, 'not valid'): + validate_resource_source_evidence(invalid, self.dataset_path, execution=True) def test_resource_source_rejects_missing_target_and_http_challenge(self): missing = self.resource_case(evidence_updates={"resources": []}) with self.assertRaisesRegex(ReferenceLifecycleError, "does not cover every"): diff --git a/tests/test_run_certified_benchmark.py b/tests/test_run_certified_benchmark.py index 9ddaec6a3..d3164dbd3 100644 --- a/tests/test_run_certified_benchmark.py +++ b/tests/test_run_certified_benchmark.py @@ -114,7 +114,15 @@ def test_frozen_contract_execution_needs_no_historical_qualification_files(self) 'source_verification': {'state': 'valid', 'checked_at': '2020-01-01T00:00:00Z', 'method': 'Independent source capture', 'reference_sha256': hashlib.sha256(b'Required label').hexdigest()}} - dataset.write_text(json.dumps({'tool': 'web_fetch', 'comparison': {}, 'cases': [case, fixed]})) + resource = {'id': 'fixed-resource', 'name': 'Fixed image target', + 'url': 'https://example.test/gallery', 'category': 'image', 'track': 'image', + 'task_kind': 'resource', 'expected_any': ['https://example.test/image.png'], + 'expected_resources': [{'id': 'image', 'urls': ['https://example.test/image.png']}], + 'source_verification': {'state': 'valid', 'checked_at': '2020-01-01T00:00:00Z', + 'method': 'Independent source capture', + 'evidence_path': 'sources/fixed-resource/evidence.json', + 'evidence_sha256': 'b' * 64}} + dataset.write_text(json.dumps({'tool': 'web_fetch', 'comparison': {}, 'cases': [case, fixed, resource]})) original = dataset.read_bytes() config = root / 'providers.json' config.write_text(json.dumps({'providers': {'fixture': {'type': 'fixture'}}})) @@ -139,15 +147,17 @@ def execute(stage, arguments): with patch('harness.web_fetch.certification.run', side_effect=execute): main(['--dataset', str(dataset), '--output', str(output), '--work-root', str(root / 'work'), '--providers', 'fixture', - '--provider-config', str(config), '--case-ids', 'stable,fixed-answer']) + '--provider-config', str(config), '--case-ids', 'stable,fixed-answer,fixed-resource']) measured = json.loads((output / 'dataset.json').read_text()) - self.assertEqual(measured['cases'], [case, fixed]) + self.assertEqual(measured['cases'], [case, fixed, resource]) self.assertEqual(dataset.read_bytes(), original) self.assertEqual(measured['comparison']['source_reference_policy'], 'frozen_benchmark_inputs_with_explicit_live_exact_checks') batches = json.loads((output / 'execution.json').read_text())['batches'] records = [json.loads(line) for line in (output / 'results.jsonl').read_text().splitlines()] - self.assertEqual(len(records), 2) + self.assertEqual(len(records), 3) + resource_record = next(row for row in records if row['case_id'] == 'fixed-resource') + self.assertTrue(resource_record['metrics']['content_pass']) self.assertNotIn('collect_source_evidence.py', stages) require_verified_reference_checks(batches, cases=measured, records=records, source=output) From 26833a4cd22047f47e64e37e4d6d32095ea4b3e5 Mon Sep 17 00:00:00 2001 From: lanyue-llk <270302213+lanyue-llk@users.noreply.github.com> Date: Sat, 10 Oct 2026 19:03:12 +0800 Subject: [PATCH 003/172] fix(webfetch): bind native variant evidence and complete resource populations --- .../browser-native-430/current-report.json | 7 +- harness/web_fetch/README.md | 11 + harness/web_fetch/assembly.py | 50 +- harness/web_fetch/publication_privacy.py | 18 +- harness/web_fetch/reference_lifecycle.py | 10 +- harness/web_fetch/report.py | 2 +- harness/web_fetch/report_check.py | 14 +- harness/web_fetch/report_explainer.py | 2 +- harness/web_fetch/report_i18n.py | 10 + harness/web_fetch/report_resources.py | 60 +- harness/web_fetch/rescoring.py | 11 +- harness/web_fetch/resource_cohort.py | 28 +- harness/web_fetch/scoring_revision.py | 25 +- harness/web_fetch/task_outputs.py | 3 + research/report-changes.json | 3021 +++++++++++++---- tests/test_assembly_evidence.py | 94 + tests/test_headless_report.py | 5 +- tests/test_native_task_outputs.py | 16 + tests/test_publication_privacy.py | 22 + tests/test_report_check_owner.py | 24 + tests/test_resource_cohort_boundary.py | 81 + tests/test_scoring_revision.py | 103 + 22 files changed, 2919 insertions(+), 698 deletions(-) diff --git a/evaluation/browser-native-430/current-report.json b/evaluation/browser-native-430/current-report.json index ef0898668..43ca2a7fb 100644 --- a/evaluation/browser-native-430/current-report.json +++ b/evaluation/browser-native-430/current-report.json @@ -9,7 +9,8 @@ "lightpanda", "obscura", "chromium", - "kitesurf" + "kitesurf", + "moli-pr919" ], "timeout_seconds": 45, "runs_per_case": 1, @@ -467,8 +468,8 @@ ] }, "frozen_inputs": { - "dataset_sha256": "7ac804c896f9af55fa5a01c052aad0dce28a5d1835bdc0559ea230a4b45fca2d", - "auxiliary_sha256": "d687f5114e52c030630399797af2f4add2061dfb17c83fb58a81cbfd1af1e312" + "dataset_sha256": "d32ec3a78bc8326b555e2a9863bfe80ef8915aadabcb58b50b89ea4e44f0ed44", + "auxiliary_sha256": "1d1726f0a03f9739cc1a0361e6eddac82cb9c01953b8d9591edaff5771a0c272" }, "report_output": "reports/browser-native-430", "source_native_dataset_sha256": "d636aa8af567da8a2e0ff8caf319dd6369528d5f65f0edfe47db1ac0d9890d64", diff --git a/harness/web_fetch/README.md b/harness/web_fetch/README.md index 56645ce06..ee8506df8 100644 --- a/harness/web_fetch/README.md +++ b/harness/web_fetch/README.md @@ -858,6 +858,17 @@ Lightpanda 使用 `--dump markdown --json` 的原生正文；Obscura 使用其 分支构建在调用前固定来源提交、基线提交、构建环境和二进制 SHA-256。新增配置只调用完整固定题库， 旧配置的原始结果保持不变。不同日期的批次用于展示各自任务完成情况；代码带来的耗时变化须另按同窗配对协议测量。 +### 资源比较的统计范围 + +`resource_cohort.py` 的主统计保留预先确定的完整方法与任务矩阵，包括失败调用。 +平均值、P50 和 P95 使用同一范围，报告同时显示各方法的通过数。各方法成功调用的统计另存为 +条件分布；所有方法共同通过的任务只作附加诊断，不能据此挑选排名样本。没有共同成功项时， +仍须保留完整调用的资源统计；没有成功项的条件指标为不可用，不填零。 + +改正统计范围时，用已保存、哈希校验通过的测量重新汇总，不重新调用浏览器，不覆盖旧汇总。 +主结果、原始输出和每次调用的资源计数保持不变。发布检查核对全部测量值、分母、指标方向 +及唯一最低值标记；并列项保持中性。资源消耗与任务通过率分别呈现，不合成总分。 + ### 已编译评测集的重跑 `scripts/run_certified_benchmark.py` 直接使用冻结的题目与验收规则。已有固定答案或稳定规则的题目 diff --git a/harness/web_fetch/assembly.py b/harness/web_fetch/assembly.py index a001c2810..d6ca86ed3 100644 --- a/harness/web_fetch/assembly.py +++ b/harness/web_fetch/assembly.py @@ -206,11 +206,53 @@ def copy_projected_review(source, output, excluded_providers): return True -def bind_assembled_scoring_revision(output): - """Bind inherited scoring history to exact assembled bytes and full blind review.""" +def bind_assembled_scoring_revision(output, *, source=None): + """Bind composed scoring; revalidate machine evidence without inventing review.""" path=output/'scoring-revisions.json' review_path=output/'result-review.json' - if not path.is_file() or not review_path.is_file():return False + if not path.is_file():return False + if not review_path.is_file(): + if source is None or valid_scoring_revision(output):return False + if not valid_scoring_revision(source): + raise ValueError('Assembly source has invalid scoring authority') + if json.loads((output/'dataset.json').read_text()).get('comparison',{}).get( + 'require_independent_result_review'): + raise ValueError('Assembly requires its declared independent result review') + inherited=path.read_bytes() + if inherited != (source/'scoring-revisions.json').read_bytes(): + raise ValueError('Inherited scoring authority differs from declared source') + from .scoring_regressions import active_scoring_regressions + from .scoring_revision import revise + controls=active_scoring_regressions() + history=json.loads(inherited) + identifiers=list(dict.fromkeys(identifier for receipt in history + for identifier in receipt.get('correction_provenance',{}).get('regression_ids',[]) + if identifier in controls)) + if not identifiers: + raise ValueError('Assembly scoring authority has no current executable controls') + categories={controls[identifier]['category'] for identifier in identifiers} + before=read_records(output) + archive=output/'diagnostics'/'inherited-scoring-history.json' + archive.parent.mkdir(parents=True,exist_ok=True) + archive.write_bytes(inherited) + path.unlink() + receipt=revise(output,output/'dataset.json', + root_cause_category=next(iter(categories)) if len(categories)==1 else 'multi-root', + root_cause='Revalidate inherited shared scoring corrections against the complete ' + 'certified assembled cohort. Task definitions, answers and selected physical ' + 'calls remain unchanged; independent result review is not inferred.', + regression_ids=identifiers,validate_only=True) + after=read_records(output) + if receipt['task_decision_changes'] or before!=after: + raise ValueError('Assembly revalidation changed measured calls or scores') + receipt['inherited_scoring_authority']={ + 'dataset_sha256':hashlib.sha256((source/'dataset.json').read_bytes()).hexdigest(), + 'results_sha256':hashlib.sha256((source/'results.jsonl').read_bytes()).hexdigest(), + 'receipts_sha256':hashlib.sha256(inherited).hexdigest()} + path.write_text(json.dumps([receipt],ensure_ascii=False,indent=2)+'\n') + if not valid_scoring_revision(output): + raise ValueError('Assembly revalidation left invalid scoring authority') + return True dataset=json.loads((output/'dataset.json').read_text()) records=read_records(output) require_result_review(output,dataset,records) @@ -906,7 +948,7 @@ def _assemble(run, certificates, replacements, output, additions=None, provider_ if not copied and not any((replacements, additions, provider_replacements, provider_additions, canonical_dataset)): copy_projected_review(run,output,[]) - bind_assembled_scoring_revision(output) + bind_assembled_scoring_revision(output, source=run) if speed_cohort: if not (speed_cohort/'speed-comparison.json').is_file(): raise ValueError('Speed cohort has not been packaged') diff --git a/harness/web_fetch/publication_privacy.py b/harness/web_fetch/publication_privacy.py index 72fc4b992..cd9c6cab6 100644 --- a/harness/web_fetch/publication_privacy.py +++ b/harness/web_fetch/publication_privacy.py @@ -57,6 +57,19 @@ def project_execution_manifest(manifest: dict) -> dict: rb'(?i)(?:proxy-)?authorization\s*[:=]\s*' rb'(?:[A-Za-z][A-Za-z0-9_-]*\s+)?(?:<[^>\r\n]+>|\[[^\]\r\n]+\])' ) +PUBLIC_CORS_CREDENTIAL_POLICY = re.compile( + rb'''(?i)["']?access[-_]control[-_]allow[-_]credentials["']?\s*[:=]\s*''' + rb'''(?:%[si])?["'`]*(?:true|false)["'`]*''' +) + + +def _public_cors_policy(data): + """A CORS permission Boolean (including ABNF) is not a credential value.""" + decoded = html.unescape(data.decode('utf-8', errors='surrogateescape')) + return bool(PUBLIC_CORS_CREDENTIAL_POLICY.fullmatch(decoded.strip().encode( + 'utf-8', errors='surrogateescape'))) + + AUTH_SENSITIVE_VALUE = re.compile( rb'(?im)((?:proxy[-_]?authorization|authorization)[ \t]*[:=][ \t]*)' rb'(?![A-Za-z][A-Za-z0-9_-]*[ \t]+(?:<|\[)|(?:<|\[))' @@ -177,6 +190,8 @@ def auth_matches(data): ) distinct = [] for match in matches: + if _public_cors_policy(match.group()): + continue if distinct and match.start() < distinct[-1].end(): continue distinct.append(match) @@ -237,7 +252,8 @@ def project_pre(match): projected = GOOGLE_KEY.sub(b'[REDACTED_GOOGLE_API_KEY]', url_projected.encode('utf-8', errors='surrogateescape')) for pattern, _, replacement in REDACTIONS: - projected = pattern.sub(replacement, projected) + projected = pattern.sub(lambda match: match.group() if _public_cors_policy( + match.group()) else match.expand(replacement), projected) projected = LOCAL_USER_PATH.sub(b'/[REDACTED_USER]/', projected) categories = [] kinds = [] diff --git a/harness/web_fetch/reference_lifecycle.py b/harness/web_fetch/reference_lifecycle.py index fbe087713..acb58e79a 100644 --- a/harness/web_fetch/reference_lifecycle.py +++ b/harness/web_fetch/reference_lifecycle.py @@ -895,6 +895,13 @@ def revision_edge(value): if (edge is None or receipt.get('kind') != 'report_interpretation_binding_revision' or receipt.get('provider_calls') != 0 or edge[0][1] != edge[1][1]): return False + asset_fields = receipt.get('bound_asset_fields', ['report_interpretation']) + if (not isinstance(asset_fields, list) or not asset_fields + or any(not isinstance(field, str) for field in asset_fields) + or len(asset_fields) != len(set(asset_fields)) + or 'report_interpretation' not in asset_fields + or set(asset_fields) - {'report_interpretation', 'implementation_provenance'}): + return False snapshots = [] for side, binding in zip(('input', 'output'), edge): raw = contained_path(root, receipt[f'{side}_dataset_path']).read_bytes() @@ -903,7 +910,8 @@ def revision_edge(value): document = json.loads(raw) if not isinstance(document, dict): return False - document.get('comparison', {}).pop('report_interpretation', None) + for field in asset_fields: + document.get('comparison', {}).pop(field, None) if not document.get('comparison'): document.pop('comparison', None) snapshots.append(document) diff --git a/harness/web_fetch/report.py b/harness/web_fetch/report.py index 96a94651a..71b4c7e79 100644 --- a/harness/web_fetch/report.py +++ b/harness/web_fetch/report.py @@ -2249,7 +2249,7 @@ def write_support(out,records,manifests,*,authored_interpretation=True): methodology+=section('每类怎么测',examples_html)+section('指标说明',metric_guide()) methodology+=section('共同输入、计时边界与调用预算','

所有方法进入同一个评分入口，但按任务交付物验收，而不是强制使用同一种中间格式。网页与文档正文最终验收可读文本；原始文件任务验收 PDF、DOCX、图片或视频字节；结构任务验收字段和稳定文字。HTML/Markdown 链接与明确配对的结构化名称—链接字段按同一规则核验；不把请求回显或诊断日志当成答案。原始输出与原始适配器状态保留，报告层不按方法名称改写任务结果。

计入耗时

从准备发起该次 provider 调用开始，到形成可送入统一评分器的最终答案为止：线程调度、SDK/CLI/HTTP 调用、远端处理与网络下载、浏览器渲染与正文提取、本地解析、统一输入转换，以及文件字节读取、格式识别和 SHA-256 计算均在计时内。

不计入耗时

数据集与参考加载、provider 配置校验、调用间限速等待、最终答案形成后的证据落盘、统一评分计算、报告汇总与页面生成不计入。供应商上报的服务端耗时只作附加证据，排名使用客户端观测的端到端耗时。

这个边界比较的是“拿到本任务最终交付物”所需时间：服务直接返回成品时计完整往返；本地方法若还需下载、渲染或解析，也必须完成后才停表。正文任务中，方法只返回原始 PDF 或 DOCX 时，本地正文解析也计时；原始文件任务则到字节校验完成即停止，不做无关的 Markdown 转换。所有方法使用同一外层计时器和截止预算。超时答案不接收；无法取消的调用会被隔离，整批结果因此不完整并触发不可靠状态，构建产物不得发布。

完整正文任务除原有核对点外，按参考正文最多五个连续位置检查字符六元组覆盖：每个位置至少达到 60%，通常要求至少四处达到 80%。所有位置一次性做全局容量分配；同一返回片段实例只能计入一个位置，位置排列顺序不影响结论。若页面存在经独立证据确认的合法重排，案例可显式登记不同位置门槛。比较忽略空白和排版符号但保留文字顺序；全文 F1 独立保留实际值。文件任务必须有字节验收，资源 URL 不等于文件。

') if (out/'resource-cohort/resource-comparison.json').is_file(): - methodology+=section('资源测量边界','

CPU、内存和网络使用独立的 20 题同窗队列，不回填历史正确率调用。每次调用在受限的 Linux 容器中运行，使用 cgroup v2 累计 CPU、独立网络命名空间累计收发字节，并以 50 ms 采样内存与瞬时速率。容器固定为 2 个逻辑 CPU、2 GiB 内存和 512 个 PID；宿主控制器与容器启停不计入。

资源汇总只纳入四种本地配置共同通过的任务，避免把快速失败当成节省资源，并覆盖容器内 worker、浏览器与转换过程。Kitesurf 的远端 CPU、内存和上游流量不可见，因此不进入资源表。缺少任一资源字段、非 Linux/cgroup v2、镜像或二进制身份变化、并发不为 1，都会阻断资源包构建。

CPU 与网络总量来自累计计数；内存峰值和速率分位数依赖 50 ms 采样，可能遗漏更短的尖峰。报告分别呈现 CPU、内存、网络和任务通过率，不合成总分。

') + methodology+=section('资源测量边界','

CPU、内存和网络使用独立的 20 题同窗队列，不回填历史正确率调用。每次调用在受限的 Linux 容器中运行，使用 cgroup v2 累计 CPU、独立网络命名空间累计收发字节，并以 50 ms 采样内存与瞬时速率。容器固定为 2 个逻辑 CPU、2 GiB 内存和 512 个 PID；宿主控制器与容器启停不计入。

资源主表纳入预先确定的全部任务，包括失败调用，统计容器内调用进程、浏览器与转换过程的消耗；同时列出各方法通过数。各方法成功调用另作条件统计，共同通过的任务仅作诊断。Kitesurf 的远端 CPU、内存和上游流量不可见，因此不进入资源表。缺少任一资源字段、非 Linux/cgroup v2、镜像或二进制身份变化、并发不为 1，都会阻断资源包构建。

CPU 与网络总量来自累计计数；内存峰值和速率分位数依赖 50 ms 采样，可能遗漏更短的尖峰。报告分别呈现 CPU、内存、网络和任务通过率，不合成总分。

') if (out/'scoring-revisions.json').exists(): revision_links=[link('../scoring-revisions.json','评分规则修订收据')] if (out/'reference-revisions.json').exists():revision_links.append(link('../reference-revisions.json','参考答案修订收据')) diff --git a/harness/web_fetch/report_check.py b/harness/web_fetch/report_check.py index c9b46bd1c..cd8eef6e4 100644 --- a/harness/web_fetch/report_check.py +++ b/harness/web_fetch/report_check.py @@ -525,14 +525,15 @@ def canonical_link(page, value): if not ("本类最高：" in home and "判分完整性" in home): raise ValueError("Report acceptance check failed") if (root / "resource-cohort/resource-comparison.json").is_file(): + from .report_resources import check_resource_presentation + resource_summary = json.loads((root / "resource-cohort/resource-comparison.json").read_text()) guide = (root / "guide/index.html").read_text() for page_name, text in [("home", home), ("guide", guide)]: + check_resource_presentation(text, resource_summary) if "data-resource-winner=" not in text: raise ValueError((page_name, "missing resource winner summary")) if not all(label in text for label in ("CPU 与计算", "内存", "网络")): raise ValueError((page_name, "missing grouped resource tables")) - if not text.count("data-resource-metric-winner=") >= 9: - raise ValueError((page_name, "resource winners are not marked")) if has_english: for page_name, text in [ ("English home", (root / "en/index.html").read_text()), @@ -542,8 +543,7 @@ def canonical_link(page, value): raise ValueError((page_name, "missing resource winner summary")) if not all(label in text for label in ("CPU and compute", "Memory", "Network")): raise ValueError((page_name, "missing grouped resource tables")) - if not text.count("data-resource-metric-winner=") >= 9: - raise ValueError((page_name, "resource winners are not marked")) + check_resource_presentation(text, resource_summary) methodology = (root / "methodology/index.html").read_text() for phrase in [ "计入耗时", @@ -696,13 +696,15 @@ def check_revision(out: Path, *, authority_root: Path): '.interface-panel[id^="interface-"]' ) from harness.web_fetch.report_interfaces import SPECS + from harness.web_fetch.methods import native_browser_engine + interface_names = set(SPECS) | {name for name in names if native_browser_engine(name)} if not ( - len(interfaces) == len(SPECS) + len(interfaces) == len(interface_names) and sum(p.has_attr("open") for p in interfaces) == bool(interfaces) ): raise ValueError("Report presentation check failed") - if {panel.get("id") for panel in interfaces} != {"interface-" + name for name in SPECS}: + if {panel.get("id") for panel in interfaces} != {"interface-" + name for name in interface_names}: raise ValueError("Interface inventory differs from registry") for panel in interfaces: if not len(panel.select("pre")) == 2: diff --git a/harness/web_fetch/report_explainer.py b/harness/web_fetch/report_explainer.py index 1caab8d75..113e9f2a9 100644 --- a/harness/web_fetch/report_explainer.py +++ b/harness/web_fetch/report_explainer.py @@ -724,7 +724,7 @@ def bars(values, unit='', maximum=None): 'resources': resource_table, 'resource_interpretation': escape(resource_interpretation), 'resource_methodology': ( - '
资源指标怎样测

资源数据来自四种本地方法的单独同题 cohort，不回填到历史正确率调用。每次调用启动一个受限的 Linux 客户端容器及独立网络；宿主控制器和容器启停不计入。CPU 与网络总量读取累计计数，内存与速率分位数按 50 ms 采样。

资源汇总只使用四种本地配置共同通过的任务，避免把快速失败或空输出当成节省资源。短于采样间隔的内存或速率峰值可能遗漏；Kitesurf 等远端服务的服务端 CPU、内存与上游流量不可观测，因而不进入资源表。

' + '
资源指标怎样测

资源数据来自四种本地方法的独立同题测量，不回填到历史通过率调用。每次调用启动一个受限的 Linux 客户端容器及独立网络；宿主控制器和容器启停不计入。CPU 与网络总量读取累计计数，内存与速率分位数按 50 ms 采样。

资源主表统计预先确定的全部任务，包括失败调用，并列出各方法的通过数；成功调用另作条件统计，共同通过的任务仅作诊断。短于采样间隔的内存或速率峰值可能遗漏；Kitesurf 等远端服务的服务端 CPU、内存与上游流量不可观测，因而不进入资源表。

' if resource_summary is not None else '' ), 'pdf_chars': f'{pdf_chars:,}', diff --git a/harness/web_fetch/report_i18n.py b/harness/web_fetch/report_i18n.py index 6b0144922..38bb17c8c 100644 --- a/harness/web_fetch/report_i18n.py +++ b/harness/web_fetch/report_i18n.py @@ -3334,6 +3334,11 @@ def release_translations(history: dict) -> dict[str, str]: "接收": "Received", "发送": "Sent", "同题口径：所列配置共同通过": "Matched-task scope: general pages passed by every listed configuration:", + "所列配置共同通过": "Tasks passed by every listed configuration:", + "；这项交集只作附加诊断，不用于选择资源表的样本。各方法成功调用的条件统计另保留在资源证据中。": "; this intersection is diagnostic only and does not select the resource-table population. Conditional statistics for each method's successful calls are retained in the resource evidence.", + "资源数据来自四种本地方法的独立同题测量，不回填到历史通过率调用。每次调用启动一个受限的 Linux 客户端容器及独立网络；宿主控制器和容器启停不计入。CPU 与网络总量读取累计计数，内存与速率分位数按 50 ms 采样。": "Resource data comes from a separate matched-task measurement of the four local methods and is not added to historical pass-rate calls. Each call starts a limited Linux client container with a dedicated network; the host controller and container lifecycle are excluded. CPU and network totals use cumulative counters, while memory and rate percentiles are sampled every 50 ms.", + "资源主表统计预先确定的全部任务，包括失败调用，并列出各方法的通过数；成功调用另作条件统计，共同通过的任务仅作诊断。短于采样间隔的内存或速率峰值可能遗漏；Kitesurf 等远端服务的服务端 CPU、内存与上游流量不可观测，因而不进入资源表。": "The primary resource table includes every predeclared task, including failed calls, and shows each method's pass count. Successful calls have separate conditional statistics; shared successes are diagnostic only. Memory or rate peaks shorter than the sampling interval may be missed. Server-side CPU, memory, and upstream traffic for remote services such as Kitesurf are unobservable, so they are excluded from the resource table.", + "资源主表纳入预先确定的全部任务，包括失败调用，统计容器内调用进程、浏览器与转换过程的消耗；同时列出各方法通过数。各方法成功调用另作条件统计，共同通过的任务仅作诊断。Kitesurf 的远端 CPU、内存和上游流量不可见，因此不进入资源表。缺少任一资源字段、非 Linux/cgroup v2、镜像或二进制身份变化、并发不为 1，都会阻断资源包构建。": "The primary resource table includes every predeclared task, including failed calls, and measures the caller process, browser, and conversion within the container. Each method's pass count is shown. Successful calls have separate conditional statistics; shared successes are diagnostic only. Kitesurf's remote CPU, memory, and upstream traffic are unobservable, so it is excluded. Missing resource fields, a non-Linux/cgroup-v2 backend, changed image or binary identity, or concurrency other than one block resource packaging.", "个普通网页；失败任务不进入资源排名。": "pages; failed tasks do not enter the resource ranking.", "CPU 与计算": "CPU and compute", "内存": "Memory", @@ -3360,6 +3365,11 @@ def release_translations(history: dict) -> dict[str, str]: "左右滑动查看完整对比": "Scroll horizontally for the full comparison", }) +REGEX.insert(0, ( + re.compile(r"^资源表覆盖预先确定的全部 (\d+) 个普通网页，包含失败调用。这组任务的通过数：(.*)。各项平均值、P50、P95 使用同一完整范围。$"), + r"The resource table covers all \1 predeclared general pages, including failed calls. Pass counts: \2. Mean, P50, and P95 share this complete population.", +)) + REGEX.insert(0, ( re.compile(r"^(\d+)/(\d+)$"), r"\1/\2", diff --git a/harness/web_fetch/report_resources.py b/harness/web_fetch/report_resources.py index d67dc4893..029295d23 100644 --- a/harness/web_fetch/report_resources.py +++ b/harness/web_fetch/report_resources.py @@ -122,10 +122,14 @@ def lowest(key, statistic): cells = [] for provider in local: rendered = escape(_format(values[provider], unit)) - if lower_is_better and values[provider] == best: - rendered = (f'' + if lower_is_better and values[provider] == best and list(values.values()).count(best) == 1: + rendered = (f'' f'{rendered}本项最低') - cells.append('') + cells.append(f'') rows.append(f'' + ''.join(cells) + '') header = ''.join('' @@ -138,9 +142,16 @@ def lowest(key, statistic): '
达标率' + rendered + '' + rendered + '
{row_index}' + escape(metric_name) + '
' + escape(PROVIDER_NAMES.get(provider, provider)) + '
' + header + '' + ''.join(rows) + '
序号指标
') - sample = (f'

同题口径：所列配置共同通过 ' - f'{len(summary["common_pass_cases"])}/{len(summary["planned_cases"])} 个普通网页；' - '失败任务不进入资源排名。

') + count = len(summary['planned_cases']) + completions = '，'.join(PROVIDER_NAMES.get(provider, provider) + ' ' + + str(indexed[provider].get('successful_attempts', '—')) + '/' + str(count) + for provider in local) + sample = (f'

资源表覆盖预先确定的全部 {count} 个普通网页，包含失败调用。' + f'这组任务的通过数：{escape(completions)}。各项平均值、P50、P95 使用同一完整范围。

' + '

所列配置共同通过 ' + f'{len(summary["common_pass_cases"])}/{count}；' + '这项交集只作附加诊断，不用于选择资源表的样本。各方法成功调用的条件统计另保留在资源证据中。

') caveat = ('

可比边界：每次调用使用一次性 Linux 客户端容器、cgroup v2 与独立网络命名空间；' '宿主控制器和容器启停不计入。Kitesurf 的远端浏览器 CPU、内存和远端上游流量不可见，' '因此不进入资源表或本地引擎资源排名。

' @@ -153,3 +164,40 @@ def lowest(key, statistic): 'ranked_metric_count': len(winners), 'winner_tied': leader_tied, } + + +def check_resource_presentation(html, summary): + """Verify complete-population values and unique extrema at release time.""" + from harness.reporting.presentation import check_presentation + soup = check_presentation(html) + if summary.get('primary_population') != 'all_declared_calls': + raise ValueError('Resource ranking must retain every declared call') + count = len(summary['planned_cases']) + sample = soup.select('[data-resource-population="all_declared_calls"]') + if len(sample) != 1 or sample[0].get('data-resource-cases') != str(count): + raise ValueError('Resource comparison has the wrong population or denominator') + headline = soup.select('[data-resource-winner]') + expected_html, _ = render_resource_comparison(summary, summary['providers']) + from bs4 import BeautifulSoup + expected_leaders = BeautifulSoup(expected_html, 'html.parser').select_one( + '[data-resource-winner]')['data-resource-winner'] + if len(headline) != 1 or headline[0]['data-resource-winner'] != expected_leaders: + raise ValueError('Resource leader differs from measured metric winners') + expected = {(row['provider'], key, statistic): row['metrics'][key][statistic] + for row in summary['rows'] for _, metrics in GROUPS + for _, key, statistic, _, _ in metrics} + cells = soup.select('td[data-resource-provider][data-resource-key][data-resource-statistic]') + observed = {} + for cell in cells: + identity = (cell['data-resource-provider'], cell['data-resource-key'], cell['data-resource-statistic']) + if identity in observed or cell.get('data-sort') is None: + raise ValueError('Resource comparison contains duplicate or unavailable measurements') + direction = next('min' if lower else 'neutral' for _, metrics in GROUPS + for _, key, statistic, _, lower in metrics if identity[1:] == (key, statistic)) + if cell.get('data-metric-direction') != direction or cell.get('data-metric') != ( + 'resource-' + identity[1] + '-' + identity[2]): + raise ValueError('Resource metric direction or identity changed') + observed[identity] = float(cell['data-sort']) + if observed != expected or any(row['attempts'] != count or any( + value['n'] != count for value in row['metrics'].values()) for row in summary['rows']): + raise ValueError('Resource comparison differs from its complete measured cohort') diff --git a/harness/web_fetch/rescoring.py b/harness/web_fetch/rescoring.py index 3a6c7ac59..8bc88fc5e 100644 --- a/harness/web_fetch/rescoring.py +++ b/harness/web_fetch/rescoring.py @@ -2,6 +2,7 @@ import hashlib import json +from copy import deepcopy from datetime import datetime, timezone from pathlib import Path @@ -25,7 +26,8 @@ from harness.web_fetch.scoring import SCORING_SOURCE_SHA256, load_list_reference, score_attempt -def rescore(source, case_ids=None, *, allow_rule_revision=False, reference_revision_hashes=None): +def rescore(source, case_ids=None, *, allow_rule_revision=False, reference_revision_hashes=None, + validate_only=False): dataset_path = contained_path(source, "dataset.json") dataset_bytes = dataset_path.read_bytes() dataset = json.loads(dataset_bytes) @@ -112,6 +114,7 @@ def rescore(source, case_ids=None, *, allow_rule_revision=False, reference_revis if case_ids is not None and r["case_id"] not in case_ids: continue c = cases[r["case_id"]] + original = deepcopy(r) if validate_only else None raw = contained_path(source, r["artifact_path"]).read_bytes() if hashlib.sha256(raw).hexdigest() != r["metrics"]["content_sha256"]: raise ValueError("Output integrity failure: " + r["attempt_id"]) @@ -129,6 +132,10 @@ def rescore(source, case_ids=None, *, allow_rule_revision=False, reference_revis list_reference=load_list_reference(c, source), ) apply_request_status(r) + if validate_only: + if r != original: + raise ValueError('Scoring validation would change a retained attempt: ' + r['attempt_id']) + continue if "run_id" not in r: r["run_id"] = execution_id(r) r["diagnosis"] = diagnose(r, c, raw.decode("utf-8", errors="replace")) @@ -140,6 +147,8 @@ def rescore(source, case_ids=None, *, allow_rule_revision=False, reference_revis temporary = contained_path(source, "results.jsonl.tmp") if path.read_bytes() != original_results or dataset_path.read_bytes() != dataset_bytes: raise ValueError("Scoring inputs changed during rescoring") + if validate_only: + return [] # JSON Lines readers split on physical line boundaries. Escaping Unicode # line/paragraph separators keeps one attempt on exactly one line too. temporary.write_text("".join(json.dumps(r, ensure_ascii=True) + "\n" for r in records)) diff --git a/harness/web_fetch/resource_cohort.py b/harness/web_fetch/resource_cohort.py index 698ddb68c..4348f2799 100644 --- a/harness/web_fetch/resource_cohort.py +++ b/harness/web_fetch/resource_cohort.py @@ -36,9 +36,9 @@ def _stats(values): return { "n": len(values), - "p50": statistics.median(values), - "p95": percentile(values, 0.95), - "mean": statistics.fmean(values), + "p50": statistics.median(values) if values else None, + "p95": percentile(values, 0.95) if values else None, + "mean": statistics.fmean(values) if values else None, } @@ -101,21 +101,29 @@ def build_summary(dataset: dict, records: list[dict], batches: list[dict]) -> di for provider in providers ) ) - if len(common) < 5: - raise ValueError("Too few all-provider successful cases for resource comparison") by_key = {(record["provider"], record["case_id"]): record for record in records} rows = [] for provider in providers: metrics = { - metric: _stats([value(by_key[provider, case_id], metric) for case_id in common]) + metric: _stats([value(by_key[provider, case_id], metric) for case_id in cases]) for metric in METRICS } - rows.append({"provider": provider, "attempts": len(common), "metrics": metrics}) + successful = [case_id for case_id, case in cases.items() if task_passed( + by_key[provider, case_id]['ok'], by_key[provider, case_id]['metrics'], + case.get('task_kind', 'body'))] + rows.append({"provider": provider, "attempts": len(cases), "metrics": metrics, + "successful_attempts": len(successful), "successful_case_ids": sorted(successful), + "successful_metrics": {metric: _stats([value(by_key[provider, case_id], metric) + for case_id in successful]) for metric in METRICS}}) summary = { - "schema_version": 1, + "schema_version": 2, + "primary_population": "all_declared_calls", + "conditional_population": "individually_successful_calls", + "common_pass_population": "diagnostic_only", "selection": ( - "Predeclared provider-blind ordinary-web cohort; aggregates use the single " - "all-provider successful case intersection." + "Every predeclared provider-task call is included in the primary distribution. " + "Per-provider successful-call distributions are conditional; the common-pass " + "intersection is diagnostic only and never selects ranking samples." ), "interpretation": "isolated_local_browser_client_container_cost", "scope": ( diff --git a/harness/web_fetch/scoring_revision.py b/harness/web_fetch/scoring_revision.py index 4d35d1fa7..911f004ec 100644 --- a/harness/web_fetch/scoring_revision.py +++ b/harness/web_fetch/scoring_revision.py @@ -83,6 +83,7 @@ def _revise_unlocked( root_cause_category: str, root_cause: str, regression_ids: list[str], + validate_only: bool = False, ): # Validate correction intent and executable controls before touching evidence. provenance = correction_provenance(root_cause_category, root_cause, regression_ids) @@ -150,13 +151,17 @@ def _revise_unlocked( changed.append({"case_id": identifier, "fields": fields}) dataset["cases"] = [updated_cases[case["id"]] for case in dataset["cases"]] validate_report_ids(dataset, parse_jsonl(before_results)) - dataset_path.write_text(json.dumps(dataset, ensure_ascii=False, indent=2) + "\n") + if validate_only and changed: + raise ValueError('Scoring validation cannot change task requirements') + if not validate_only: + dataset_path.write_text(json.dumps(dataset, ensure_ascii=False, indent=2) + "\n") staged_dataset = read_dataset_bytes(dataset_path) try: decisions = rescore( source, allow_rule_revision=True, reference_revision_hashes=revised_reference_hashes or None, + validate_only=validate_only, ) except BaseException: # Never roll stale bytes over a different process's completed revision. @@ -203,6 +208,7 @@ def _revise_unlocked( "changed_cases": changed, "task_decision_changes": decisions, "provider_calls": 0, + **({'validation_only': True} if validate_only else {}), "immutable_call_fields": ( "attempt identity, output bytes/hash, timestamps and elapsed time retained" ), @@ -224,7 +230,7 @@ def _revise_unlocked( return receipt -def rebind_report_interpretation(source: Path, binding: dict): +def rebind_report_interpretation(source: Path, binding: dict, *, implementation_binding: dict | None = None): """Bind presentation content while preserving the scored output bytes.""" from .reference_lifecycle import valid_scoring_revision source = source.resolve() @@ -232,17 +238,28 @@ def rebind_report_interpretation(source: Path, binding: dict): raise ValueError('Report binding requires a valid existing scoring chain') if set(binding) != {'path', 'sha256'}: raise ValueError('Invalid report interpretation binding') - asset = contained_path(source, binding['path']) + asset = contained_path(source, binding['path'], reject_symlinks=True) if hashlib.sha256(asset.read_bytes()).hexdigest() != binding['sha256']: raise ValueError('Report interpretation content hash mismatch') + if implementation_binding is not None: + if (set(implementation_binding) != {'path', 'sha256'} + or implementation_binding['path'] != 'implementation-provenance.json'): + raise ValueError('Invalid implementation provenance binding') + implementation = contained_path(source, implementation_binding['path'], reject_symlinks=True) + if hashlib.sha256(implementation.read_bytes()).hexdigest() != implementation_binding['sha256']: + raise ValueError('Implementation provenance content hash mismatch') dataset_path = contained_path(source, 'dataset.json') before = dataset_path.read_bytes() dataset = json.loads(before) dataset.setdefault('comparison', {})['report_interpretation'] = binding + if implementation_binding is not None: + dataset['comparison']['implementation_provenance'] = implementation_binding after = (json.dumps(dataset, ensure_ascii=False, indent=2)+'\n').encode() results_hash = hashlib.sha256((source/'results.jsonl').read_bytes()).hexdigest() receipt = {'kind': 'report_interpretation_binding_revision', 'provider_calls': 0, 'input_results_sha256': results_hash, 'output_results_sha256': results_hash} + receipt['bound_asset_fields'] = ['report_interpretation'] + ( + ['implementation_provenance'] if implementation_binding is not None else []) for side, raw in (('input', before), ('output', after)): digest = hashlib.sha256(raw).hexdigest() relative = f'revision-history/dataset-{digest}.json' @@ -268,6 +285,7 @@ def revise( root_cause_category: str, root_cause: str, regression_ids: list[str], + validate_only: bool = False, ): """Serialize evidence revisions and make rollback ownership explicit.""" source = source.resolve() @@ -282,6 +300,7 @@ def revise( root_cause_category=root_cause_category, root_cause=root_cause, regression_ids=regression_ids, + validate_only=validate_only, ) finally: fcntl.flock(lock.fileno(), fcntl.LOCK_UN) diff --git a/harness/web_fetch/task_outputs.py b/harness/web_fetch/task_outputs.py index 17bfd3e69..1f7ad9106 100644 --- a/harness/web_fetch/task_outputs.py +++ b/harness/web_fetch/task_outputs.py @@ -192,6 +192,9 @@ def require_native_task_routes(cases, records): if mode == 'markdown': if provider == 'moli': valid = meta.get('route') == 'ordinary_markdown' and meta.get('output') == 'markdown' + if meta.get('route') == 'dynamic_native_markdown': + valid = (case.track == 'dynamic' and meta.get('output') == 'markdown' + and meta.get('dump') == 'markdown') elif provider == 'chromium': valid = (meta.get('extractor') == 'node-html-markdown' and (meta.get('extractor_version') == '2.0.0' diff --git a/research/report-changes.json b/research/report-changes.json index b1df9c92c..d87433d25 100644 --- a/research/report-changes.json +++ b/research/report-changes.json @@ -174,16 +174,18 @@ "status": "已处理", "scope": "Browser Use 首页、核心指标表、同题矩阵、性能统计和结果分析页。", "findings": "上一版误把排除 14 个已知外部干扰任务后的 76 题事后诊断子集写成正式主结果，造成报告看起来只有 76 题；首页还重复突出首次执行与重试恢复，掩盖了每题一个最终逻辑结果的正式口径。", - "corrections": "首页和结果分析恢复完整 90 题官方 evaluator 结果；四种配置的核心指标、同题矩阵和性能统计均使用 90 题。76 题仅保留为明确标注的事后诊断子集，不改分、不替代官方结果。首页删除首次/重试拆分模块；共同成功性能统计按最终逻辑结果及其全部物理执行计算，物理执行明细只在单题证据页下钻。", + "corrections": "首页和结果分析恢复完整 90 题官方 evaluator 结果；四种配置的核心指标、同题矩阵和性能统计均使用 90 题。76 题仅保留为明确标注的事后诊断子集，不改分、不替代官方结果。首页删除首次/重试拆分模块；共同成功性能统计按最终逻辑结果及其全部物理执行计算，物理执行明细只在单题证据页下钻。", "remaining": "14 个外部干扰任务的历史分数保持不变；只有修复后按冻结协议产生新的完整 90 题证据批次，才能更新官方结果。", - "i18n": {"en": { - "title": "Restore the complete 90-task official result as the primary view", - "status": "Resolved", - "scope": "The Browser Use homepage, core metrics, same-task matrix, performance statistics, and result analysis.", - "findings": "The prior version incorrectly presented the 76-task post-hoc diagnostic subset, which excludes 14 known interference tasks, as the formal primary result. It therefore appeared to contain only 76 tasks. The homepage also overemphasized first attempts and retry recovery instead of one final logical result per task.", - "corrections": "Restored the complete official 90-task evaluator result on the homepage and analysis page. Core metrics, the same-task matrix, and performance statistics now use all 90 tasks across all four configurations. The 76-task view remains only as an explicitly labeled post-hoc diagnostic subset and does not rescore or replace the official result. Removed the first-attempt/retry split from the homepage; common-success performance now uses each final logical result and all of its physical executions, while physical-execution details remain on task evidence pages.", - "remaining": "Historical scores for the 14 interference tasks remain unchanged. Only a new complete 90-task evidence cohort produced after corrections under the frozen protocol can update the official result." - }} + "i18n": { + "en": { + "title": "Restore the complete 90-task official result as the primary view", + "status": "Resolved", + "scope": "The Browser Use homepage, core metrics, same-task matrix, performance statistics, and result analysis.", + "findings": "The prior version incorrectly presented the 76-task post-hoc diagnostic subset, which excludes 14 known interference tasks, as the formal primary result. It therefore appeared to contain only 76 tasks. The homepage also overemphasized first attempts and retry recovery instead of one final logical result per task.", + "corrections": "Restored the complete official 90-task evaluator result on the homepage and analysis page. Core metrics, the same-task matrix, and performance statistics now use all 90 tasks across all four configurations. The 76-task view remains only as an explicitly labeled post-hoc diagnostic subset and does not rescore or replace the official result. Removed the first-attempt/retry split from the homepage; common-success performance now uses each final logical result and all of its physical executions, while physical-execution details remain on task evidence pages.", + "remaining": "Historical scores for the 14 interference tasks remain unchanged. Only a new complete 90-task evidence cohort produced after corrections under the frozen protocol can update the official result." + } + } }, { "id": "review-2026-09-11-interpretation-reader-audit", @@ -194,14 +196,16 @@ "findings": "原稿把不同模型配置放在一起标记优胜，混淆了程序判分位置与人工复核根因；重试累计也会扭曲性能对比。部分失败缺少直接下钻入口，英文页仍有中文残留，平板宽度过早堆成单列。", "corrections": "优胜只在同一模型内比较浏览器；分别披露 43 个仅网络规则失败、20 个仅回答规则失败、12 个双规则失败，以及 29/24/10/8/2/2 的复核根因。性能资源改用四配置首次共同成功的 64 题且只取首次执行，并补齐逐结果链接、英文门禁和响应式布局。", "remaining": "当前仍是每种配置一个正式样本。14 个已确认外部干扰任务需在修复后完整重跑；增加重复次数前不发布模型或浏览器总体排名。", - "i18n": {"en": { - "title": "Reader audit of interpretation, comparison semantics, and bilingual layout", - "status": "Resolved", - "scope": "The report home, guide, analysis, failure drill-down, English localization, and desktop, tablet, and phone breakpoints.", - "findings": "The draft marked winners across different model configurations, conflated programmatic failure location with human-reviewed root cause, and allowed retry accumulation to distort performance comparisons. Some failures lacked direct drill-down, English pages retained Chinese prose, and tablet layouts collapsed into one column too early.", - "corrections": "Winner marks now compare browsers only within one model. The report separately discloses 43 network-only, 20 response-only, and 12 dual-rule failures, plus the 29/24/10/8/2/2 reviewed root-cause split. Performance and resource statistics use the 64 tasks all four configurations passed on the first attempt and only the first execution; direct result links, English gates, and responsive layouts were added.", - "remaining": "Each configuration still has one formal sample. The 14 confirmed interference tasks require a complete rerun after fixes, and no overall model or browser ranking should be published before repetitions are added." - }} + "i18n": { + "en": { + "title": "Reader audit of interpretation, comparison semantics, and bilingual layout", + "status": "Resolved", + "scope": "The report home, guide, analysis, failure drill-down, English localization, and desktop, tablet, and phone breakpoints.", + "findings": "The draft marked winners across different model configurations, conflated programmatic failure location with human-reviewed root cause, and allowed retry accumulation to distort performance comparisons. Some failures lacked direct drill-down, English pages retained Chinese prose, and tablet layouts collapsed into one column too early.", + "corrections": "Winner marks now compare browsers only within one model. The report separately discloses 43 network-only, 20 response-only, and 12 dual-rule failures, plus the 29/24/10/8/2/2 reviewed root-cause split. Performance and resource statistics use the 64 tasks all four configurations passed on the first attempt and only the first execution; direct result links, English gates, and responsive layouts were added.", + "remaining": "Each configuration still has one formal sample. The 14 confirmed interference tasks require a complete rerun after fixes, and no overall model or browser ranking should be published before repetitions are added." + } + } }, { "id": "review-2026-09-11-astra-layer-and-environment-isolation", @@ -212,14 +216,16 @@ "findings": "Sol 层为 Chromium 73/90、Moli 70/90；Astra 层两者均为 71/90。Astra Task 77 的两个失败来自共享 Shopping Admin 状态在准备后被并发执行改写，不是 Agent 误读或镜像基线错误。两个模型层使用的 browser-eval 提交不同，因此跨模型变化只能作观察性比较。", "corrections": "所有模型、浏览器和任务仍只保留一个逻辑结果；公开包补齐每次物理执行的脱敏证据与哈希；75/75 个失败均绑定当前证据并核验根因。运行器现在用按 Docker daemon 区分的跨进程文件锁覆盖 reset、Agent、证据收尾和 evaluator；同一环境 lane 串行。", "remaining": "历史官方分数不以人工核验覆盖；Task 77 只有在修复后的隔离环境中形成新的完整证据批次后才能更新成绩。真正并行执行仍要求每个 worker 拥有独立的完整站点栈、端口和浏览器容器。", - "i18n": {"en": { - "title": "Astra model layer, per-execution evidence, and environment-isolation review", - "status": "Resolved", - "scope": "The GPT-5.6 Sol and GPT-6 Astra model layers, 360 logical results, 438 physical executions, 75 final failures, and the execution boundary of the shared WebArena site stack.", - "findings": "In the Sol layer Chromium scored 73/90 and Moli 70/90; in the Astra layer both scored 71/90. The two Astra Task 77 failures came from concurrent modification of shared Shopping Admin state after preparation, not an Agent reading error or a bad image baseline. The model layers used different browser-eval commits, so cross-model changes are observational only.", - "corrections": "Kept one logical result per model, browser, and task; published sanitized evidence and hashes for every physical execution; and bound all 75 failures to verified root causes. The runner now holds a Docker-daemon-scoped cross-process lock across reset, Agent execution, evidence finalization, and evaluation, serializing every run within one environment lane.", - "remaining": "Human review does not override historical official scores. Task 77 can change only after a new complete evidence cohort is produced under the corrected isolation protocol. Safe parallel execution still requires a separate full site stack, ports, and browser containers for every worker." - }} + "i18n": { + "en": { + "title": "Astra model layer, per-execution evidence, and environment-isolation review", + "status": "Resolved", + "scope": "The GPT-5.6 Sol and GPT-6 Astra model layers, 360 logical results, 438 physical executions, 75 final failures, and the execution boundary of the shared WebArena site stack.", + "findings": "In the Sol layer Chromium scored 73/90 and Moli 70/90; in the Astra layer both scored 71/90. The two Astra Task 77 failures came from concurrent modification of shared Shopping Admin state after preparation, not an Agent reading error or a bad image baseline. The model layers used different browser-eval commits, so cross-model changes are observational only.", + "corrections": "Kept one logical result per model, browser, and task; published sanitized evidence and hashes for every physical execution; and bound all 75 failures to verified root causes. The runner now holds a Docker-daemon-scoped cross-process lock across reset, Agent execution, evidence finalization, and evaluation, serializing every run within one environment lane.", + "remaining": "Human review does not override historical official scores. Task 77 can change only after a new complete evidence cohort is produced under the corrected isolation protocol. Safe parallel execution still requires a separate full site stack, ports, and browser containers for every worker." + } + } }, { "id": "review-2026-09-11-90-task-expansion", @@ -230,14 +236,16 @@ "findings": "Chromium 通过 73/90，Moli 通过 70/90；69 题共同通过、16 题共同失败、5 题结果不一致。37 个最终失败覆盖 Agent 检索与执行、站点依赖、官方验收合同及 Moli 网络证据采集四类责任层，不能用“模型能力不行”统一解释。", "corrections": "逐结果绑定官方合同、最终回答、HAR 和 Agent 轨迹，完成 37/37 根因核验；报告改用 90 题动态口径并披露 220/220 环境准备回执。运行器对 Agent 启动前的瞬态 BrowserContextNotLoaded 增加一次仅限会话准备的恢复，不改变逻辑结果协议。", "remaining": "Astra 未由当前 OpenAI 代理暴露，本轮保持 gpt-5.6-sol、reasoning=low。Wikipedia 与 Map 尚未进入实跑；官方 evaluator 的已核验缺陷需上游修复后以新证据复测。", - "i18n": {"en": { - "title": "90-task expansion and failure-cause review", - "status": "Resolved", - "scope": "The 90 preregistered tasks, 180 logical browser results, 220 physical executions, environment-preparation receipts, official scoring, and every final failure.", - "findings": "Chromium passed 73/90 and Moli passed 70/90. The paired results contain 69 joint passes, 16 joint failures, and five disagreements. The 37 final failures span agent retrieval or execution, site dependencies, official acceptance contracts, and Moli network-evidence collection; they do not support one blanket model-capability explanation.", - "corrections": "Bound each failed result to its official contract, final answer, HAR, and agent trajectory and completed 37/37 root-cause reviews. Updated the report to the 90-task cohort and disclosed 220/220 environment-preparation receipts. Session preparation now recovers once from a transient BrowserContextNotLoaded before the agent starts, without changing logical-result authority.", - "remaining": "This run remains on gpt-5.6-sol with reasoning=low because Astra was unavailable through the current model endpoint. Wikipedia and Map are not measured. Verified defects in the official evaluator require upstream correction and new execution evidence." - }} + "i18n": { + "en": { + "title": "90-task expansion and failure-cause review", + "status": "Resolved", + "scope": "The 90 preregistered tasks, 180 logical browser results, 220 physical executions, environment-preparation receipts, official scoring, and every final failure.", + "findings": "Chromium passed 73/90 and Moli passed 70/90. The paired results contain 69 joint passes, 16 joint failures, and five disagreements. The 37 final failures span agent retrieval or execution, site dependencies, official acceptance contracts, and Moli network-evidence collection; they do not support one blanket model-capability explanation.", + "corrections": "Bound each failed result to its official contract, final answer, HAR, and agent trajectory and completed 37/37 root-cause reviews. Updated the report to the 90-task cohort and disclosed 220/220 environment-preparation receipts. Session preparation now recovers once from a transient BrowserContextNotLoaded before the agent starts, without changing logical-result authority.", + "remaining": "This run remains on gpt-5.6-sol with reasoning=low because Astra was unavailable through the current model endpoint. Wikipedia and Map are not measured. Verified defects in the official evaluator require upstream correction and new execution evidence." + } + } }, { "id": "review-2026-09-10-shared-failure-and-expansion", @@ -248,14 +256,16 @@ "findings": "相同官方分数掩盖了不同根因：Task 68 的官方 expected 与 schema 冲突；124 缺少候选语义核验；142 的类别验收边界未定义；168 的本项目响应协议与官方 null 合同冲突；506 中 Chromium 为 HAR 采集误判，Moli 为步数耗尽且未确认完成。", "corrections": "保留 evaluator 的程序自动定位；新增按浏览器与任务记录、绑定 attempts 哈希的根因子类和原因核验，未核验字段保持为空，正式发布要求全部失败结果完成核验。另新增运行前合同一致性校验、候选语义与提交前检查、空结果状态协议、JSON 响应体即时 HAR 采集、末步工具调用 fail-closed 和可配置步数上限。历史分数不改写。", "remaining": "修复后的 #124、#168、#506 尚需形成新执行证据；#68 与 #142 退出下一轮干净 A/B，直至合同被修复或验收口径可观察。Wikipedia 与 Map 尚未在本机实跑。", - "i18n": {"en": { - "title": "Failure causes, corrections, and local-environment expansion", - "status": "Resolved", - "scope": "Automatic failure location, evidence-bound root-cause review, runtime corrections, result-page hierarchy, Colima network identity, and the locally runnable environment set.", - "findings": "The ten failed logical results span task-contract defects, agent protocol or policy defects, and evidence-collection defects. Equal official scores can therefore have different causes, including different causes for the two browsers on the same task.", - "corrections": "Kept automatic evaluator location separate from reviewed causes; added per-browser, per-task cause records bound to the attempts hash and required complete failure coverage for release. Added contract admission checks, candidate and pre-submit validation, an explicit empty-result protocol, immediate JSON HAR-body capture, fail-closed final-step handling, and a configurable step ceiling. Historical scores remain unchanged.", - "remaining": "Tasks affected by runtime corrections require new execution evidence. Tasks with defective or unobservable contracts remain outside the next clean A/B until those contracts are corrected. Wikipedia and Map remain unmeasured locally." - }} + "i18n": { + "en": { + "title": "Failure causes, corrections, and local-environment expansion", + "status": "Resolved", + "scope": "Automatic failure location, evidence-bound root-cause review, runtime corrections, result-page hierarchy, Colima network identity, and the locally runnable environment set.", + "findings": "The ten failed logical results span task-contract defects, agent protocol or policy defects, and evidence-collection defects. Equal official scores can therefore have different causes, including different causes for the two browsers on the same task.", + "corrections": "Kept automatic evaluator location separate from reviewed causes; added per-browser, per-task cause records bound to the attempts hash and required complete failure coverage for release. Added contract admission checks, candidate and pre-submit validation, an explicit empty-result protocol, immediate JSON HAR-body capture, fail-closed final-step handling, and a configurable step ceiling. Historical scores remain unchanged.", + "remaining": "Tasks affected by runtime corrections require new execution evidence. Tasks with defective or unobservable contracts remain outside the next clean A/B until those contracts are corrected. Wikipedia and Map remain unmeasured locally." + } + } }, { "id": "review-2026-09-10-track-naming-and-release-gates", @@ -266,14 +276,16 @@ "findings": "轨道名称按实现层命名，读者不易判断测量对象；并列最优和显示值相同的指标仍可能被标优；英文可访问属性与失效筛选链接缺少回归保护；SVG 和跨断点布局只受人工规范约束。", "corrections": "统一命名为 Browser Use 与 Browser Trajectory Replay（轨迹重放），完成代码与命令迁移且不保留旧入口；优胜只允许显示值唯一领先；补充英文属性翻译、失效链接回退、SVG 拒绝和核心路由双语双主题宽窄屏渲染检查。", "remaining": "视觉门禁验证结构性溢出、裁切和表头竖排，仍不能替代人工对信息层级与审美质量的判断。", - "i18n": {"en": { - "title": "Track naming and release-gate review", - "status": "Resolved", - "scope": "Names for autonomous and fixed-trajectory evaluation, winner marks, English accessibility text, filter state, and visual release gates.", - "findings": "Implementation-layer names obscured what each track measures; ties or display-equal metrics could be marked as winners; English accessibility attributes and invalid filter links lacked regression coverage; and responsive layout depended on manual checks.", - "corrections": "Standardized the names Browser Use and Browser Trajectory Replay, marked only a unique displayed winner, localized English accessibility attributes, restored invalid filters to valid defaults, rejected SVG assets, and added bilingual, two-theme desktop and narrow-screen checks for core routes.", - "remaining": "Automated visual gates detect structural overflow, clipping, and vertical headers, but do not replace human review of information hierarchy and visual quality." - }} + "i18n": { + "en": { + "title": "Track naming and release-gate review", + "status": "Resolved", + "scope": "Names for autonomous and fixed-trajectory evaluation, winner marks, English accessibility text, filter state, and visual release gates.", + "findings": "Implementation-layer names obscured what each track measures; ties or display-equal metrics could be marked as winners; English accessibility attributes and invalid filter links lacked regression coverage; and responsive layout depended on manual checks.", + "corrections": "Standardized the names Browser Use and Browser Trajectory Replay, marked only a unique displayed winner, localized English accessibility attributes, restored invalid filters to valid defaults, rejected SVG assets, and added bilingual, two-theme desktop and narrow-screen checks for core routes.", + "remaining": "Automated visual gates detect structural overflow, clipping, and vertical headers, but do not replace human review of information hierarchy and visual quality." + } + } }, { "id": "review-2026-09-10-scalable-comparison-ui", @@ -284,13 +296,16 @@ "findings": "同题视图虽动态生成列，但交互仍以两浏览器关系为中心；动作表层级拥挤且对不同调用量的总耗时判优；完整 evaluator 错误被放入窄指标卡。", "corrections": "同题比较改为基准浏览器加多选对比浏览器和明确关系条件；动作延迟改为按 action 分卡并只比较单次均值；详情首屏只保留简明规则定位，完整 expected、actual 和错误进入规则证据区。", "remaining": "当前仅有两个参评浏览器；扩展行为已由三浏览器测试模型覆盖，新增真实浏览器仍需形成完整同题结果。", - "i18n": {"en": { - "title": "Multi-browser comparison and result-detail review", "status": "Resolved", - "scope": "Same-task comparison controls, BrowserTool action latency, and the opening section of result-detail pages.", - "findings": "The same-task view remained centered on a two-browser relationship; action tables ranked totals across unequal call counts; and full evaluator errors occupied narrow KPI cards.", - "corrections": "Introduced one baseline with multi-select comparison browsers and explicit outcome relations, compared action latency only by mean per call, and moved complete expected, actual, and error evidence out of the result summary.", - "remaining": "Only two browsers are measured. Three-browser fixtures cover interface scaling, but every new real browser still requires a complete same-task result set." - }} + "i18n": { + "en": { + "title": "Multi-browser comparison and result-detail review", + "status": "Resolved", + "scope": "Same-task comparison controls, BrowserTool action latency, and the opening section of result-detail pages.", + "findings": "The same-task view remained centered on a two-browser relationship; action tables ranked totals across unequal call counts; and full evaluator errors occupied narrow KPI cards.", + "corrections": "Introduced one baseline with multi-select comparison browsers and explicit outcome relations, compared action latency only by mean per call, and moved complete expected, actual, and error evidence out of the result summary.", + "remaining": "Only two browsers are measured. Three-browser fixtures cover interface scaling, but every new real browser still requires a complete same-task result set." + } + } }, { "id": "review-2026-09-10-report-model", @@ -301,13 +316,16 @@ "findings": "统计标签重复表达 P50；比较表缺少方向一致的优胜标记；结果页存在重复失败清单；核验页承担了逐结果清单职责。", "corrections": "统一为均值、P50、P95；按指标方向突出唯一优胜者；删除重复失败清单；核验页改为按轮次记录范围、发现、处理和遗留事项。", "remaining": "当前固定样本没有浏览器单边通过任务；优胜标记主要反映耗时与 Tokens，不改变官方任务成绩。", - "i18n": {"en": { - "title": "Report model and comparison review", "status": "Resolved", - "scope": "Summary statistics, winner presentation, result drill-down, review history, and browser-participant descriptions.", - "findings": "P50 was duplicated under another label, comparison tables lacked directionally consistent winner marks, the results page duplicated failure inventory, and the review page carried a per-result list.", - "corrections": "Standardized summaries on mean, P50, and P95; marked only unique directionally valid winners; removed the duplicate failure list; and made review history round-based.", - "remaining": "This fixed cohort has no one-browser-only pass. Winner marks mainly describe latency and token usage and do not alter official task scores." - }} + "i18n": { + "en": { + "title": "Report model and comparison review", + "status": "Resolved", + "scope": "Summary statistics, winner presentation, result drill-down, review history, and browser-participant descriptions.", + "findings": "P50 was duplicated under another label, comparison tables lacked directionally consistent winner marks, the results page duplicated failure inventory, and the review page carried a per-result list.", + "corrections": "Standardized summaries on mean, P50, and P95; marked only unique directionally valid winners; removed the duplicate failure list; and made review history round-based.", + "remaining": "This fixed cohort has no one-browser-only pass. Winner marks mainly describe latency and token usage and do not alter official task scores." + } + } }, { "id": "review-2026-09-10-evidence-boundaries", @@ -318,13 +336,16 @@ "findings": "环境拓扑与网关含义缺少集中说明；历史总耗时边界描述不准确；逐执行环境准备没有独立回执。", "corrections": "补充部署流程图、端口和镜像身份；区分 Agent 链路与 BrowserTool 墙钟时间；新版 runner 保存环境准备和分阶段计时。", "remaining": "本批冻结结果不具备逐执行准备回执，报告仅按现有证据声明可复现边界。", - "i18n": {"en": { - "title": "Deployment evidence and timing-boundary review", "status": "Resolved", - "scope": "Local-site deployment, the Colima network path, environment-preparation evidence, and Agent and BrowserTool timing.", - "findings": "The topology and gateway lacked a consolidated explanation, historical total-latency wording overstated its boundary, and per-execution preparation lacked an independent receipt.", - "corrections": "Added the deployment flow, ports, and image identities; separated agent-stack and BrowserTool wall time; and made the current runner retain environment-preparation and phase timings.", - "remaining": "This frozen run lacks per-execution preparation receipts, so reproducibility claims remain limited to the evidence actually retained." - }} + "i18n": { + "en": { + "title": "Deployment evidence and timing-boundary review", + "status": "Resolved", + "scope": "Local-site deployment, the Colima network path, environment-preparation evidence, and Agent and BrowserTool timing.", + "findings": "The topology and gateway lacked a consolidated explanation, historical total-latency wording overstated its boundary, and per-execution preparation lacked an independent receipt.", + "corrections": "Added the deployment flow, ports, and image identities; separated agent-stack and BrowserTool wall time; and made the current runner retain environment-preparation and phase timings.", + "remaining": "This frozen run lacks per-execution preparation receipts, so reproducibility claims remain limited to the evidence actually retained." + } + } }, { "id": "review-2026-09-09-runner-integrity", @@ -335,13 +356,16 @@ "findings": "异常清理可能遮蔽原始错误；HAR 停止操作缺少幂等保护；异常响应和原始证据目录需要更严格保护。", "corrections": "隔离清理异常、使 HAR 停止幂等、加强模型响应结构校验，并阻止原始浏览器证据进入 Git。", "remaining": "无阻塞项。", - "i18n": {"en": { - "title": "Runner and evidence-integrity review", "status": "Resolved", - "scope": "Model-response validation, BrowserTool cleanup, HAR retention, evidence directories, and resume behavior.", - "findings": "Cleanup exceptions could mask primary failures, HAR shutdown lacked idempotency, and malformed responses and raw-evidence directories required stronger protection.", - "corrections": "Isolated cleanup failures, made HAR shutdown idempotent, strengthened model-response validation, and prevented raw browser evidence from entering Git.", - "remaining": "No blocking items." - }} + "i18n": { + "en": { + "title": "Runner and evidence-integrity review", + "status": "Resolved", + "scope": "Model-response validation, BrowserTool cleanup, HAR retention, evidence directories, and resume behavior.", + "findings": "Cleanup exceptions could mask primary failures, HAR shutdown lacked idempotency, and malformed responses and raw-evidence directories required stronger protection.", + "corrections": "Isolated cleanup failures, made HAR shutdown idempotent, strengthened model-response validation, and prevented raw browser evidence from entering Git.", + "remaining": "No blocking items." + } + } } ], "browser_use_releases": [ @@ -356,17 +380,39 @@ }, "selection_sha256": "2cb716552e36df7535e83ed6d2cf3f84283e3aca86072817cd6e039a38547c7b", "changes": [ - ["跨报告比较", "相同任务编号并不保证相同验收条件；已验证轨迹也不等于自主决策。", "列出轨迹来源与合同修订范围，并绑定冻结定义的校验和。", "重放 90/90 表示给定合格轨迹与对应合同下的执行结果，不能作为仅移除 Agent 的同条件对照。"], - ["失败机制", "聚合成功率不能区分决策错误、预算限制与错误验收。", "增加 Task 124、510、676 的轨迹和验收证据入口，解释具体失败机制及归因限制。", "官方分数、90 题分母及原始耗时均保持不变；没有用诊断结果替换历史测量。"] - ], - "i18n": {"en": { - "title": "Clarify autonomous-execution and trajectory-replay comparison limits", - "summary": "Added trajectory provenance and acceptance-contract differences: 60 of 90 trajectories come from successful historical executions and 30 were expert-repaired; 81 retain the original contract and nine use revised contracts. Per-task evidence distinguishes Agent decisions, budget exhaustion, evaluator event selection, and browser execution.", - "changes": [ - ["Cross-report comparison", "Matching task IDs do not guarantee matching acceptance conditions, and executing a qualified trajectory is not autonomous planning.", "List trajectory provenance and contract revisions, bound to checksums of the frozen definitions.", "Replay's 90/90 describes execution under qualified trajectories and their contracts, not an otherwise identical experiment with only the Agent removed."], - ["Failure mechanisms", "Aggregate pass rates do not separate decision errors, budget limits, and incorrect acceptance checks.", "Add links to trajectory and evaluation evidence for tasks 124, 510, and 676, with concrete mechanisms and attribution limits.", "Official scores, the 90-task denominator, and original timings remain unchanged. Diagnostic findings do not replace historical measurements."] + [ + "跨报告比较", + "相同任务编号并不保证相同验收条件；已验证轨迹也不等于自主决策。", + "列出轨迹来源与合同修订范围，并绑定冻结定义的校验和。", + "重放 90/90 表示给定合格轨迹与对应合同下的执行结果，不能作为仅移除 Agent 的同条件对照。" + ], + [ + "失败机制", + "聚合成功率不能区分决策错误、预算限制与错误验收。", + "增加 Task 124、510、676 的轨迹和验收证据入口，解释具体失败机制及归因限制。", + "官方分数、90 题分母及原始耗时均保持不变；没有用诊断结果替换历史测量。" ] - }} + ], + "i18n": { + "en": { + "title": "Clarify autonomous-execution and trajectory-replay comparison limits", + "summary": "Added trajectory provenance and acceptance-contract differences: 60 of 90 trajectories come from successful historical executions and 30 were expert-repaired; 81 retain the original contract and nine use revised contracts. Per-task evidence distinguishes Agent decisions, budget exhaustion, evaluator event selection, and browser execution.", + "changes": [ + [ + "Cross-report comparison", + "Matching task IDs do not guarantee matching acceptance conditions, and executing a qualified trajectory is not autonomous planning.", + "List trajectory provenance and contract revisions, bound to checksums of the frozen definitions.", + "Replay's 90/90 describes execution under qualified trajectories and their contracts, not an otherwise identical experiment with only the Agent removed." + ], + [ + "Failure mechanisms", + "Aggregate pass rates do not separate decision errors, budget limits, and incorrect acceptance checks.", + "Add links to trajectory and evaluation evidence for tasks 124, 510, and 676, with concrete mechanisms and attribution limits.", + "Official scores, the 90-task denominator, and original timings remain unchanged. Diagnostic findings do not replace historical measurements." + ] + ] + } + } }, { "id": "2026-09-14-restore-complete-90-task-results", @@ -668,16 +714,38 @@ "发布质量门槛不降低，常规迭代时间缩短。" ] ], - "i18n": {"en": { - "title": "Reorder the results report and migrate to production Sites", - "summary": "The home page now starts with the four core score and latency rows, followed by same-task differences, verified causes for 75 failures, and results for the three task types. This release changes presentation, checks, and the publication target only; no evaluation score changes.", - "changes": [ - ["Home-page structure", "Core results were split across cards, making comparison difficult.", "Replaced them with one model-by-browser core-metrics table at the start of the home page.", "Pass count, pass rate, agent total time, BrowserTool time, and tokens can now be compared directly."], - ["Result interpretation", "Several short labels required implementation context.", "Replaced them with explicit sentences and column names for paired outcomes, failure causes, scope, and limitations.", "Calculation rules are unchanged; the report is easier to understand."], - ["Failure causes", "The 75 failure causes were spread across a large card grid.", "Grouped them into a table covering agent, task or evaluator, test site, browser evidence, shared environment, and mixed causes, while retaining links to every reviewed result.", "Each failure is still counted once, and its reviewed cause and evidence remain unchanged."], - ["Checks and publication", "Routine edits also launched the complete browser-based visual suite.", "Routine checks now cover data, structure, links, and copy; packaging still runs every language, theme, and viewport visual check.", "Release quality gates remain intact while ordinary iteration is faster."] - ] - }} + "i18n": { + "en": { + "title": "Reorder the results report and migrate to production Sites", + "summary": "The home page now starts with the four core score and latency rows, followed by same-task differences, verified causes for 75 failures, and results for the three task types. This release changes presentation, checks, and the publication target only; no evaluation score changes.", + "changes": [ + [ + "Home-page structure", + "Core results were split across cards, making comparison difficult.", + "Replaced them with one model-by-browser core-metrics table at the start of the home page.", + "Pass count, pass rate, agent total time, BrowserTool time, and tokens can now be compared directly." + ], + [ + "Result interpretation", + "Several short labels required implementation context.", + "Replaced them with explicit sentences and column names for paired outcomes, failure causes, scope, and limitations.", + "Calculation rules are unchanged; the report is easier to understand." + ], + [ + "Failure causes", + "The 75 failure causes were spread across a large card grid.", + "Grouped them into a table covering agent, task or evaluator, test site, browser evidence, shared environment, and mixed causes, while retaining links to every reviewed result.", + "Each failure is still counted once, and its reviewed cause and evidence remain unchanged." + ], + [ + "Checks and publication", + "Routine edits also launched the complete browser-based visual suite.", + "Routine checks now cover data, structure, links, and copy; packaging still runs every language, theme, and viewport visual check.", + "Release quality gates remain intact while ordinary iteration is faster." + ] + ] + } + } }, { "id": "2026-09-11-astra-two-model-browser-comparison", @@ -721,42 +789,44 @@ "读者可重算累计值；HAR、Playwright trace、Cookie 和凭据继续不公开。" ] ], - "i18n": {"en": { - "title": "Add GPT-6 Astra and complete a two-model browser comparison", - "summary": "Chromium and Moli are compared within two model layers on the same 90 tasks: 73/90 versus 70/90 with GPT-5.6 Sol, and 71/90 versus 71/90 with GPT-6 Astra. All 75 final failures have evidence-bound root-cause reviews; verified environment contamination does not manually change official scores.", - "changes": [ - [ - "Two model layers", - "The previous release contained only GPT-5.6 Sol, so it could not show whether browser differences persisted after changing the model.", - "Added an independent GPT-6 Astra low evidence layer. Models are not pooled; browsers are paired task by task only within each model layer.", - "In the Astra layer Chromium and Moli both scored 71/90, with two browser-only passes on each side and exact McNemar p=1.0; no significant pass-rate difference was observed." - ], - [ - "Cross-model boundary", - "The model layers were separate runs and used different browser-eval commits.", - "The report retains each layer's run ID, commit, manifest, and attempts hashes and labels cross-model differences as observational.", - "Sol–Astra score or latency changes cannot be attributed to the model version alone." - ], - [ - "Failure causes", - "Official evaluator messages identify unmet rules but not the responsible layer.", - "Reviewed the contract, final answer, network evidence, and autonomous trajectory for all 75 failed logical results. Automatic classification remains available, while detailed causes stay blank until verified.", - "All 75 failures are reviewed with none unresolved, spanning Agent, site, environment isolation, evaluator contracts, and browser evidence capture." - ], - [ - "Environment isolation and parallelism", - "The shared fixed-port site stack allowed another process to mutate state after reset and before the Agent observed it.", - "Added a cross-process lock across reset, browser session, CowAgent, evidence finalization, and evaluation. Parallel execution is allowed only with separate full site stacks.", - "The fix protects future runs. Historical Task 77 scores remain unchanged until replaced by a new evidence cohort." - ], - [ - "Public evidence", - "Logical results accumulated every execution's latency and cost, but the previous page exposed evidence only for the score-authoritative execution.", - "The report still shows one logical result per model, browser, and task, while publishing sanitized responses, evaluator output, events, trajectories, metrics, and hashes for all 438 physical executions.", - "Readers can recompute aggregates; HAR, Playwright trace, cookies, and credentials remain private." + "i18n": { + "en": { + "title": "Add GPT-6 Astra and complete a two-model browser comparison", + "summary": "Chromium and Moli are compared within two model layers on the same 90 tasks: 73/90 versus 70/90 with GPT-5.6 Sol, and 71/90 versus 71/90 with GPT-6 Astra. All 75 final failures have evidence-bound root-cause reviews; verified environment contamination does not manually change official scores.", + "changes": [ + [ + "Two model layers", + "The previous release contained only GPT-5.6 Sol, so it could not show whether browser differences persisted after changing the model.", + "Added an independent GPT-6 Astra low evidence layer. Models are not pooled; browsers are paired task by task only within each model layer.", + "In the Astra layer Chromium and Moli both scored 71/90, with two browser-only passes on each side and exact McNemar p=1.0; no significant pass-rate difference was observed." + ], + [ + "Cross-model boundary", + "The model layers were separate runs and used different browser-eval commits.", + "The report retains each layer's run ID, commit, manifest, and attempts hashes and labels cross-model differences as observational.", + "Sol–Astra score or latency changes cannot be attributed to the model version alone." + ], + [ + "Failure causes", + "Official evaluator messages identify unmet rules but not the responsible layer.", + "Reviewed the contract, final answer, network evidence, and autonomous trajectory for all 75 failed logical results. Automatic classification remains available, while detailed causes stay blank until verified.", + "All 75 failures are reviewed with none unresolved, spanning Agent, site, environment isolation, evaluator contracts, and browser evidence capture." + ], + [ + "Environment isolation and parallelism", + "The shared fixed-port site stack allowed another process to mutate state after reset and before the Agent observed it.", + "Added a cross-process lock across reset, browser session, CowAgent, evidence finalization, and evaluation. Parallel execution is allowed only with separate full site stacks.", + "The fix protects future runs. Historical Task 77 scores remain unchanged until replaced by a new evidence cohort." + ], + [ + "Public evidence", + "Logical results accumulated every execution's latency and cost, but the previous page exposed evidence only for the score-authoritative execution.", + "The report still shows one logical result per model, browser, and task, while publishing sanitized responses, evaluator output, events, trajectories, metrics, and hashes for all 438 physical executions.", + "Readers can recompute aggregates; HAR, Playwright trace, cookies, and credentials remain private." + ] ] - ] - }} + } + } }, { "id": "2026-09-11-90-task-browser-use-comparison", @@ -791,16 +861,38 @@ "支持确认本批确在自托管站点执行；不代表 Wikipedia 与 Map 已部署。" ] ], - "i18n": {"en": { - "title": "Expand Browser Use to 90 tasks", - "summary": "The preregistered cohort now contains 90 tasks: 15 easy, 30 medium, and 45 hard. Chromium passed 73 and Moli passed 70. All 37 final failures have evidence-bound root-cause reviews; human review does not override official scores.", - "changes": [ - ["Evaluation scope", "The previous 30-task cohort did not provide enough observations across short, medium, and long workflows.", "Expanded to 90 outcome-blind tasks, preregistered by minimum visible UI actions as 15 easy, 30 medium, and 45 hard across Shopping, Shopping Admin, Reddit, and GitLab.", "The denominator increased from 30 to 90. Tiers describe project-defined operation length, not official difficulty."], - ["Official results", "The previous cohort gave Chromium and Moli 25/30 each.", "This cohort gives Chromium 73/90 and Moli 70/90: 69 joint passes, 16 joint failures, four Chromium-only passes, and one Moli-only pass.", "Results apply only to the current CowAgent, gpt-5.6-sol low, four local sites, and frozen run protocol; they do not generalize to the full benchmark."], - ["Failure causes", "The programmatic evaluator locates failed rules but cannot by itself identify the responsible layer.", "Reviewed the contract, answer, network evidence, and autonomous trajectory for all 37 failed logical results and recorded a specific cause subtype and rationale.", "All 37 failures are reviewed with none unresolved; human review does not change official scores."], - ["Environment evidence", "The prior cohort lacked preparation receipts bound to each execution.", "All 220 physical executions retain reset, readiness, application-health, and sign-in preparation status, alongside the Colima gateway and local image topology.", "This establishes that the cohort ran against self-hosted sites; it does not claim that Wikipedia or Map was deployed."] - ] - }} + "i18n": { + "en": { + "title": "Expand Browser Use to 90 tasks", + "summary": "The preregistered cohort now contains 90 tasks: 15 easy, 30 medium, and 45 hard. Chromium passed 73 and Moli passed 70. All 37 final failures have evidence-bound root-cause reviews; human review does not override official scores.", + "changes": [ + [ + "Evaluation scope", + "The previous 30-task cohort did not provide enough observations across short, medium, and long workflows.", + "Expanded to 90 outcome-blind tasks, preregistered by minimum visible UI actions as 15 easy, 30 medium, and 45 hard across Shopping, Shopping Admin, Reddit, and GitLab.", + "The denominator increased from 30 to 90. Tiers describe project-defined operation length, not official difficulty." + ], + [ + "Official results", + "The previous cohort gave Chromium and Moli 25/30 each.", + "This cohort gives Chromium 73/90 and Moli 70/90: 69 joint passes, 16 joint failures, four Chromium-only passes, and one Moli-only pass.", + "Results apply only to the current CowAgent, gpt-5.6-sol low, four local sites, and frozen run protocol; they do not generalize to the full benchmark." + ], + [ + "Failure causes", + "The programmatic evaluator locates failed rules but cannot by itself identify the responsible layer.", + "Reviewed the contract, answer, network evidence, and autonomous trajectory for all 37 failed logical results and recorded a specific cause subtype and rationale.", + "All 37 failures are reviewed with none unresolved; human review does not change official scores." + ], + [ + "Environment evidence", + "The prior cohort lacked preparation receipts bound to each execution.", + "All 220 physical executions retain reset, readiness, application-health, and sign-in preparation status, alongside the Colima gateway and local image topology.", + "This establishes that the cohort ran against self-hosted sites; it does not claim that Wikipedia or Map was deployed." + ] + ] + } + } }, { "id": "2026-09-10-shared-failure-diagnosis-and-mac-expansion", @@ -847,18 +939,50 @@ "Wikipedia、Map 仍标为尚未实跑，不把理论可部署写成已验证。" ] ], - "i18n": {"en": { - "title": "Trace failures to causes and apply runtime corrections", - "summary": "Ten failed logical results were reviewed across task contracts, agent protocol or policy, and evidence collection. Automatic failure location and reviewed causes are presented separately and bound to frozen evidence. Runtime corrections await validation by new executions; historical official scores remain 25/30 for each browser.", - "changes": [ - ["Report interpretation", "Equal totals did not explain outcomes, and equal zero scores did not imply equal causes.", "Made cause, owning layer, evidence chain, corrective action, and retest gate central to the report interpretation.", "The explanatory hierarchy changed; scores, denominators, and frozen evidence did not."], - ["Shared failures", "Earlier interpretation stopped at symptoms rather than locating the contract, policy, or collector responsible.", "Used official contracts, final answers, evaluator output, HAR, and events to identify four cause classes, including browser-specific causes on Task 506.", "No hidden model reasoning is inferred; historical official scores remain unchanged."], - ["Cause review", "Automatic evaluators locate an unmet acceptance type but do not prove its cause.", "Stored per-browser, per-task causes bound to the attempts hash; unreviewed fields remain blank and complete failure coverage is required for release.", "All ten failed logical results in this version are reviewed; this does not change official scores."], - ["Runtime corrections", "Invalid contracts and weaknesses in empty-result handling, candidate aggregation, HAR capture, and final-step handling could create avoidable failures or false negatives.", "Added contract admission, candidate and pre-submit checks, immediate JSON HAR capture, fail-closed final-step handling, and a recorded step ceiling.", "Code gates passed, but any task-outcome improvement still requires new execution evidence."], - ["Result details", "Contracts, answers, and rule results were duplicated while raw JSON dominated the reading order.", "Combined them into a task-and-acceptance section with readable conclusions first and technical values collapsed.", "Public evidence and download links remain unchanged."], - ["Local environment and expansion", "The Colima gateway and the scope of locally runnable official environments were unclear.", "Distinguished VM and host gateway roles, documented the four measured sites, and corrected the available candidate counts.", "Wikipedia and Map remain explicitly unmeasured; deployability is not presented as verification."] - ] - }} + "i18n": { + "en": { + "title": "Trace failures to causes and apply runtime corrections", + "summary": "Ten failed logical results were reviewed across task contracts, agent protocol or policy, and evidence collection. Automatic failure location and reviewed causes are presented separately and bound to frozen evidence. Runtime corrections await validation by new executions; historical official scores remain 25/30 for each browser.", + "changes": [ + [ + "Report interpretation", + "Equal totals did not explain outcomes, and equal zero scores did not imply equal causes.", + "Made cause, owning layer, evidence chain, corrective action, and retest gate central to the report interpretation.", + "The explanatory hierarchy changed; scores, denominators, and frozen evidence did not." + ], + [ + "Shared failures", + "Earlier interpretation stopped at symptoms rather than locating the contract, policy, or collector responsible.", + "Used official contracts, final answers, evaluator output, HAR, and events to identify four cause classes, including browser-specific causes on Task 506.", + "No hidden model reasoning is inferred; historical official scores remain unchanged." + ], + [ + "Cause review", + "Automatic evaluators locate an unmet acceptance type but do not prove its cause.", + "Stored per-browser, per-task causes bound to the attempts hash; unreviewed fields remain blank and complete failure coverage is required for release.", + "All ten failed logical results in this version are reviewed; this does not change official scores." + ], + [ + "Runtime corrections", + "Invalid contracts and weaknesses in empty-result handling, candidate aggregation, HAR capture, and final-step handling could create avoidable failures or false negatives.", + "Added contract admission, candidate and pre-submit checks, immediate JSON HAR capture, fail-closed final-step handling, and a recorded step ceiling.", + "Code gates passed, but any task-outcome improvement still requires new execution evidence." + ], + [ + "Result details", + "Contracts, answers, and rule results were duplicated while raw JSON dominated the reading order.", + "Combined them into a task-and-acceptance section with readable conclusions first and technical values collapsed.", + "Public evidence and download links remain unchanged." + ], + [ + "Local environment and expansion", + "The Colima gateway and the scope of locally runnable official environments were unclear.", + "Distinguished VM and host gateway roles, documented the four measured sites, and corrected the available candidate counts.", + "Wikipedia and Map remain explicitly unmeasured; deployability is not presented as verification." + ] + ] + } + } }, { "id": "2026-09-10-track-naming-and-release-hardening", @@ -887,15 +1011,32 @@ "新增 104 个渲染状态的结构性视觉门禁；不改变报告数据。" ] ], - "i18n": {"en": { - "title": "Standardize track names and strengthen release checks", - "summary": "The autonomous track is Browser Use and the fixed-action track is Browser Trajectory Replay. Winner marks, English accessibility text, filter recovery, asset format, and responsive layout now have automated gates. Scores, denominators, and evidence are unchanged.", - "changes": [ - ["Track names", "Earlier names mixed implementation layers with evaluation subjects.", "Renamed the tracks Browser Use and Browser Trajectory Replay across code, commands, configuration, and publication paths without compatibility aliases.", "Only names and entry points changed; tasks, runs, and scoring did not."], - ["Comparison and localization", "Ties or display-equal values could be marked as winners, and some English attributes retained Chinese.", "Mark only uniquely better displayed values; localize titles and accessibility attributes; recover invalid browser filters to a valid default.", "Reader interface defects were corrected without changing measurements or scores."], - ["Regression gates", "Naming, graphics, and responsive layout relied mainly on manual memory.", "Added retired-name checks, SVG rejection, and bilingual light/dark desktop and narrow-screen rendering checks.", "The 104 structural render states do not change report data."] - ] - }} + "i18n": { + "en": { + "title": "Standardize track names and strengthen release checks", + "summary": "The autonomous track is Browser Use and the fixed-action track is Browser Trajectory Replay. Winner marks, English accessibility text, filter recovery, asset format, and responsive layout now have automated gates. Scores, denominators, and evidence are unchanged.", + "changes": [ + [ + "Track names", + "Earlier names mixed implementation layers with evaluation subjects.", + "Renamed the tracks Browser Use and Browser Trajectory Replay across code, commands, configuration, and publication paths without compatibility aliases.", + "Only names and entry points changed; tasks, runs, and scoring did not." + ], + [ + "Comparison and localization", + "Ties or display-equal values could be marked as winners, and some English attributes retained Chinese.", + "Mark only uniquely better displayed values; localize titles and accessibility attributes; recover invalid browser filters to a valid default.", + "Reader interface defects were corrected without changing measurements or scores." + ], + [ + "Regression gates", + "Naming, graphics, and responsive layout relied mainly on manual memory.", + "Added retired-name checks, SVG rejection, and bilingual light/dark desktop and narrow-screen rendering checks.", + "The 104 structural render states do not change report data." + ] + ] + } + } }, { "id": "2026-09-10-comparison-and-review-presentation", @@ -936,17 +1077,44 @@ "修正比较解释与页面层级，不改变测量值、判分或证据。" ] ], - "i18n": {"en": { - "title": "Standardize comparison statistics and scalable browser comparison", - "summary": "Comparison summaries now use mean, P50, and P95. Same-task comparison uses a baseline plus multiple comparison browsers. Action latency ranks mean per call only, and detailed evaluator evidence no longer appears in KPI cards. Both official scores remain 25/30.", - "changes": [ - ["Summary statistics", "Latency and token summaries lacked a consistent central and tail structure.", "Show mean, P50, and P95 for each metric without duplicating P50 under another name.", "Only presentation changed; scores, denominators, and evidence did not."], - ["Comparison presentation", "Readers had to infer metric direction cell by cell.", "Mark higher passes and pass rates and lower latency and token usage as better; keep ties and call counts neutral.", "Winner marks are descriptive and do not change task verdicts."], - ["Page responsibilities", "A failure list duplicated result filters, and review history contained a per-result inventory.", "Keep pass/fail filters and details in Results; keep only review rounds, corrections, and remaining items in Review.", "Duplicate presentation was removed without deleting evidence."], - ["Multi-browser comparison", "The matrix widened per browser and its filter could not compare one baseline with several browsers.", "Use one baseline, multi-select comparison browsers, explicit outcome relations, and linked pass/fail browser groups.", "The interface can add browsers without changing current results."], - ["Action latency and result details", "Action cells mixed three metrics and ranked totals despite unequal call counts; long errors disrupted result summaries.", "Use one card per action, rank only mean per call, keep totals and counts neutral, and retain full diagnostics in rule evidence.", "Interpretation and hierarchy changed; measurements, verdicts, and evidence did not."] - ] - }} + "i18n": { + "en": { + "title": "Standardize comparison statistics and scalable browser comparison", + "summary": "Comparison summaries now use mean, P50, and P95. Same-task comparison uses a baseline plus multiple comparison browsers. Action latency ranks mean per call only, and detailed evaluator evidence no longer appears in KPI cards. Both official scores remain 25/30.", + "changes": [ + [ + "Summary statistics", + "Latency and token summaries lacked a consistent central and tail structure.", + "Show mean, P50, and P95 for each metric without duplicating P50 under another name.", + "Only presentation changed; scores, denominators, and evidence did not." + ], + [ + "Comparison presentation", + "Readers had to infer metric direction cell by cell.", + "Mark higher passes and pass rates and lower latency and token usage as better; keep ties and call counts neutral.", + "Winner marks are descriptive and do not change task verdicts." + ], + [ + "Page responsibilities", + "A failure list duplicated result filters, and review history contained a per-result inventory.", + "Keep pass/fail filters and details in Results; keep only review rounds, corrections, and remaining items in Review.", + "Duplicate presentation was removed without deleting evidence." + ], + [ + "Multi-browser comparison", + "The matrix widened per browser and its filter could not compare one baseline with several browsers.", + "Use one baseline, multi-select comparison browsers, explicit outcome relations, and linked pass/fail browser groups.", + "The interface can add browsers without changing current results." + ], + [ + "Action latency and result details", + "Action cells mixed three metrics and ranked totals despite unequal call counts; long errors disrupted result summaries.", + "Use one card per action, rank only mean per call, keep totals and counts neutral, and retain full diagnostics in rule evidence.", + "Interpretation and hierarchy changed; measurements, verdicts, and evidence did not." + ] + ] + } + } }, { "id": "2026-09-10-environment-timing-and-report-model-correction", @@ -987,17 +1155,44 @@ "降低结论强度；不改变 evaluator 分数。" ] ], - "i18n": {"en": { - "title": "Correct deployment evidence, timing boundaries, and report responsibilities", - "summary": "The report documents the four WebArena-Verified sites deployed in Colima/Docker and the private host gateway used by browser containers. Dataset pages contain definitions only; result pages add agent end-to-end and BrowserTool latency. Both scores remain 25/30.", - "changes": [ - ["Frozen protocol", "The renderer chose the last execution even though the frozen manifest made the primary run authoritative.", "Read result authority from the frozen manifest and restore the matching result, answer, and trajectory.", "Neither browser recovered on later executions, so scores remain 25/30; only result identity and evidence semantics changed."], - ["Deployment environment", "The diagram did not locate the four sites, explain the private gateway, or disclose missing preparation receipts.", "Document the macOS, Colima/Docker, site-container, and browser-container topology, ports, image identities, and evidence boundary.", "The 30 tasks cover four sites only; Wikipedia and Map are neither deployed nor scored. No results were rerun."], - ["Timing boundary", "Earlier wording incorrectly included site reset in end-to-end latency and did not separate browser-tool time.", "Exclude the preceding site reset, aggregate BrowserTool wall time from events, and retain phase timings for future runs.", "Only diagnostics and wording changed; BrowserTool wall time still includes more than browser-core CPU time."], - ["Report model", "Dataset pages mixed outcomes with definitions, result tables assumed two browsers, and participants were unclear.", "Keep contracts in Dataset, render browsers dynamically in Results, and distinguish browser participants from shared controls.", "Information ownership and extensibility changed; current data and scores did not."], - ["Review boundary", "Evidence-integrity checks could be mistaken for independent semantic review.", "Show evidence integrity and semantic-review coverage separately and remove population-style confidence intervals from the fixed cohort.", "Claims were narrowed without changing evaluator scores."] - ] - }} + "i18n": { + "en": { + "title": "Correct deployment evidence, timing boundaries, and report responsibilities", + "summary": "The report documents the four WebArena-Verified sites deployed in Colima/Docker and the private host gateway used by browser containers. Dataset pages contain definitions only; result pages add agent end-to-end and BrowserTool latency. Both scores remain 25/30.", + "changes": [ + [ + "Frozen protocol", + "The renderer chose the last execution even though the frozen manifest made the primary run authoritative.", + "Read result authority from the frozen manifest and restore the matching result, answer, and trajectory.", + "Neither browser recovered on later executions, so scores remain 25/30; only result identity and evidence semantics changed." + ], + [ + "Deployment environment", + "The diagram did not locate the four sites, explain the private gateway, or disclose missing preparation receipts.", + "Document the macOS, Colima/Docker, site-container, and browser-container topology, ports, image identities, and evidence boundary.", + "The 30 tasks cover four sites only; Wikipedia and Map are neither deployed nor scored. No results were rerun." + ], + [ + "Timing boundary", + "Earlier wording incorrectly included site reset in end-to-end latency and did not separate browser-tool time.", + "Exclude the preceding site reset, aggregate BrowserTool wall time from events, and retain phase timings for future runs.", + "Only diagnostics and wording changed; BrowserTool wall time still includes more than browser-core CPU time." + ], + [ + "Report model", + "Dataset pages mixed outcomes with definitions, result tables assumed two browsers, and participants were unclear.", + "Keep contracts in Dataset, render browsers dynamically in Results, and distinguish browser participants from shared controls.", + "Information ownership and extensibility changed; current data and scores did not." + ], + [ + "Review boundary", + "Evidence-integrity checks could be mistaken for independent semantic review.", + "Show evidence integrity and semantic-review coverage separately and remove population-style confidence intervals from the fixed cohort.", + "Claims were narrowed without changing evaluator scores." + ] + ] + } + } }, { "id": "2026-09-10-logical-results-and-publication-qa", @@ -1032,16 +1227,38 @@ "后续报告若重新出现这些已知问题，将在发布前检查中失败。" ] ], - "i18n": {"en": { - "title": "Unify logical-result semantics and complete report readability corrections", - "summary": "The public report now exposes one logical result per task and browser, with all physical-execution latency and model usage accumulated. Scores remain 25/30 for both browsers. Dense tables, code contrast, diagrams, narrow layouts, and release history now share one publication gate.", - "changes": [ - ["Logical results", "Physical executions were presented as separate headline results.", "Publish one logical result per task and browser; let the final allowed execution decide the outcome while accumulating every execution's cost.", "Scores and denominators remain unchanged and physical evidence is retained."], - ["Readability", "Short table headers wrapped vertically, code contrast was weak, and the text-heavy diagram was unreadable at normal size.", "Add semantic column widths and scrolling, correct theme contrast, replace the diagram with a readable raster asset, and inspect desktop and narrow layouts.", "Only presentation changed."], - ["Release history", "The page lacked a current summary, compact archive, effect boundaries, and a structured download.", "Use a report-style current release, expandable details, compact history, and structured data.", "Release history became part of the report without changing evidence or results."], - ["Repeatable publication gate", "Manual memory could not reliably prevent known regressions.", "Encode logical-result, privacy, table, contrast, responsive, and changelog requirements in automated release checks.", "Known regressions now fail before publication."] - ] - }} + "i18n": { + "en": { + "title": "Unify logical-result semantics and complete report readability corrections", + "summary": "The public report now exposes one logical result per task and browser, with all physical-execution latency and model usage accumulated. Scores remain 25/30 for both browsers. Dense tables, code contrast, diagrams, narrow layouts, and release history now share one publication gate.", + "changes": [ + [ + "Logical results", + "Physical executions were presented as separate headline results.", + "Publish one logical result per task and browser; let the final allowed execution decide the outcome while accumulating every execution's cost.", + "Scores and denominators remain unchanged and physical evidence is retained." + ], + [ + "Readability", + "Short table headers wrapped vertically, code contrast was weak, and the text-heavy diagram was unreadable at normal size.", + "Add semantic column widths and scrolling, correct theme contrast, replace the diagram with a readable raster asset, and inspect desktop and narrow layouts.", + "Only presentation changed." + ], + [ + "Release history", + "The page lacked a current summary, compact archive, effect boundaries, and a structured download.", + "Use a report-style current release, expandable details, compact history, and structured data.", + "Release history became part of the report without changing evidence or results." + ], + [ + "Repeatable publication gate", + "Manual memory could not reliably prevent known regressions.", + "Encode logical-result, privacy, table, contrast, responsive, and changelog requirements in automated release checks.", + "Known regressions now fail before publication." + ] + ] + } + } }, { "id": "2026-09-10-deployment-and-autonomy-evidence", @@ -1064,14 +1281,26 @@ "既有测量不重跑；新增说明直接绑定现有 manifest、事件、HAR 与 trace 哈希。" ] ], - "i18n": {"en": { - "title": "Complete deployment and autonomous-execution evidence", - "summary": "The title now identifies Chromium and Moli as the Browser Use participants. Methodology adds the full path from local WebArena-Verified deployment and per-execution reset through the CowAgent loop, official acceptance, and retained evidence. Measurements are unchanged.", - "changes": [ - ["Report title", "The earlier title foregrounded a control variable rather than the evaluated browsers.", "Make Chromium and Moli the title subjects and present CowAgent, model, reasoning, and tasks as conditions.", "Narrative hierarchy changed; design and results did not."], - ["Deployment and autonomy evidence", "Reset and autonomous planning were asserted without showing deployment ownership or the agent loop.", "Add a bilingual flow and evidence table linking site startup, health checks, sign-in, and dynamic BrowserTool calls.", "No measurements were rerun; the explanation binds to existing manifests, events, HAR, and trace hashes."] - ] - }} + "i18n": { + "en": { + "title": "Complete deployment and autonomous-execution evidence", + "summary": "The title now identifies Chromium and Moli as the Browser Use participants. Methodology adds the full path from local WebArena-Verified deployment and per-execution reset through the CowAgent loop, official acceptance, and retained evidence. Measurements are unchanged.", + "changes": [ + [ + "Report title", + "The earlier title foregrounded a control variable rather than the evaluated browsers.", + "Make Chromium and Moli the title subjects and present CowAgent, model, reasoning, and tasks as conditions.", + "Narrative hierarchy changed; design and results did not." + ], + [ + "Deployment and autonomy evidence", + "Reset and autonomous planning were asserted without showing deployment ownership or the agent loop.", + "Add a bilingual flow and evidence table linking site startup, health checks, sign-in, and dynamic BrowserTool calls.", + "No measurements were rerun; the explanation binds to existing manifests, events, HAR, and trace hashes." + ] + ] + } + } }, { "id": "2026-09-10-report-evidence-restructure", @@ -1106,16 +1335,38 @@ "缺失或错配 selection、模型、reasoning、官方数据或证据审计时禁止发布。" ] ], - "i18n": {"en": { - "title": "Restructure the Browser Use report around the evidence chain", - "summary": "The report adds task-family coverage, Wilson intervals, 30 paired-task comparisons, and route-for-route English pages. It correctly labels deterministic evidence-integrity review and avoids attributing five shared failures without a successful path. Measurements are unchanged.", - "changes": [ - ["Home and results", "The home page did not jointly expose cohort coverage, uncertainty, call completeness, and paired outcomes.", "Add task families, Wilson intervals, solvability coverage, evidence-audit status, and a filterable 30-task Chromium/Moli comparison.", "Only explanation and drill-down changed; 25/30 scores and 30 pairs did not."], - ["Review terminology and attribution", "Deterministic consistency checks were called independent result review, and shared failures were attributed too early.", "Call the process evidence-integrity review and require separate non-scoring solvability review for tasks without a successful trajectory.", "The official evaluator remains the sole scoring authority; shared failures are no longer presented as confirmed causes."], - ["Bilingual and mobile", "English was summary-only and fixed card spans clipped narrow layouts.", "Generate an English counterpart for every Chinese route, verify links, and force cards to one column on narrow screens.", "Both languages use the same measured data; all 206 logical pages had valid links."], - ["Build and publication gates", "A formal build could silently degrade without official data, while routine checks did not require complete evidence.", "Bind builds to the official checksum and current publication profile, require complete evidence, and replace output atomically with a marker.", "Publication fails on missing or mismatched selection, model, reasoning, official data, or evidence review."] - ] - }} + "i18n": { + "en": { + "title": "Restructure the Browser Use report around the evidence chain", + "summary": "The report adds task-family coverage, Wilson intervals, 30 paired-task comparisons, and route-for-route English pages. It correctly labels deterministic evidence-integrity review and avoids attributing five shared failures without a successful path. Measurements are unchanged.", + "changes": [ + [ + "Home and results", + "The home page did not jointly expose cohort coverage, uncertainty, call completeness, and paired outcomes.", + "Add task families, Wilson intervals, solvability coverage, evidence-audit status, and a filterable 30-task Chromium/Moli comparison.", + "Only explanation and drill-down changed; 25/30 scores and 30 pairs did not." + ], + [ + "Review terminology and attribution", + "Deterministic consistency checks were called independent result review, and shared failures were attributed too early.", + "Call the process evidence-integrity review and require separate non-scoring solvability review for tasks without a successful trajectory.", + "The official evaluator remains the sole scoring authority; shared failures are no longer presented as confirmed causes." + ], + [ + "Bilingual and mobile", + "English was summary-only and fixed card spans clipped narrow layouts.", + "Generate an English counterpart for every Chinese route, verify links, and force cards to one column on narrow screens.", + "Both languages use the same measured data; all 206 logical pages had valid links." + ], + [ + "Build and publication gates", + "A formal build could silently degrade without official data, while routine checks did not require complete evidence.", + "Bind builds to the official checksum and current publication profile, require complete evidence, and replace output atomically with a marker.", + "Publication fails on missing or mismatched selection, model, reasoning, official data, or evidence review." + ] + ] + } + } }, { "id": "2026-09-09-webarena-verified-pilot", @@ -1162,18 +1413,50 @@ "本轮是 GPT-5.6 条件下的正式对比，不能写成 Astra 成绩；Astra 获权后必须新建独立运行。" ] ], - "i18n": {"en": { - "title": "Complete the WebArena-Verified autonomous-agent browser comparison", - "summary": "CowAgent with gpt-5.6-sol at reasoning=low produced 60 logical results across 30 preregistered tasks in Chromium and Moli. Each browser passed 25 tasks. Deterministic evidence-integrity checks covered every result with no unresolved inconsistency.", - "changes": [ - ["Evidence and interpretation", "Navigation alone did not make task selection, executions, review coverage, solvability, or conclusions auditable.", "Use the home page as a decision entry point and publish contracts, measured results, assertions, trajectories, sanitized events, an English edition, and an evidence index.", "Publication structure changed; tasks, denominator, outputs, scores, and evidence bindings did not."], - ["Autonomous-agent track", "The repository also contained a trajectory-replay harness, making the phrase 'the original 30 tasks' ambiguous.", "State that CowAgent autonomously observes, plans, and acts for this report, while trajectory replay is a separate microbenchmark.", "Only the autonomous-agent track contributes to these scores."], - ["Benchmark selection", "Browser-use, computer-use, offline action prediction, and agentic-search claims used incompatible evaluation units.", "Compare primary-source adoption and evaluation contracts across relevant benchmark families.", "Choose self-hosted WebArena-Verified for deterministic outcome evaluation; live-site and model-judge tracks remain future work."], - ["Complete formal run", "The cohort needed short, medium, and long workflows without post-run task selection.", "Preregister 5/10/15 operation-length tiers and one logical result per task-browser pair, with at most one declared execution after failure.", "Both browsers passed 25/30; five tasks failed jointly and none produced a one-browser-only pass."], - ["Evidence integrity", "Scores needed traceability to final answers, network side effects, and official evaluators without manual overrides.", "Retain responses, events, HAR, traces, and evaluator details and verify files, hashes, scores, statuses, and transport attempts.", "All logical results passed evidence-integrity checks with no scoring disagreement or unresolved item; sensitive network bodies remain private."], - ["Model-coverage boundary", "Availability in a model catalog did not establish access for this run.", "Record gpt-5.6-sol as the only measured model after Astra access was unavailable.", "This is a GPT-5.6 comparison, not an Astra result; Astra requires a separate run when available."] - ] - }} + "i18n": { + "en": { + "title": "Complete the WebArena-Verified autonomous-agent browser comparison", + "summary": "CowAgent with gpt-5.6-sol at reasoning=low produced 60 logical results across 30 preregistered tasks in Chromium and Moli. Each browser passed 25 tasks. Deterministic evidence-integrity checks covered every result with no unresolved inconsistency.", + "changes": [ + [ + "Evidence and interpretation", + "Navigation alone did not make task selection, executions, review coverage, solvability, or conclusions auditable.", + "Use the home page as a decision entry point and publish contracts, measured results, assertions, trajectories, sanitized events, an English edition, and an evidence index.", + "Publication structure changed; tasks, denominator, outputs, scores, and evidence bindings did not." + ], + [ + "Autonomous-agent track", + "The repository also contained a trajectory-replay harness, making the phrase 'the original 30 tasks' ambiguous.", + "State that CowAgent autonomously observes, plans, and acts for this report, while trajectory replay is a separate microbenchmark.", + "Only the autonomous-agent track contributes to these scores." + ], + [ + "Benchmark selection", + "Browser-use, computer-use, offline action prediction, and agentic-search claims used incompatible evaluation units.", + "Compare primary-source adoption and evaluation contracts across relevant benchmark families.", + "Choose self-hosted WebArena-Verified for deterministic outcome evaluation; live-site and model-judge tracks remain future work." + ], + [ + "Complete formal run", + "The cohort needed short, medium, and long workflows without post-run task selection.", + "Preregister 5/10/15 operation-length tiers and one logical result per task-browser pair, with at most one declared execution after failure.", + "Both browsers passed 25/30; five tasks failed jointly and none produced a one-browser-only pass." + ], + [ + "Evidence integrity", + "Scores needed traceability to final answers, network side effects, and official evaluators without manual overrides.", + "Retain responses, events, HAR, traces, and evaluator details and verify files, hashes, scores, statuses, and transport attempts.", + "All logical results passed evidence-integrity checks with no scoring disagreement or unresolved item; sensitive network bodies remain private." + ], + [ + "Model-coverage boundary", + "Availability in a model catalog did not establish access for this run.", + "Record gpt-5.6-sol as the only measured model after Astra access was unavailable.", + "This is a GPT-5.6 comparison, not an Astra result; Astra requires a separate run when available." + ] + ] + } + } } ], "review_rounds": [ @@ -1276,43 +1559,138 @@ ], "releases": [ { - "id": "2026-10-10-native-content-acceptance-correction", + "id": "2026-10-10-moli-pr919-native-430-and-complete-resource-scope", "publication_state": "published", "date": "2026-10-10", "comparison_kinds": [ "headless_markdown" ], "scoring_revision_input_sha256": [ - "47a1494ee2c6d55aeb33ce726f68fac5829ab16d6c6a077103834a06131ee4bf" + "0bbbe0264c9715da5438d05cbd5d140fe0e1a5abc4c80625e078e7b4548c3774" ], - "title": "统一原生内容接口与正文验收", - "summary": "五种浏览器仍各有 430 个结果：Lightpanda 238、Moli 227、Chromium 191、Obscura 189、Kitesurf 171。文档正文不再借助外部解析器，最终网址缺失不单独扣分。", + "title": "增加 Moli PR #919 的完整评测，修正资源统计范围", + "summary": "五种浏览器、六种配置各有 430 个结果，共 2,580 条。整体通过数：Lightpanda 238/430，Moli 稳定版 228/430，Chromium 191/430，Obscura 189/430，Moli PR #919 187/430，Kitesurf 171/430。接口说明覆盖七类任务与六种配置；资源主比较保留全部预定调用。", "changes": [ [ - "正文接口", - "Moli 风控任务含 JSON DOM 转换", - "100 个风控任务改用原生 Markdown", - "保留旧调用，不用其他接口补正文。" + "新增配置", + "此前只包含五种配置", + "Moli PR #919 在 rebase 后的提交 98be1f10 上构建，使用原生接口完成全部 430 题", + "新增 430 次调用；程序版本号为 1.1.15，报告明确标为源码分支构建。" ], [ - "文档正文", - "此前部分任务含外部 PDF/DOCX 解析", - "五种浏览器各补跑完整 10 题文档专项", - "直接文档使用原生输出；嵌入文档只读取浏览器内已加载文字，无法取得内容时保留失败。" + "资源地址", + "包含逗号的单个地址可能被误拆成多个候选", + "五种原配置补跑完整受影响范围：16 题，共 80 次调用", + "Moli 视频增加一例通过；Lightpanda 图片一例由通过变失败、另一例由失败变通过，总通过数不变。原题、答案和规则不变，旧调用保留。" ], [ - "正文验收", - "最终网址或 HTML 语言标签可能影响通过", - "仅作为诊断信息，按目标正文统一重算 2,150 个结果", - "登录、验证、错误页、空正文和缺少必需内容仍失败；原题与答案不变。" + "评分依据", + "合并新增配置和补跑结果后，旧评分记录不能直接代表新矩阵", + "逐条核对完整结果与评分依据，并保留旧评分记录", + "核对本身未新增浏览器调用，没有修改保存的输出、调用身份、成绩和耗时。" ], [ - "类型与方法表", - "风控类型名称不清晰，接口处理说明不完整", - "风控网站正文紧接热门网站正文；公开 7×5 接口表", - "获取接口、原生脚本与 Chromium HTML 转换均明示，实际路线不合规时拒绝发布。" - ] - ], + "资源统计", + "主表此前只采用四种方法共同通过的 10/20 个网页", + "主表统计全部 20 个预定网页的 80 次调用，包括失败；成功调用另作条件统计，共同通过项仅作诊断", + "不重复资源测量；公开各方法通过数，并列指标保持中性。资源消耗与任务通过率分别判断。" + ], + [ + "接口与证据", + "原方法表只有 35 个类型与配置组合", + "扩展为 42 个组合，列出官方接口、浏览器内读取脚本、字段提取和 Chromium HTML 转换", + "文档正文不使用外部 PDF/DOCX 解析器；最终网址仍只作诊断。结果详情保留输出、验收规则和程序错误。" + ], + [ + "耗时比较", + "稳定版与源码分支的调用发生在不同日期", + "分别展示完整调用与各方法成功调用的耗时及样本数", + "这些批次的差值不作为 PR 导致提速的证据；分支未加入独立的资源测量。" + ] + ], + "i18n": { + "en": { + "title": "Add the complete Moli PR #919 cohort and correct the resource population", + "summary": "Five browsers and six configurations each have 430 results, totaling 2,580. Overall passes: Lightpanda 238/430, Moli stable 228/430, Chromium 191/430, Obscura 189/430, Moli PR #919 187/430, Kitesurf 171/430. The interface matrix covers seven families and six configurations; resource comparisons retain every predeclared call.", + "changes": [ + [ + "New configuration", + "The report previously included five configurations", + "Build rebased Moli PR #919 at commit 98be1f10 and execute all 430 tasks through native interfaces", + "430 new calls. The reported version is 1.1.15, explicitly labeled as a source-branch build." + ], + [ + "Resource URLs", + "A single URL containing commas could be split into multiple candidates", + "Rerun the full affected scope for the five existing configurations: 16 tasks and 80 calls", + "Moli gains one video pass. Lightpanda loses one image pass and gains another, leaving its total unchanged. Tasks, answers and rules are unchanged; earlier calls are retained." + ], + [ + "Scoring authority", + "Earlier scoring records do not directly identify a matrix with added and replacement results", + "Verify the complete results against scoring authority and retain earlier records", + "This verification adds no browser calls and changes no saved output, call identity, score or latency." + ], + [ + "Resource population", + "The primary comparison previously selected the 10/20 pages passed by all four methods", + "Include all 80 calls across the 20 predeclared pages, including failures. Successful-call distributions are conditional; shared passes are diagnostic only", + "No resource calls are repeated. Show each pass count and keep tied values neutral. Resource consumption and task pass rate remain separate." + ], + [ + "Interfaces and evidence", + "The interface matrix had 35 family–configuration combinations", + "Expand to 42 combinations with official interfaces, in-browser reading scripts, field extraction and Chromium HTML conversion", + "External PDF/DOCX parsers do not supply document text. Final URLs remain diagnostic. Result details retain outputs, acceptance rules and exact errors." + ], + [ + "Latency comparison", + "Stable and source-branch calls were collected on different dates", + "Show all-call and each method’s successful-call latency with population counts", + "Batch differences are not evidence of PR-induced speed improvements. The branch is not part of the independent resource cohort." + ] + ] + } + } + }, + { + "id": "2026-10-10-native-content-acceptance-correction", + "publication_state": "published", + "date": "2026-10-10", + "comparison_kinds": [ + "headless_markdown" + ], + "scoring_revision_input_sha256": [ + "47a1494ee2c6d55aeb33ce726f68fac5829ab16d6c6a077103834a06131ee4bf" + ], + "title": "统一原生内容接口与正文验收", + "summary": "五种浏览器仍各有 430 个结果：Lightpanda 238、Moli 227、Chromium 191、Obscura 189、Kitesurf 171。文档正文不再借助外部解析器，最终网址缺失不单独扣分。", + "changes": [ + [ + "正文接口", + "Moli 风控任务含 JSON DOM 转换", + "100 个风控任务改用原生 Markdown", + "保留旧调用，不用其他接口补正文。" + ], + [ + "文档正文", + "此前部分任务含外部 PDF/DOCX 解析", + "五种浏览器各补跑完整 10 题文档专项", + "直接文档使用原生输出；嵌入文档只读取浏览器内已加载文字，无法取得内容时保留失败。" + ], + [ + "正文验收", + "最终网址或 HTML 语言标签可能影响通过", + "仅作为诊断信息，按目标正文统一重算 2,150 个结果", + "登录、验证、错误页、空正文和缺少必需内容仍失败；原题与答案不变。" + ], + [ + "类型与方法表", + "风控类型名称不清晰，接口处理说明不完整", + "风控网站正文紧接热门网站正文；公开 7×5 接口表", + "获取接口、原生脚本与 Chromium HTML 转换均明示，实际路线不合规时拒绝发布。" + ] + ], "i18n": { "en": { "title": "Native content interfaces and body acceptance", @@ -1346,92 +1724,91 @@ } } }, - -{ - "id": "2026-10-10-five-browsers-seven-families-430", - "publication_state": "published", - "scoring_revision_input_sha256": [ - "7a79357747f8a0a9bdf9d216667edfeaf3bedf0ff25a3685ad29f17abfa1d712", - "884edf8ca3f9111aea956f9635688f9c7af6492ac32a1966f4275c4d9f19dd21" - ], - "date": "2026-10-10", - "comparison_kinds": [ - "headless_markdown", - "headless_html" - ], - "title": "发布五种浏览器、七类任务的 430 题报告", - "summary": "每种浏览器完成 430 题，共 2,150 个结果。Lightpanda 通过 243 题，Moli 233 题，Chromium 199 题，Obscura 192 题，Kitesurf 171 题。新增风控网页正文专项；动态内容与热门站点各覆盖 100 题。", - "changes": [ - [ - "评测范围", - "此前未覆盖当前七类完整题集", - "普通网页、动态内容、热门站点、风控网页正文各 100 题，文档、图片、视频各 10 题", - "总分母为每种浏览器 430 题；旧报告保留。" - ], - [ - "文档和资源接口", - "部分调用将文档当网页、借用外部转换或未明确选择参评引擎", - "文档通过参评浏览器读取原始字节并共用正文解析；资源使用已加载页面与明确指定的引擎", - "替换全部受影响的调用，保留此前证据；题目、固定答案及评分规则不变。" - ], - [ - "耗时与资源", - "任务通过率无法说明获取耗时和资源开销", - "分别列出全部调用、通过任务的均值、P50、P95；保留 20 题四种本地浏览器的 80 次资源测量", - "提取与解析计入时限；托管 Kitesurf 的后端资源未公开。" - ], - [ - "原生接口与共同处理", - "正文格式和资源接口未完整区分，部分 CDP 调用混用了托管 API 的等待参数", - "展示七类任务与五种浏览器的 35 个接口组合；Kitesurf 资源使用官方 CDP 与共同 DOM 范围", - "Moli 全部 100 个热门站点和 Kitesurf 全部 19 个受影响任务重新执行，旧调用保留。后一次纠正使 Kitesurf 视频通过数从 5/10 改为 6/10，整体从 168/430 改为 169/430；分母不变，耗时采用修正后的完整调用。" - ], - [ - "共享评分器", - "Markdown 加号列表、相邻卡片和表头可能漏计已返回的内容", - "采用最新共享评分器重算全部保留输出，并用成对测试防止漏判和误判", - "四个动态任务结果由失败变为通过：Kitesurf 增加两例，Lightpanda 与 Obscura 各增加一例。未新增浏览器调用，题目、答案、输出和耗时均未改变。" - ] - ], - "i18n": { - "en": { - "title": "Five browsers across 430 tasks in seven families", - "summary": "All five browsers completed 430 tasks, yielding 2,150 results. Lightpanda passed 243, Moli 233, Chromium 199, Obscura 192, and Kitesurf 171. Risk-controlled webpage text is a new family; dynamic and popular-site tasks each cover 100 cases.", + { + "id": "2026-10-10-five-browsers-seven-families-430", + "publication_state": "published", + "scoring_revision_input_sha256": [ + "7a79357747f8a0a9bdf9d216667edfeaf3bedf0ff25a3685ad29f17abfa1d712", + "884edf8ca3f9111aea956f9635688f9c7af6492ac32a1966f4275c4d9f19dd21" + ], + "date": "2026-10-10", + "comparison_kinds": [ + "headless_markdown", + "headless_html" + ], + "title": "发布五种浏览器、七类任务的 430 题报告", + "summary": "每种浏览器完成 430 题，共 2,150 个结果。Lightpanda 通过 243 题，Moli 233 题，Chromium 199 题，Obscura 192 题，Kitesurf 171 题。新增风控网页正文专项；动态内容与热门站点各覆盖 100 题。", "changes": [ [ - "Scope", - "The previous cohort did not cover the current seven-family matrix", - "100 cases each for ordinary, dynamic, popular and risk-controlled webpage text; 10 each for documents, images and video", - "430 cases per browser; the earlier report is preserved." + "评测范围", + "此前未覆盖当前七类完整题集", + "普通网页、动态内容、热门站点、风控网页正文各 100 题，文档、图片、视频各 10 题", + "总分母为每种浏览器 430 题；旧报告保留。" ], [ - "Document and resource routes", - "Some calls treated documents as pages, used external conversion or omitted explicit engine selection", - "Browser-native original document bytes with shared text extraction; resources from loaded pages with explicit engine binding", - "All affected calls are replaced with prior evidence retained; tasks, fixed answers and scoring remain unchanged." + "文档和资源接口", + "部分调用将文档当网页、借用外部转换或未明确选择参评引擎", + "文档通过参评浏览器读取原始字节并共用正文解析；资源使用已加载页面与明确指定的引擎", + "替换全部受影响的调用，保留此前证据；题目、固定答案及评分规则不变。" ], [ - "Latency and resources", - "Task acceptance alone does not describe retrieval cost", - "All-call and successful-call mean, P50 and P95 latency; 80 retained resource calls across 20 cases and four local browsers", - "Extraction and parsing share the deadline; hosted Kitesurf backend resources are unavailable." + "耗时与资源", + "任务通过率无法说明获取耗时和资源开销", + "分别列出全部调用、通过任务的均值、P50、P95；保留 20 题四种本地浏览器的 80 次资源测量", + "提取与解析计入时限；托管 Kitesurf 的后端资源未公开。" ], [ - "Native interfaces and shared processing", - "Output modes and resource scope were conflated; some CDP calls received a hosted API completion value", - "Expose all 35 browser-family interface combinations; Kitesurf resources use official CDP with the same DOM scope", - "Rerun all 100 Moli popular-site tasks and all 19 affected Kitesurf tasks, preserving prior calls. The latter correction changes Kitesurf video passes from 5/10 to 6/10 and overall passes from 168/430 to 169/430. Denominators stay fixed; latency uses the corrected complete calls." + "原生接口与共同处理", + "正文格式和资源接口未完整区分，部分 CDP 调用混用了托管 API 的等待参数", + "展示七类任务与五种浏览器的 35 个接口组合；Kitesurf 资源使用官方 CDP 与共同 DOM 范围", + "Moli 全部 100 个热门站点和 Kitesurf 全部 19 个受影响任务重新执行，旧调用保留。后一次纠正使 Kitesurf 视频通过数从 5/10 改为 6/10，整体从 168/430 改为 169/430；分母不变，耗时采用修正后的完整调用。" ], [ - "Shared scoring", - "Markdown plus lists, adjacent cards and table headers could omit returned content", - "Rescore all retained outputs with the current shared scorer and paired controls against false positives and false negatives", - "Four dynamic results change from fail to pass: two for Kitesurf and one each for Lightpanda and Obscura. No browser calls are added; tasks, answers, outputs and latency remain unchanged." + "共享评分器", + "Markdown 加号列表、相邻卡片和表头可能漏计已返回的内容", + "采用最新共享评分器重算全部保留输出，并用成对测试防止漏判和误判", + "四个动态任务结果由失败变为通过：Kitesurf 增加两例，Lightpanda 与 Obscura 各增加一例。未新增浏览器调用，题目、答案、输出和耗时均未改变。" ] - ] - } - } -}, + ], + "i18n": { + "en": { + "title": "Five browsers across 430 tasks in seven families", + "summary": "All five browsers completed 430 tasks, yielding 2,150 results. Lightpanda passed 243, Moli 233, Chromium 199, Obscura 192, and Kitesurf 171. Risk-controlled webpage text is a new family; dynamic and popular-site tasks each cover 100 cases.", + "changes": [ + [ + "Scope", + "The previous cohort did not cover the current seven-family matrix", + "100 cases each for ordinary, dynamic, popular and risk-controlled webpage text; 10 each for documents, images and video", + "430 cases per browser; the earlier report is preserved." + ], + [ + "Document and resource routes", + "Some calls treated documents as pages, used external conversion or omitted explicit engine selection", + "Browser-native original document bytes with shared text extraction; resources from loaded pages with explicit engine binding", + "All affected calls are replaced with prior evidence retained; tasks, fixed answers and scoring remain unchanged." + ], + [ + "Latency and resources", + "Task acceptance alone does not describe retrieval cost", + "All-call and successful-call mean, P50 and P95 latency; 80 retained resource calls across 20 cases and four local browsers", + "Extraction and parsing share the deadline; hosted Kitesurf backend resources are unavailable." + ], + [ + "Native interfaces and shared processing", + "Output modes and resource scope were conflated; some CDP calls received a hosted API completion value", + "Expose all 35 browser-family interface combinations; Kitesurf resources use official CDP with the same DOM scope", + "Rerun all 100 Moli popular-site tasks and all 19 affected Kitesurf tasks, preserving prior calls. The latter correction changes Kitesurf video passes from 5/10 to 6/10 and overall passes from 168/430 to 169/430. Denominators stay fixed; latency uses the corrected complete calls." + ], + [ + "Shared scoring", + "Markdown plus lists, adjacent cards and table headers could omit returned content", + "Rescore all retained outputs with the current shared scorer and paired controls against false positives and false negatives", + "Four dynamic results change from fail to pass: two for Kitesurf and one each for Lightpanda and Obscura. No browser calls are added; tasks, answers, outputs and latency remain unchanged." + ] + ] + } + } + }, { "id": "2026-10-09-dynamic-1000-native-presentation", "publication_state": "published", @@ -1661,47 +2038,125 @@ "id": "webmainbench-545-pr919-e944d066-lexmain-f7cb97d-20260929", "publication_state": "published", "date": "2026-09-29", - "comparison_kinds": ["webmainbench_frozen_html_to_markdown"], + "comparison_kinds": [ + "webmainbench_frozen_html_to_markdown" + ], "title": "补齐渐显正文并按最新 lexmain 重算七方法结果", "summary": "Moli PR919 提交 e944d066b 修复一次性前向渐显页面的正文缺失，以固定 release 二进制完成 545 页认证运行。七方法共 3,815 条结果按 WebMainBench lexmain f7cb97d 的参考与固定公共总体重算；PR919 目标正文 545/545，综合分 39.41%。", "changes": [ - ["Moli 正文交付", "CNN 冻结页的正文以 opacity:0 开始，页面加载后由同名关键帧渐显；旧可见性快照将其误判为永久隐藏。", "仅在最终关键帧明确恢复不透明度且动画为正时长、无正延迟、单次、正向、运行中并保持终态时保留正文；零时长、循环、反向和无关关键帧继续视为隐藏。", "固定二进制 e146b853… 完成 545 页认证运行，CNN 正文恢复，PR919 目标正文为 545/545。"], - ["固定计分总体", "先前报告仍使用公式 113 页、表格 81 页的旧抽取总体。", "按 lexmain f7cb97d 的参考抽取结果固定为全文 545、代码 110、公式 257、表格内容与结构各 179 页；适用页面遗漏或调用失败计零。", "七种方法全部使用同一总体；3,815 条既有输出离线重算，未新增 provider 调用。"], - ["当前结果", "旧图和文案绑定较早的 PR919 二进制与评分器，不能说明当前提交。", "报告绑定源码 e944d066b、release 二进制 e146b853…、评分器 f7cb97d，并保留原版 Moli、PR785 与 PR919 三行。", "PR919 综合分 39.41%，全文 40.79%、代码 91.74%、公式 28.16%、表格内容 22.61%、表格结构 13.74%；综合分和代码为七方法唯一最高。"] - ], - "i18n": {"en": { - "title": "Restore fade-in content and rescore seven methods with current lexmain", - "summary": "Moli PR919 commit e944d066b fixes missing content on one-shot forward fade-in pages and completes a certified 545-page run with a pinned release binary. WebMainBench lexmain f7cb97d rescored all 3,815 results against reference-defined fixed populations; PR919 delivers 545/545 target bodies and scores 39.41% overall.", - "changes": [ - ["Moli body delivery", "The frozen CNN page starts its body at opacity zero and reveals it with a named keyframe after load; the previous visibility snapshot treated it as permanently hidden.", "Retain the body only when the final keyframe explicitly restores opacity and the animation has positive duration, no positive delay, one forward running iteration, and a retained final state. Zero-duration, looping, reverse, and unrelated animations remain hidden.", "Pinned binary e146b853… completes the certified 545-page run, restores the CNN body, and delivers 545/545 target bodies."], - ["Fixed scoring populations", "The prior report still used the older 113-page formula and 81-page table extraction populations.", "Bind lexmain f7cb97d reference extraction to 545 text, 110 code, 257 formula, and 179 table-content and table-structure pages. Omissions and failed calls score zero within applicable populations.", "All seven methods use the same populations; the 3,815 stored outputs are rescored offline with zero provider calls."], - ["Current result", "Earlier figures and prose were bound to an older PR919 binary and scorer.", "Bind the report to source e944d066b, release binary e146b853…, and scorer f7cb97d while keeping separate rows for original Moli, PR785, and PR919.", "PR919 scores 39.41% overall, 40.79% text, 91.74% code, 28.16% formula, 22.61% table content, and 13.74% table structure; overall and code are the unique seven-method maxima."] + [ + "Moli 正文交付", + "CNN 冻结页的正文以 opacity:0 开始，页面加载后由同名关键帧渐显；旧可见性快照将其误判为永久隐藏。", + "仅在最终关键帧明确恢复不透明度且动画为正时长、无正延迟、单次、正向、运行中并保持终态时保留正文；零时长、循环、反向和无关关键帧继续视为隐藏。", + "固定二进制 e146b853… 完成 545 页认证运行，CNN 正文恢复，PR919 目标正文为 545/545。" + ], + [ + "固定计分总体", + "先前报告仍使用公式 113 页、表格 81 页的旧抽取总体。", + "按 lexmain f7cb97d 的参考抽取结果固定为全文 545、代码 110、公式 257、表格内容与结构各 179 页；适用页面遗漏或调用失败计零。", + "七种方法全部使用同一总体；3,815 条既有输出离线重算，未新增 provider 调用。" + ], + [ + "当前结果", + "旧图和文案绑定较早的 PR919 二进制与评分器，不能说明当前提交。", + "报告绑定源码 e944d066b、release 二进制 e146b853…、评分器 f7cb97d，并保留原版 Moli、PR785 与 PR919 三行。", + "PR919 综合分 39.41%，全文 40.79%、代码 91.74%、公式 28.16%、表格内容 22.61%、表格结构 13.74%；综合分和代码为七方法唯一最高。" ] - }} + ], + "i18n": { + "en": { + "title": "Restore fade-in content and rescore seven methods with current lexmain", + "summary": "Moli PR919 commit e944d066b fixes missing content on one-shot forward fade-in pages and completes a certified 545-page run with a pinned release binary. WebMainBench lexmain f7cb97d rescored all 3,815 results against reference-defined fixed populations; PR919 delivers 545/545 target bodies and scores 39.41% overall.", + "changes": [ + [ + "Moli body delivery", + "The frozen CNN page starts its body at opacity zero and reveals it with a named keyframe after load; the previous visibility snapshot treated it as permanently hidden.", + "Retain the body only when the final keyframe explicitly restores opacity and the animation has positive duration, no positive delay, one forward running iteration, and a retained final state. Zero-duration, looping, reverse, and unrelated animations remain hidden.", + "Pinned binary e146b853… completes the certified 545-page run, restores the CNN body, and delivers 545/545 target bodies." + ], + [ + "Fixed scoring populations", + "The prior report still used the older 113-page formula and 81-page table extraction populations.", + "Bind lexmain f7cb97d reference extraction to 545 text, 110 code, 257 formula, and 179 table-content and table-structure pages. Omissions and failed calls score zero within applicable populations.", + "All seven methods use the same populations; the 3,815 stored outputs are rescored offline with zero provider calls." + ], + [ + "Current result", + "Earlier figures and prose were bound to an older PR919 binary and scorer.", + "Bind the report to source e944d066b, release binary e146b853…, and scorer f7cb97d while keeping separate rows for original Moli, PR785, and PR919.", + "PR919 scores 39.41% overall, 40.79% text, 91.74% code, 28.16% formula, 22.61% table content, and 13.74% table structure; overall and code are the unique seven-method maxima." + ] + ] + } + } }, { "id": "webmainbench-545-lexmain-reference-and-final-pr919-20260928", "publication_state": "published", "date": "2026-09-28", - "comparison_kinds": ["webmainbench_frozen_html_to_markdown"], + "comparison_kinds": [ + "webmainbench_frozen_html_to_markdown" + ], "title": "修正 WebMainBench 参考、专项计分范围与 TEDS", "summary": "Lexmount WebMainBench lexmain 修正 TEDS 有序树计算、22 条参考或代码标注，并让七种方法在相同的参考定义专项页面上计分。PR919 提交 b7a860600 的 545 页认证输出保持不变；3,815 条结果离线重算后，修复版成功生成正文 545/545，综合分为 55.25%。", "changes": [ - ["评测基准", "官方 TEDS 把序列化字符串交给 APTED；任务 f080fbec 的参考包含隐藏编辑器副本，另有 21 条代码参考或标注与冻结页面不一致。", "在 Lexmount WebMainBench lexmain 对显式有序树计算 TEDS，按冻结页面修正参考与代码范围，并用哈希绑定每条数据修订。", "TEDS、参考和代码范围均对全部 3,815 条冻结结果离线复算，没有新增 provider 调用。"], - ["统一计分范围", "代码、公式和表格分母曾由每种方法的输出决定；漏输出、抽取失败或额外输出会让不同方法落入不同计分页集合。", "专项指标改用参考答案定义的公共页面范围；参考适用但方法遗漏或调用失败时计零，参考无法抽取为可评分表示的页面记为不可用。", "七种方法统一使用全文 545、代码 110、公式 113、表格内容和结构各 81 页。成对回归控制同时覆盖漏计和误计。"], - ["实现复审", "上一候选把所有 pre 输出为围栏，并拆平任意含 pre 的表格；这提高代码提取分，却删除真实链接、行列和跨行关系。", "保留含链接的 pre 与语义表格；布局表继续按已有规则展开，同时修正祖先渐变、只读 DOM 查询和原始 fragment 优先级。", "相关行为测试通过；固定 release 二进制完成 545 页全量认证，545 次最终调用均成功。"], - ["分数解释", "旧报告的 33.11% 来自方法相关分母和不完整代码提取，不能作为同范围七方法比较。", "保留每份 provider 输出与官方原分，另存修正前指标，再用统一参考范围和修正评分器生成当前指标。", "PR919 综合分 55.25%，代码 91.06%、公式 64.08%、表格内容 50.09%、表格结构 30.25%，成功生成正文 545/545。"] - ], - "i18n": {"en": { - "title": "Correct WebMainBench references, component populations, and TEDS", - "summary": "Lexmount WebMainBench lexmain corrects ordered-tree TEDS, 22 reference or code annotations, and uses the same reference-defined component pages for all seven methods. The certified 545-page PR919 outputs at commit b7a860600 are unchanged. Offline rescoring of all 3,815 results yields 545/545 nonempty bodies and a 55.25% overall score.", - "changes": [ - ["Evaluation basis", "Official TEDS passed a serialized string to APTED; task f080fbec included a hidden editor buffer, and 21 other code references or annotations differed from the frozen page.", "Compute TEDS over the explicit ordered tree, correct references and code scope against the frozen pages, and hash-bind each data correction in Lexmount WebMainBench lexmain.", "TEDS, references, and code scope are recomputed over all 3,815 frozen results with zero provider calls."], - ["Shared scoring scope", "Code, formula, and table denominators previously depended on each method output, so omissions, extraction failures, and extra output changed the scored population.", "Use reference-defined component populations for every method. Missing output and failed calls score zero inside an applicable population; pages whose references cannot be extracted into a scoreable form are unavailable.", "All seven methods now use 545 text, 110 code, 113 formula, and 81 table-content and table-structure pages. Paired controls cover both error directions."], - ["Implementation review", "The prior candidate fenced every pre element and flattened every table containing pre content, improving code extraction while deleting real links, rows, and spans.", "Preserve pre elements with links and semantic tables, retain existing layout-table handling, and correct ancestor gradients, read-only DOM lookups, and raw fragment priority.", "Scenario tests pass and the fixed release binary completes all 545 certified calls successfully."], - ["Score interpretation", "The former 33.11% value used method-dependent populations and incomplete code extraction, so it was not a same-scope seven-method comparison.", "Preserve each provider output and original official score, retain pre-correction metrics, and derive current metrics with the corrected scorer and shared reference scope.", "PR919 scores 55.25% overall, 91.06% code, 64.08% formula, 50.09% table content, and 30.25% table structure, with 545/545 nonempty bodies."] + [ + "评测基准", + "官方 TEDS 把序列化字符串交给 APTED；任务 f080fbec 的参考包含隐藏编辑器副本，另有 21 条代码参考或标注与冻结页面不一致。", + "在 Lexmount WebMainBench lexmain 对显式有序树计算 TEDS，按冻结页面修正参考与代码范围，并用哈希绑定每条数据修订。", + "TEDS、参考和代码范围均对全部 3,815 条冻结结果离线复算，没有新增 provider 调用。" + ], + [ + "统一计分范围", + "代码、公式和表格分母曾由每种方法的输出决定；漏输出、抽取失败或额外输出会让不同方法落入不同计分页集合。", + "专项指标改用参考答案定义的公共页面范围；参考适用但方法遗漏或调用失败时计零，参考无法抽取为可评分表示的页面记为不可用。", + "七种方法统一使用全文 545、代码 110、公式 113、表格内容和结构各 81 页。成对回归控制同时覆盖漏计和误计。" + ], + [ + "实现复审", + "上一候选把所有 pre 输出为围栏，并拆平任意含 pre 的表格；这提高代码提取分，却删除真实链接、行列和跨行关系。", + "保留含链接的 pre 与语义表格；布局表继续按已有规则展开，同时修正祖先渐变、只读 DOM 查询和原始 fragment 优先级。", + "相关行为测试通过；固定 release 二进制完成 545 页全量认证，545 次最终调用均成功。" + ], + [ + "分数解释", + "旧报告的 33.11% 来自方法相关分母和不完整代码提取，不能作为同范围七方法比较。", + "保留每份 provider 输出与官方原分，另存修正前指标，再用统一参考范围和修正评分器生成当前指标。", + "PR919 综合分 55.25%，代码 91.06%、公式 64.08%、表格内容 50.09%、表格结构 30.25%，成功生成正文 545/545。" ] - }} + ], + "i18n": { + "en": { + "title": "Correct WebMainBench references, component populations, and TEDS", + "summary": "Lexmount WebMainBench lexmain corrects ordered-tree TEDS, 22 reference or code annotations, and uses the same reference-defined component pages for all seven methods. The certified 545-page PR919 outputs at commit b7a860600 are unchanged. Offline rescoring of all 3,815 results yields 545/545 nonempty bodies and a 55.25% overall score.", + "changes": [ + [ + "Evaluation basis", + "Official TEDS passed a serialized string to APTED; task f080fbec included a hidden editor buffer, and 21 other code references or annotations differed from the frozen page.", + "Compute TEDS over the explicit ordered tree, correct references and code scope against the frozen pages, and hash-bind each data correction in Lexmount WebMainBench lexmain.", + "TEDS, references, and code scope are recomputed over all 3,815 frozen results with zero provider calls." + ], + [ + "Shared scoring scope", + "Code, formula, and table denominators previously depended on each method output, so omissions, extraction failures, and extra output changed the scored population.", + "Use reference-defined component populations for every method. Missing output and failed calls score zero inside an applicable population; pages whose references cannot be extracted into a scoreable form are unavailable.", + "All seven methods now use 545 text, 110 code, 113 formula, and 81 table-content and table-structure pages. Paired controls cover both error directions." + ], + [ + "Implementation review", + "The prior candidate fenced every pre element and flattened every table containing pre content, improving code extraction while deleting real links, rows, and spans.", + "Preserve pre elements with links and semantic tables, retain existing layout-table handling, and correct ancestor gradients, read-only DOM lookups, and raw fragment priority.", + "Scenario tests pass and the fixed release binary completes all 545 certified calls successfully." + ], + [ + "Score interpretation", + "The former 33.11% value used method-dependent populations and incomplete code extraction, so it was not a same-scope seven-method comparison.", + "Preserve each provider output and original official score, retain pre-correction metrics, and derive current metrics with the corrected scorer and shared reference scope.", + "PR919 scores 55.25% overall, 91.06% code, 64.08% formula, 50.09% table content, and 30.25% table structure, with 545/545 nonempty bodies." + ] + ] + } + } }, { "id": "webmainbench-545-moli-pr919-final-20260928", @@ -1762,24 +2217,58 @@ { "id": "2026-09-24-dynamic-100-moli-119-local-audit", "publication_state": "published", - "comparison_kinds": ["single_provider_dynamic_native_fetch"], + "comparison_kinds": [ + "single_provider_dynamic_native_fetch" + ], "date": "2026-09-24", "title": "完成100例动态网站 Moli 1.1.9 本地审核版", "summary": "合并两个已保存的原生 Fetch 批次，以同一评分实现离线核对100例；结果为44例通过、56例未通过。没有新增Moli调用，原始调用时间、输出和日志继续保留在持久证据目录。", "changes": [ - ["调用口径", "旧的逐站等待与提取设置会替产品决定何时抓取。", "全部100例使用Moli 1.1.9原生Fetch、默认done和统一参数；只有网址不同。", "不同日期的两个批次合并展示，每例保留真实调用时间。"], - ["规则与评分", "动态名单、表格和正文需要同一套可复核规则。", "冻结100例当前规则，使用稳定文字、同条目字段、名称链接、数量门槛和正文区域的公共评分实现。", "保存的原始返回离线重算仍为44/56，没有判定变化。"], - ["证据与报告", "原始结果、debug日志和生成HTML不适合进入Git。", "Git只保存案例定义、直接来源证据、公共评分与报告代码及哈希收据；运行产物留在外部持久证据目录。", "报告显示Moli版本、失败分类、正文证据及完整物理调用链下钻。"] - ], - "i18n": {"en": { - "title": "Complete the local audit build for 100 dynamic sites with Moli 1.1.9", - "summary": "Two persisted native Fetch cohorts are combined and checked offline with one scoring implementation across 100 cases: 44 pass and 56 fail. No new Moli calls were made; original call times, outputs and logs remain in persistent evidence.", - "changes": [ - ["Call contract", "Earlier per-site waits and extractors could decide retrieval timing for the product.", "All 100 cases use Moli 1.1.9 native Fetch, default done completion and uniform parameters; only the URL changes.", "The two dated cohorts are combined while each case retains its actual call time."], - ["Rules and scoring", "Dynamic lists, tables and body text require one auditable rule set.", "Freeze the current 100-case rules and use shared stable-text, same-record field, name-link, count-threshold and body-region scoring.", "Offline rescoring of saved raw responses remains 44/56 with no verdict changes."], - ["Evidence and report", "Raw results, debug logs and generated HTML do not belong in Git.", "Git retains case definitions, direct source evidence, shared scoring and report code, and hash receipts; run artifacts stay in persistent evidence.", "The report shows the Moli version, failure classes, body evidence and a drill-down through the complete physical-call chain."] + [ + "调用口径", + "旧的逐站等待与提取设置会替产品决定何时抓取。", + "全部100例使用Moli 1.1.9原生Fetch、默认done和统一参数；只有网址不同。", + "不同日期的两个批次合并展示，每例保留真实调用时间。" + ], + [ + "规则与评分", + "动态名单、表格和正文需要同一套可复核规则。", + "冻结100例当前规则，使用稳定文字、同条目字段、名称链接、数量门槛和正文区域的公共评分实现。", + "保存的原始返回离线重算仍为44/56，没有判定变化。" + ], + [ + "证据与报告", + "原始结果、debug日志和生成HTML不适合进入Git。", + "Git只保存案例定义、直接来源证据、公共评分与报告代码及哈希收据；运行产物留在外部持久证据目录。", + "报告显示Moli版本、失败分类、正文证据及完整物理调用链下钻。" ] - }} + ], + "i18n": { + "en": { + "title": "Complete the local audit build for 100 dynamic sites with Moli 1.1.9", + "summary": "Two persisted native Fetch cohorts are combined and checked offline with one scoring implementation across 100 cases: 44 pass and 56 fail. No new Moli calls were made; original call times, outputs and logs remain in persistent evidence.", + "changes": [ + [ + "Call contract", + "Earlier per-site waits and extractors could decide retrieval timing for the product.", + "All 100 cases use Moli 1.1.9 native Fetch, default done completion and uniform parameters; only the URL changes.", + "The two dated cohorts are combined while each case retains its actual call time." + ], + [ + "Rules and scoring", + "Dynamic lists, tables and body text require one auditable rule set.", + "Freeze the current 100-case rules and use shared stable-text, same-record field, name-link, count-threshold and body-region scoring.", + "Offline rescoring of saved raw responses remains 44/56 with no verdict changes." + ], + [ + "Evidence and report", + "Raw results, debug logs and generated HTML do not belong in Git.", + "Git retains case definitions, direct source evidence, shared scoring and report code, and hash receipts; run artifacts stay in persistent evidence.", + "The report shows the Moli version, failure classes, body evidence and a drill-down through the complete physical-call chain." + ] + ] + } + } }, { "id": "webmainbench-545-table-controls-and-moli-provenance-20260928", @@ -1972,130 +2461,366 @@ { "id": "2026-09-24-webmainbench-target-body-population", "publication_state": "published", - "comparison_kinds": ["webmainbench_frozen_html_to_markdown"], + "comparison_kinds": [ + "webmainbench_frozen_html_to_markdown" + ], "date": "2026-09-24", "title": "区分请求成功与目标正文", "summary": "核心表和条件耗时改按实际生成目标正文的页面统计；浏览器错误页、空输出和请求失败不再计入正文页数。评分、原始输出和 3,270 条逻辑结果不变。", - "changes": [["正文与耗时口径", "进程成功退出会把 ERR_ACCESS_DENIED 等浏览器错误页计入“返回成功”，并进入成功页条件耗时。", "从已保存的逐页诊断生成目标正文标记；核心表、条件耗时和 Moli 修复前后对照共用该标记。", "Chrome 的错误页不再算目标正文；全部调用耗时、质量评分、重试成本和冻结证据保持不变。"]], - "i18n": {"en": {"title": "Separate successful requests from target-body delivery", "summary": "The core table and conditional latency now count pages that actually produced target body content. Browser error pages, empty output, and request failures are excluded. Scores, raw outputs, and all 3,270 logical results are unchanged.", "changes": [["Body and latency population", "A zero-exit process could count an ERR_ACCESS_DENIED browser page as a successful return and include it in conditional success latency.", "Derive target-body delivery from the retained per-page diagnosis and use it consistently in the core table, conditional latency, and Moli before/after comparison.", "The Chrome error page no longer counts as target body. All-call latency, quality scores, retry cost, and frozen evidence remain unchanged."]]}} + "changes": [ + [ + "正文与耗时口径", + "进程成功退出会把 ERR_ACCESS_DENIED 等浏览器错误页计入“返回成功”，并进入成功页条件耗时。", + "从已保存的逐页诊断生成目标正文标记；核心表、条件耗时和 Moli 修复前后对照共用该标记。", + "Chrome 的错误页不再算目标正文；全部调用耗时、质量评分、重试成本和冻结证据保持不变。" + ] + ], + "i18n": { + "en": { + "title": "Separate successful requests from target-body delivery", + "summary": "The core table and conditional latency now count pages that actually produced target body content. Browser error pages, empty output, and request failures are excluded. Scores, raw outputs, and all 3,270 logical results are unchanged.", + "changes": [ + [ + "Body and latency population", + "A zero-exit process could count an ERR_ACCESS_DENIED browser page as a successful return and include it in conditional success latency.", + "Derive target-body delivery from the retained per-page diagnosis and use it consistently in the core table, conditional latency, and Moli before/after comparison.", + "The Chrome error page no longer counts as target body. All-call latency, quality scores, retry cost, and frozen evidence remain unchanged." + ] + ] + } + } }, { "id": "2026-09-24-webmainbench-six-versions-certified", "publication_state": "published", - "comparison_kinds": ["webmainbench_frozen_html_to_markdown"], + "comparison_kinds": [ + "webmainbench_frozen_html_to_markdown" + ], "date": "2026-09-24", "title": "将修复版 Moli 纳入完整六方法比较", "summary": "修复版 Moli 按同一批 545 页完成独立认证运行；主表、结果清单和同页对照均展示六种方法，共 3,270 条结果。", - "changes": [["六方法结果", "修复版先前只在附注页出现，主表与同页证据仍只有五种方法。", "将修复版作为独立第六方法纳入完整矩阵，两轮结果使用同一表格结构修正口径。", "逐页输出和失败可直接对照；修复版另日运行，其耗时不参加原五方法的速度标优。"]], - "i18n": {"en": {"title": "Include repaired Moli in the complete six-version comparison", "summary": "A separate certified run measured repaired Moli on the same 545 pages. The main table, result inventory, and same-case comparisons now include all six versions and 3,270 results.", "changes": [["Six-version results", "The repaired build previously appeared only on a follow-up page, outside the main result matrix.", "Include it as a sixth version, and apply the same corrected table-structure scoring to both complete cohorts.", "Page outputs and failures are directly inspectable; the later timing run is not ranked against the original five."]]}} + "changes": [ + [ + "六方法结果", + "修复版先前只在附注页出现，主表与同页证据仍只有五种方法。", + "将修复版作为独立第六方法纳入完整矩阵，两轮结果使用同一表格结构修正口径。", + "逐页输出和失败可直接对照；修复版另日运行，其耗时不参加原五方法的速度标优。" + ] + ], + "i18n": { + "en": { + "title": "Include repaired Moli in the complete six-version comparison", + "summary": "A separate certified run measured repaired Moli on the same 545 pages. The main table, result inventory, and same-case comparisons now include all six versions and 3,270 results.", + "changes": [ + [ + "Six-version results", + "The repaired build previously appeared only on a follow-up page, outside the main result matrix.", + "Include it as a sixth version, and apply the same corrected table-structure scoring to both complete cohorts.", + "Page outputs and failures are directly inspectable; the later timing run is not ranked against the original five." + ] + ] + } + } }, { "id": "2026-09-24-webmainbench-moli-repair-545-followup", "publication_state": "published", - "comparison_kinds": ["webmainbench_frozen_html_to_markdown"], + "comparison_kinds": [ + "webmainbench_frozen_html_to_markdown" + ], "date": "2026-09-24", "title": "补充 Moli 修复后的 545 页复测", "summary": "同一批冻结输入上的 Moli 单方法复测由 542/545 页返回成功增至 544/545 页；两条解析器崩溃恢复，一条隔离环境阻断的跳转仍失败。", - "changes": [["修复复测", "原五方法报告只记录修复前的 Moli 结果，读者无法看到后续解析器修复的实际效果。", "单列 545 页复测、同口径四项原评分、恢复页面的完整输出与剩余失败。", "五方法认证结果和排名未改；恢复的 Autochina360 页仍夹杂大量无关链接，全文分 22.8%。不同时间的耗时和未经同一 TEDS 修正的综合分不作前后比较。"]], - "i18n": {"en": {"title": "Add the 545-page Moli repair follow-up", "summary": "In a Moli-only rerun of the same frozen pages, request success rose from 542/545 to 544/545. Two parser crashes recovered; one redirect blocked by the isolated fixture still failed.", "changes": [["Repair follow-up", "The five-method report only showed Moli before the parser repair.", "Present the full-page rerun, four original-scorer metrics, complete outputs for recovered pages, and the remaining failure.", "The certified five-method ranking is unchanged. The recovered Autochina360 output still contains many unrelated links and scores 22.8% on full text; timing across dates and aggregate scores without the same TEDS correction are not compared."]]}} + "changes": [ + [ + "修复复测", + "原五方法报告只记录修复前的 Moli 结果，读者无法看到后续解析器修复的实际效果。", + "单列 545 页复测、同口径四项原评分、恢复页面的完整输出与剩余失败。", + "五方法认证结果和排名未改；恢复的 Autochina360 页仍夹杂大量无关链接，全文分 22.8%。不同时间的耗时和未经同一 TEDS 修正的综合分不作前后比较。" + ] + ], + "i18n": { + "en": { + "title": "Add the 545-page Moli repair follow-up", + "summary": "In a Moli-only rerun of the same frozen pages, request success rose from 542/545 to 544/545. Two parser crashes recovered; one redirect blocked by the isolated fixture still failed.", + "changes": [ + [ + "Repair follow-up", + "The five-method report only showed Moli before the parser repair.", + "Present the full-page rerun, four original-scorer metrics, complete outputs for recovered pages, and the remaining failure.", + "The certified five-method ranking is unchanged. The recovered Autochina360 output still contains many unrelated links and scores 22.8% on full text; timing across dates and aggregate scores without the same TEDS correction are not compared." + ] + ] + } + } }, { "id": "2026-09-23-webmainbench-error-attribution-and-case-comparison", "publication_state": "published", - "comparison_kinds": ["webmainbench_frozen_html_to_markdown"], + "comparison_kinds": [ + "webmainbench_frozen_html_to_markdown" + ], "date": "2026-09-23", "title": "按方法定位机器问题并展示同页完整输出", "summary": "机器问题图与解读页交叉列出错误类型、方法和可确认的归属。逐题对照页在同一处显示人工主内容答案与五种方法的完整输出，结果清单可按复核原因筛选。", - "changes": [["错误归属与逐题阅读", "汇总数不能直接显示每种错误发生在哪个方法，逐题页只列分数而需分别打开答案和输出。", "增加错误类型与方法矩阵、归属说明及原因筛选；同页显示参考答案和五种完整结果。", "545 题输入、2725 条结果、分数、耗时和已复核的 28 条机器问题均不变。"]], - "i18n": {"en": {"title": "Attribute machine issues by method and show complete same-page outputs", "summary": "The issue graphic and interpretation cross-tabulate error type, method, and evidenced attribution. Each case comparison displays the human main-content reference beside all five complete outputs, and the result inventory can filter by reviewed cause.", "changes": [["Issue attribution and case reading", "Aggregate counts did not show which method encountered each issue, and case pages listed scores without the reference and outputs in one view.", "Add an error-type-by-method matrix, attribution notes, and cause filtering; display the reference and five full outputs on each case page.", "The 545 inputs, 2,725 results, scores, timings, and 28 reviewed machine issues remain unchanged."]]}} + "changes": [ + [ + "错误归属与逐题阅读", + "汇总数不能直接显示每种错误发生在哪个方法，逐题页只列分数而需分别打开答案和输出。", + "增加错误类型与方法矩阵、归属说明及原因筛选；同页显示参考答案和五种完整结果。", + "545 题输入、2725 条结果、分数、耗时和已复核的 28 条机器问题均不变。" + ] + ], + "i18n": { + "en": { + "title": "Attribute machine issues by method and show complete same-page outputs", + "summary": "The issue graphic and interpretation cross-tabulate error type, method, and evidenced attribution. Each case comparison displays the human main-content reference beside all five complete outputs, and the result inventory can filter by reviewed cause.", + "changes": [ + [ + "Issue attribution and case reading", + "Aggregate counts did not show which method encountered each issue, and case pages listed scores without the reference and outputs in one view.", + "Add an error-type-by-method matrix, attribution notes, and cause filtering; display the reference and five full outputs on each case page.", + "The 545 inputs, 2,725 results, scores, timings, and 28 reviewed machine issues remain unchanged." + ] + ] + } + } }, { "id": "2026-09-23-webmainbench-complete-545-five-methods", "publication_state": "published", - "comparison_kinds": ["webmainbench_frozen_html_to_markdown"], + "comparison_kinds": [ + "webmainbench_frozen_html_to_markdown" + ], "date": "2026-09-23", "title": "发布 WebMainBench 完整 545 题五方法对照", "summary": "以同一批 545 个官方冻结 HTML 和人工参考，实测五种稳定版浏览器方法。新报告提供逐题输入、输出、评分、失败证据和已复核的机器问题分析。", - "changes": [["完整评测与评分", "先前公开报告只覆盖 200 题，无法代表完整数据集。官方 TEDS 实现还可能高估部分表格相似度。", "认证五种方法各 545 题，并用保留原始输出和官方分数的结构化树比较校正 TEDS；报告同时展示请求重试和失败分类。", "共 2725 个逻辑结果、2747 次实际调用；292 个 TEDS 分数校正。28 个机器问题已逐条复核，其余 2697 个结果未逐条人工复核。"]], - "i18n": {"en": {"title": "Publish the complete 545-case WebMainBench comparison", "summary": "Five stable browser methods were measured against the same 545 official frozen HTML inputs and human references. The new report exposes case inputs, outputs, scores, failure evidence, and reviewed machine-issue analysis.", "changes": [["Full evaluation and scoring", "The earlier public report covered only 200 cases, and the official TEDS implementation could overstate some table similarities.", "Certify 545 cases for each of five methods and correct TEDS with structured tree comparison while preserving raw outputs and official scores; show retries and failure classes.", "There are 2,725 logical results and 2,747 physical calls; 292 TEDS scores were corrected. All 28 machine issues were reviewed individually, while the other 2,697 results were not individually human reviewed."]]}} + "changes": [ + [ + "完整评测与评分", + "先前公开报告只覆盖 200 题，无法代表完整数据集。官方 TEDS 实现还可能高估部分表格相似度。", + "认证五种方法各 545 题，并用保留原始输出和官方分数的结构化树比较校正 TEDS；报告同时展示请求重试和失败分类。", + "共 2725 个逻辑结果、2747 次实际调用；292 个 TEDS 分数校正。28 个机器问题已逐条复核，其余 2697 个结果未逐条人工复核。" + ] + ], + "i18n": { + "en": { + "title": "Publish the complete 545-case WebMainBench comparison", + "summary": "Five stable browser methods were measured against the same 545 official frozen HTML inputs and human references. The new report exposes case inputs, outputs, scores, failure evidence, and reviewed machine-issue analysis.", + "changes": [ + [ + "Full evaluation and scoring", + "The earlier public report covered only 200 cases, and the official TEDS implementation could overstate some table similarities.", + "Certify 545 cases for each of five methods and correct TEDS with structured tree comparison while preserving raw outputs and official scores; show retries and failure classes.", + "There are 2,725 logical results and 2,747 physical calls; 292 TEDS scores were corrected. All 28 machine issues were reviewed individually, while the other 2,697 results were not individually human reviewed." + ] + ] + } + } }, { "id": "2026-09-20-wcxb-measured-mode-labels", "publication_state": "published", - "comparison_kinds": ["wcxb_frozen_html_to_markdown"], + "comparison_kinds": [ + "wcxb_frozen_html_to_markdown" + ], "date": "2026-09-20", "title": "明确可选清理模式与实测版本", - "title_i18n": {"en": "Clarify optional stripping and measured revisions"}, + "title_i18n": { + "en": "Clarify optional stripping and measured revisions" + }, "summary": "商品说明案例的两列图注明确标注可选 UI 清理。PR 对照继续指向实测提交 384d1d2a，不随后续代码提交自动变为新测量。", - "summary_i18n": {"en": "Both output captions for the product-description case identify optional UI stripping. PR comparisons remain bound to measured commit 384d1d2a; subsequent commits do not become new measurements."}, - "changes": [["图注与版本说明", "可选清理案例与常规调用共用图注，版本措辞未明确冻结测量范围。", "两列分别标注可选 UI 清理，将当前 PR 改为实测 PR。", "504 页分数、耗时、响应与二进制绑定均不变。"]], - "changes_i18n": {"en": [["Captions and revision scope", "Optional stripping shared captions with ordinary calls, and revision wording did not make the frozen scope explicit.", "Label both outputs with optional UI stripping and refer to the measured PR revision.", "Scores, latency, responses and binary bindings for all 504 pages remain unchanged."]]} + "summary_i18n": { + "en": "Both output captions for the product-description case identify optional UI stripping. PR comparisons remain bound to measured commit 384d1d2a; subsequent commits do not become new measurements." + }, + "changes": [ + [ + "图注与版本说明", + "可选清理案例与常规调用共用图注，版本措辞未明确冻结测量范围。", + "两列分别标注可选 UI 清理，将当前 PR 改为实测 PR。", + "504 页分数、耗时、响应与二进制绑定均不变。" + ] + ], + "changes_i18n": { + "en": [ + [ + "Captions and revision scope", + "Optional stripping shared captions with ordinary calls, and revision wording did not make the frozen scope explicit.", + "Label both outputs with optional UI stripping and refer to the measured PR revision.", + "Scores, latency, responses and binary bindings for all 504 pages remain unchanged." + ] + ] + } }, { "id": "2026-09-18-wcxb-adjacent-code-scope-correction", "publication_state": "published", - "comparison_kinds": ["wcxb_frozen_html_to_markdown"], + "comparison_kinds": [ + "wcxb_frozen_html_to_markdown" + ], "date": "2026-09-18", "title": "纠正相邻代码修复的含义与评分解释", - "title_i18n": {"en": "Correct the adjacent-code repair scope and score explanation"}, + "title_i18n": { + "en": "Correct the adjacent-code repair scope and score explanation" + }, "summary": "5458 页的当前 PR 输出保留两个 code 节点，但不会还原由 CSS gap-2 产生的视觉间距。人工参考本身是分开的 name string；该页 F1 下降是因为评分器把内联 HTML 的 code 标签名称也计入输出词数。评测结果未改。", - "summary_i18n": {"en": "The current PR output on page 5458 preserves two code nodes without reconstructing the visual gap supplied by CSS gap-2. The human reference already contains separate name and string tokens; F1 falls because the scorer also counts inline-HTML code tag names as output words. Measured results are unchanged."}, - "changes": [["报告解读", "此前把节点边界保真写成了完整的可读间距修复，并错误解释了该页降分。", "展示实际 Markdown 与渲染语义，区分 DOM 节点边界、CSS 布局和原始源码分词。", "5458 页参考召回率由 98.45% 升至 100%；额外的 code 标签词使精确率与 F1 下降。504 页成绩、分母、耗时和原始响应保持不变。"]], - "changes_i18n": {"en": [["Interpretation", "The earlier copy overstated node-boundary preservation as a complete readable-spacing repair and misattributed the page-level score drop.", "Show the actual Markdown and rendered semantics while separating DOM boundaries, CSS layout and raw-source tokenization.", "Reference recall on page 5458 rises from 98.45% to 100%; extra code-tag words lower precision and F1. The 504-page scores, denominator, latency and raw responses are unchanged."]]} + "summary_i18n": { + "en": "The current PR output on page 5458 preserves two code nodes without reconstructing the visual gap supplied by CSS gap-2. The human reference already contains separate name and string tokens; F1 falls because the scorer also counts inline-HTML code tag names as output words. Measured results are unchanged." + }, + "changes": [ + [ + "报告解读", + "此前把节点边界保真写成了完整的可读间距修复，并错误解释了该页降分。", + "展示实际 Markdown 与渲染语义，区分 DOM 节点边界、CSS 布局和原始源码分词。", + "5458 页参考召回率由 98.45% 升至 100%；额外的 code 标签词使精确率与 F1 下降。504 页成绩、分母、耗时和原始响应保持不变。" + ] + ], + "changes_i18n": { + "en": [ + [ + "Interpretation", + "The earlier copy overstated node-boundary preservation as a complete readable-spacing repair and misattributed the page-level score drop.", + "Show the actual Markdown and rendered semantics while separating DOM boundaries, CSS layout and raw-source tokenization.", + "Reference recall on page 5458 rises from 98.45% to 100%; extra code-tag words lower precision and F1. The 504-page scores, denominator, latency and raw responses are unchanged." + ] + ] + } }, { "id": "2026-09-18-wcxb-moli-118-current-pr-comparison", "publication_state": "published", - "comparison_kinds": ["wcxb_frozen_html_to_markdown"], + "comparison_kinds": [ + "wcxb_frozen_html_to_markdown" + ], "date": "2026-09-18", "title": "更新 Moli 稳定版与当前修复版对照", - "title_i18n": {"en": "Compare the current Moli release and PR build"}, + "title_i18n": { + "en": "Compare the current Moli release and PR build" + }, "summary": "使用 Moli 1.1.8 正式发布文件与 PR #643 当前源码分别实测同一批 504 页。三张修复图现在统一展示冻结输入、1.1.8 输出和当前 PR 输出；可选界面清理另行标注。", - "summary_i18n": {"en": "The official Moli 1.1.8 binary and current PR #643 source were measured on the same 504 pages. All three repair figures compare frozen input, 1.1.8 output and current PR output; optional UI stripping is identified separately."}, - "changes": [["结果与报告解读", "此前使用旧版与早期 PR 的混合证据，版本含义不清。", "重新认证两款 Moli 二进制的完整 504 页结果，并用同窗口交叉测量更新耗时；说明两页因结构保真而降分。", "两版均有 504/504 个非空响应，平均 F1 分别为 52.5510% 和 52.5492%；最终轮平均耗时分别为 150.5 ms 和 151.9 ms。"]], - "changes_i18n": {"en": [["Results and interpretation", "Earlier figures mixed an older release with an earlier PR source, making version identity unclear.", "Certified full 504-page results for both Moli binaries and refreshed latency using a same-window crossover; explained two pages whose score falls while code structure improves.", "Both builds return nonempty content on 504/504 pages; mean F1 is 52.5510% and 52.5492%, and final-round mean latency is 150.5 ms and 151.9 ms."]]} + "summary_i18n": { + "en": "The official Moli 1.1.8 binary and current PR #643 source were measured on the same 504 pages. All three repair figures compare frozen input, 1.1.8 output and current PR output; optional UI stripping is identified separately." + }, + "changes": [ + [ + "结果与报告解读", + "此前使用旧版与早期 PR 的混合证据，版本含义不清。", + "重新认证两款 Moli 二进制的完整 504 页结果，并用同窗口交叉测量更新耗时；说明两页因结构保真而降分。", + "两版均有 504/504 个非空响应，平均 F1 分别为 52.5510% 和 52.5492%；最终轮平均耗时分别为 150.5 ms 和 151.9 ms。" + ] + ], + "changes_i18n": { + "en": [ + [ + "Results and interpretation", + "Earlier figures mixed an older release with an earlier PR source, making version identity unclear.", + "Certified full 504-page results for both Moli binaries and refreshed latency using a same-window crossover; explained two pages whose score falls while code structure improves.", + "Both builds return nonempty content on 504/504 pages; mean F1 is 52.5510% and 52.5492%, and final-round mean latency is 150.5 ms and 151.9 ms." + ] + ] + } }, { "id": "2026-09-18-wcxb-moli-repair-figure-provenance", "publication_state": "published", - "comparison_kinds": ["wcxb_frozen_html_to_markdown"], + "comparison_kinds": [ + "wcxb_frozen_html_to_markdown" + ], "date": "2026-09-18", "title": "标清 Moli 修复图的版本与证据范围", - "title_i18n": {"en": "Identify versions and evidence scope in Moli repair figures"}, + "title_i18n": { + "en": "Identify versions and evidence scope in Moli repair figures" + }, "summary": "三张图分别标出正式版 1.1.7、PR 初版与当前源码，并区分整批实测、可选功能额外对照和场景测试。评测数据未改。", - "summary_i18n": {"en": "The three figures identify released 1.1.7, the initial PR and current source, and distinguish cohort results, an extra optional-feature comparison and scenario tests. Measured results are unchanged."}, + "summary_i18n": { + "en": "The three figures identify released 1.1.7, the initial PR and current source, and distinguish cohort results, an extra optional-feature comparison and scenario tests. Measured results are unchanged." + }, "changes": [ - ["报告解读", "图中的版本简称容易使源码场景测试看起来像稳定版或 504 页实测结果。", "逐图标明两列的源码版本和证据范围，并在 PR 中展示相同图片。", "分数、分母、耗时和原始响应不变；当前 PR 源码不冒充已完成整批重测。"] + [ + "报告解读", + "图中的版本简称容易使源码场景测试看起来像稳定版或 504 页实测结果。", + "逐图标明两列的源码版本和证据范围，并在 PR 中展示相同图片。", + "分数、分母、耗时和原始响应不变；当前 PR 源码不冒充已完成整批重测。" + ] ], - "changes_i18n": {"en": [["Interpretation", "Short figure labels could make a source-level scenario look like a released or full-cohort result.", "Identify both versions and evidence scope for each figure and embed the same images in the PR.", "Scores, denominators, latency and raw responses are unchanged; current PR source is not presented as a completed cohort rerun."]]} + "changes_i18n": { + "en": [ + [ + "Interpretation", + "Short figure labels could make a source-level scenario look like a released or full-cohort result.", + "Identify both versions and evidence scope for each figure and embed the same images in the PR.", + "Scores, denominators, latency and raw responses are unchanged; current PR source is not presented as a completed cohort rerun." + ] + ] + } }, { "id": "2026-09-18-wcxb-moli-repair-visual-guide", "publication_state": "published", - "comparison_kinds": ["wcxb_frozen_html_to_markdown"], + "comparison_kinds": [ + "wcxb_frozen_html_to_markdown" + ], "date": "2026-09-18", "title": "用图和源码样本说明 Moli 的内容修复", - "title_i18n": {"en": "Explain the Moli fixes with diagrams and source examples"}, + "title_i18n": { + "en": "Explain the Moli fixes with diagrams and source examples" + }, "summary": "报告解读逐项展示代码块粘连、商品说明误删和多加空格，并在图下对照真实 HTML、转换结果与源码处理规则。评测数据未改。", - "summary_i18n": {"en": "The interpretation shows joined code blocks, deleted product descriptions and an added space, pairing each image with actual HTML, output and source-level handling. Results are unchanged."}, + "summary_i18n": { + "en": "The interpretation shows joined code blocks, deleted product descriptions and an added space, pairing each image with actual HTML, output and source-level handling. Results are unchanged." + }, "changes": [ - ["报告解读", "原有三栏代码片段和长段解释不易看出原文、错误和修复的关系。", "每个问题用一张图及真实 HTML、输出和源码规则展示差异；标明整批实测与后续测试的边界。", "分数、分母、耗时、评测响应和审查范围不变；读者可从图旁链接核对完整证据。"] + [ + "报告解读", + "原有三栏代码片段和长段解释不易看出原文、错误和修复的关系。", + "每个问题用一张图及真实 HTML、输出和源码规则展示差异；标明整批实测与后续测试的边界。", + "分数、分母、耗时、评测响应和审查范围不变；读者可从图旁链接核对完整证据。" + ] ], - "changes_i18n": {"en": [["Interpretation", "The former three-column excerpts and long explanations obscured the relation between source, error and fix.", "One image per issue is paired with real HTML, output and source rules; the measured cohort remains distinct from later source tests.", "Scores, denominators, latency, responses and review scope are unchanged; adjacent links open the full evidence."]]} + "changes_i18n": { + "en": [ + [ + "Interpretation", + "The former three-column excerpts and long explanations obscured the relation between source, error and fix.", + "One image per issue is paired with real HTML, output and source rules; the measured cohort remains distinct from later source tests.", + "Scores, denominators, latency, responses and review scope are unchanged; adjacent links open the full evidence." + ] + ] + } }, { "id": "2026-09-18-wcxb-moli-repair-explanation", "publication_state": "published", - "comparison_kinds": ["wcxb_frozen_html_to_markdown"], + "comparison_kinds": [ + "wcxb_frozen_html_to_markdown" + ], "date": "2026-09-18", "title": "补充 Moli 修复的输入与输出对照", - "title_i18n": {"en": "Explain Moli repairs with input and output evidence"}, + "title_i18n": { + "en": "Explain Moli repairs with input and output evidence" + }, "summary": "报告解读新增代码块、商品说明和相邻代码的可视对照，区分 504 页实测构建与其后的源码回归测试。评测结果、分母和耗时未改变。", - "summary_i18n": {"en": "The interpretation now illustrates code blocks, product descriptions and adjacent code with input/output evidence. It separates the measured 504-page build from later source regression tests. Scores, denominators and latency are unchanged."}, + "summary_i18n": { + "en": "The interpretation now illustrates code blocks, product descriptions and adjacent code with input/output evidence. It separates the measured 504-page build from later source regression tests. Scores, denominators and latency are unchanged." + }, "changes": [ - ["报告解读", "修复缘由和源码行为分散在逐页结果与 PR 中。", "在解读页展示冻结输入、原输出、修复输出及可核查证据；明确后续源码修改不属于已测二进制。", "读者可以核对代码边界与正文保留的必要性；504 页成绩和耗时保持原值。"] + [ + "报告解读", + "修复缘由和源码行为分散在逐页结果与 PR 中。", + "在解读页展示冻结输入、原输出、修复输出及可核查证据；明确后续源码修改不属于已测二进制。", + "读者可以核对代码边界与正文保留的必要性；504 页成绩和耗时保持原值。" + ] ], - "changes_i18n": {"en": [["Interpretation", "The repair rationale and converter behavior were scattered across result pages and the PR.", "Show frozen input, released output, repaired output and inspection links together; identify later source changes outside the measured binary.", "Readers can verify why code boundaries and body text must be preserved; the 504-page scores and timing remain unchanged."]]} + "changes_i18n": { + "en": [ + [ + "Interpretation", + "The repair rationale and converter behavior were scattered across result pages and the PR.", + "Show frozen input, released output, repaired output and inspection links together; identify later source changes outside the measured binary.", + "Readers can verify why code boundaries and body text must be preserved; the 504-page scores and timing remain unchanged." + ] + ] + } }, { "id": "2026-09-21-moli-seven-linux-release-resources", @@ -2396,12 +3121,19 @@ { "id": "2026-09-18-wcxb-frozen-html-reader-clarification", "publication_state": "published", - "comparison_kinds": ["wcxb_frozen_html_to_markdown"], + "comparison_kinds": [ + "wcxb_frozen_html_to_markdown" + ], "date": "2026-09-18", "title": "澄清冻结 HTML 阅读视图", "summary": "逐页入口明确区分冻结 HTML 的静态阅读视图与原站画面，提供完整 HTML 输入。504 页评测结果、评分和耗时均未改变。", "changes": [ - ["逐页输入", "原入口称作网页预览，容易误认为复现了原站排版。", "标明只按冻结 HTML 的文档顺序阅读；说明未冻结外部样式和图片，添加完整输入链接。", "读者可核对输入内容，不会把未排版视图当作原站截图。"] + [ + "逐页输入", + "原入口称作网页预览，容易误认为复现了原站排版。", + "标明只按冻结 HTML 的文档顺序阅读；说明未冻结外部样式和图片，添加完整输入链接。", + "读者可核对输入内容，不会把未排版视图当作原站截图。" + ] ] }, { @@ -2623,21 +3355,43 @@ { "id": "2026-09-14-public-benchmark-research", "publication_state": "published", - "comparison_kinds": ["public_benchmark_research"], + "comparison_kinds": [ + "public_benchmark_research" + ], "date": "2026-09-14", "title": "完成公开评测集调研与选型", "summary": "核查公开数据、评分器与使用证据，选定 WebMainBench-545，并冻结覆盖官方分类的 200 页抽样规则。调研版没有供应商实测结果。", "changes": [ - ["评测集", "候选集的真值、评分器和使用范围尚未并列核查。", "核对 WebMainBench-545、WCXB 及相关评测集的一手材料。", "选定 WebMainBench-545 作为后续实测主集；调研结论不构成方法排名。"], - ["抽样", "尚无可复现的测量输入范围。", "定义覆盖官方分类的固定 200 页抽样合同。", "只固定未来测量范围，未增加任何供应商调用或分数。"] + [ + "评测集", + "候选集的真值、评分器和使用范围尚未并列核查。", + "核对 WebMainBench-545、WCXB 及相关评测集的一手材料。", + "选定 WebMainBench-545 作为后续实测主集；调研结论不构成方法排名。" + ], + [ + "抽样", + "尚无可复现的测量输入范围。", + "定义覆盖官方分类的固定 200 页抽样合同。", + "只固定未来测量范围，未增加任何供应商调用或分数。" + ] ], "i18n": { "en": { "title": "Complete public benchmark research and selection", "summary": "Audited public data, scorers, and adoption evidence; selected WebMainBench-545 and froze a 200-page sampling rule covering official categories. The research edition contains no measured provider results.", "changes": [ - ["Benchmark", "Candidate ground truth, scorers, and use had not been audited side by side.", "Checked primary sources for WebMainBench-545, WCXB, and adjacent benchmarks.", "Selected WebMainBench-545 as the primary set for later measurement; research findings are not a method ranking."], - ["Sampling", "There was no reproducible scope for calls.", "Defined a fixed 200-page sampling contract covering official categories.", "Only the future measurement scope was fixed; no provider calls or scores were added."] + [ + "Benchmark", + "Candidate ground truth, scorers, and use had not been audited side by side.", + "Checked primary sources for WebMainBench-545, WCXB, and adjacent benchmarks.", + "Selected WebMainBench-545 as the primary set for later measurement; research findings are not a method ranking." + ], + [ + "Sampling", + "There was no reproducible scope for calls.", + "Defined a fixed 200-page sampling contract covering official categories.", + "Only the future measurement scope was fixed; no provider calls or scores were added." + ] ] } } @@ -2645,23 +3399,55 @@ { "id": "2026-09-17-public-webmainbench-moli-balanced-latency-correction", "publication_state": "published", - "comparison_kinds": ["webmainbench_frozen_html_to_markdown"], + "comparison_kinds": [ + "webmainbench_frozen_html_to_markdown" + ], "date": "2026-09-17", "title": "纠正 Moli 两版的正式耗时", "summary": "同一预选 200 页在无并行基准进程的窗口串行交叉复测两轮，按测前固定规则采用第二轮。Moli 1.1.5 的成功均值由 153 ms 改为 105 ms，1.1.7 由 285 ms 改为 98 ms；两版各有 199/200 页成功。原始逐页结果、正文、质量分和判定保留。", "changes": [ - ["正式耗时", "两版来自不同时间的独立批次，1.1.7 表面上更慢。", "200 页两轮交叉顺序、共 800 次物理调用；预先指定第二轮的 400 次作为两版新的逐页耗时。", "1.1.7 的均值、P50、P95 均未高于 1.1.5；旧值可从纠正收据核对。"], - ["失败页", "两版在同一页未生成正文。", "保留页面脚本和隔离网络条件，复测各轮及此前重试。", "原站跳转仍被网络隔离阻断；两版继续各计 199/200 页成功。"], - ["评分证据", "耗时纠正可能与已审核的原始输出混淆。", "旧结果和原始物理调用不覆盖；逐条仅替换 Moli 的 elapsed_ms，并对其余字段及另外四种方法做相等校验。", "全部质量分、计分分母、正文哈希和结果判定不变。"] + [ + "正式耗时", + "两版来自不同时间的独立批次，1.1.7 表面上更慢。", + "200 页两轮交叉顺序、共 800 次物理调用；预先指定第二轮的 400 次作为两版新的逐页耗时。", + "1.1.7 的均值、P50、P95 均未高于 1.1.5；旧值可从纠正收据核对。" + ], + [ + "失败页", + "两版在同一页未生成正文。", + "保留页面脚本和隔离网络条件，复测各轮及此前重试。", + "原站跳转仍被网络隔离阻断；两版继续各计 199/200 页成功。" + ], + [ + "评分证据", + "耗时纠正可能与已审核的原始输出混淆。", + "旧结果和原始物理调用不覆盖；逐条仅替换 Moli 的 elapsed_ms，并对其余字段及另外四种方法做相等校验。", + "全部质量分、计分分母、正文哈希和结果判定不变。" + ] ], "i18n": { "en": { "title": "Correct formal timing for both Moli versions", "summary": "Two serial, order-balanced rounds used the same preselected 200 pages without concurrent benchmark processes. Round two was designated before calls as the formal timing source. Successful-fetch means changed from 153 to 105 ms for Moli 1.1.5 and from 285 to 98 ms for 1.1.7; each succeeded on 199/200 pages. Prior per-page results, output text, quality scores, and verdicts are retained.", "changes": [ - ["Formal timing", "The two versions came from separate batches at different times, making 1.1.7 appear slower.", "Two order-balanced rounds on 200 pages produced 800 physical calls; the 400 calls in the predesignated second round became the formal per-page timings.", "Mean, P50, and P95 for 1.1.7 are no higher than for 1.1.5. Prior values remain in the correction receipt."], - ["Failed page", "Neither version produced target content on the same page.", "Preserved the page script and isolated network conditions while checking both new rounds and earlier retries.", "The source-site redirect is still blocked by network isolation; each version remains at 199/200 successful pages."], - ["Scoring evidence", "Timing corrections could be confused with the reviewed original output.", "Preserved prior results and physical calls; replaced elapsed_ms only for Moli and checked every other field and the other four methods for equality.", "All quality scores, scored-page counts, output hashes, and verdicts are unchanged."] + [ + "Formal timing", + "The two versions came from separate batches at different times, making 1.1.7 appear slower.", + "Two order-balanced rounds on 200 pages produced 800 physical calls; the 400 calls in the predesignated second round became the formal per-page timings.", + "Mean, P50, and P95 for 1.1.7 are no higher than for 1.1.5. Prior values remain in the correction receipt." + ], + [ + "Failed page", + "Neither version produced target content on the same page.", + "Preserved the page script and isolated network conditions while checking both new rounds and earlier retries.", + "The source-site redirect is still blocked by network isolation; each version remains at 199/200 successful pages." + ], + [ + "Scoring evidence", + "Timing corrections could be confused with the reviewed original output.", + "Preserved prior results and physical calls; replaced elapsed_ms only for Moli and checked every other field and the other four methods for equality.", + "All quality scores, scored-page counts, output hashes, and verdicts are unchanged." + ] ] } } @@ -2785,13 +3571,25 @@ { "id": "2026-09-17-public-webmainbench-denominators-and-version-examples", "publication_state": "published", - "comparison_kinds": ["webmainbench_frozen_html_to_markdown"], + "comparison_kinds": [ + "webmainbench_frozen_html_to_markdown" + ], "date": "2026-09-17", "title": "用公式分实例解释计分页，并核对 Moli 版本差异", "summary": "首页用 Moli 1.1.7 的 200 页公式分流程和四格实例解释计分分母；报告解读增加两版同页对照及正文、代码、公式的五个原始输出案例。所有分数、分母和耗时保持不变。", "changes": [ - ["计分分母", "表格仅显示 n=xx/200，难以判断哪些页被计入。", "首页展示正文生成、公式识别与计分分支，并按人工答案和输出各有无公式列出真实页数。", "两侧均无公式跳过；仅一侧有公式以 0 分计入；分母不是非空输出数或通过数。"], - ["版本差异", "Moli 1.1.7 的其他分项下降缺少逐页解释。", "并列展示两版同页均值及五个可核对案例，区分实际输出差异、计分页变化和评分器误识别。", "固定评分器下的下降成立；新版输出变长、缩进和转义改变了正文与结构项的得分。"] + [ + "计分分母", + "表格仅显示 n=xx/200，难以判断哪些页被计入。", + "首页展示正文生成、公式识别与计分分支，并按人工答案和输出各有无公式列出真实页数。", + "两侧均无公式跳过；仅一侧有公式以 0 分计入；分母不是非空输出数或通过数。" + ], + [ + "版本差异", + "Moli 1.1.7 的其他分项下降缺少逐页解释。", + "并列展示两版同页均值及五个可核对案例，区分实际输出差异、计分页变化和评分器误识别。", + "固定评分器下的下降成立；新版输出变长、缩进和转义改变了正文与结构项的得分。" + ] ], "i18n": { "en": { @@ -2837,15 +3635,37 @@ { "id": "2026-09-16-public-webmainbench-reading-view", "publication_state": "published", - "comparison_kinds": ["webmainbench_frozen_html_to_markdown"], + "comparison_kinds": [ + "webmainbench_frozen_html_to_markdown" + ], "date": "2026-09-16", "title": "明确主内容评测范围并提供网页阅读视图", "summary": "首页用输入、目标和分数含义说明低分来源，不再把本轮结果泛化为通用抓取排名。冻结 HTML 可直接阅读，来源网站与原始源码分开查看。分数、分母、耗时和供应商调用不变。", "changes": [ - ["评测范围", "整页转换输出与主内容参考混在通用抓取能力描述中。", "首页明确主内容提取目标，并提供低分案例入口；解读结论限定于本轮转换。", "不代表实时网站访问、反爬或浏览器综合能力。"], - ["网页查看", "HTML 入口只提供原始文件，来源链接藏在折叠区。", "增加隔离的静态阅读视图与完整源码展开入口，来源网站直接可见。", "阅读视图禁用脚本、外部资源和跳转，不冒充实测浏览器截图。"], - ["输入路径", "同一 HTML 在远端服务和本地浏览器中的输入方式不直观。", "并列展示 Kitesurf 的 API html 输入与四种本地方法的 HTTPS 路径。", "输入字节相同，不等于执行环境相同。"], - ["表格评分", "容易把表格可识别、完整恢复和得满分混为一谈。", "并列展示完整表格、缺行表格和摊平文字的 Markdown 与阅读效果。", "部分恢复仍计分；table_edit 比较规范化表格 HTML，TEDS 比较层级与内容。"] + [ + "评测范围", + "整页转换输出与主内容参考混在通用抓取能力描述中。", + "首页明确主内容提取目标，并提供低分案例入口；解读结论限定于本轮转换。", + "不代表实时网站访问、反爬或浏览器综合能力。" + ], + [ + "网页查看", + "HTML 入口只提供原始文件，来源链接藏在折叠区。", + "增加隔离的静态阅读视图与完整源码展开入口，来源网站直接可见。", + "阅读视图禁用脚本、外部资源和跳转，不冒充实测浏览器截图。" + ], + [ + "输入路径", + "同一 HTML 在远端服务和本地浏览器中的输入方式不直观。", + "并列展示 Kitesurf 的 API html 输入与四种本地方法的 HTTPS 路径。", + "输入字节相同，不等于执行环境相同。" + ], + [ + "表格评分", + "容易把表格可识别、完整恢复和得满分混为一谈。", + "并列展示完整表格、缺行表格和摊平文字的 Markdown 与阅读效果。", + "部分恢复仍计分；table_edit 比较规范化表格 HTML，TEDS 比较层级与内容。" + ] ], "i18n": { "en": { @@ -2856,16 +3676,43 @@ { "id": "2026-09-16-public-webmainbench-decision-guide", "publication_state": "published", - "comparison_kinds": ["webmainbench_frozen_html_to_markdown"], + "comparison_kinds": [ + "webmainbench_frozen_html_to_markdown" + ], "date": "2026-09-16", "title": "用真实案例重写评测解读", "summary": "成绩、分母、耗时和正式调用均未改变。报告解读现在从选型结论出发，用一页共同低分案例、一页表格结构案例和六页异常根因解释分数含义与适用边界；旧的重复说明表已删除。", "changes": [ - ["选型结论", "旧页面罗列测量口径和方法画像，读者仍需自行拼出选择。", "明确区分通用候选、低延迟文字与代码、以及严格按 overall 排名三种目标，并给出对应候选。", "结论只适用于冻结 HTML 转 Markdown；真实 URL 仍需同时间窗复测。"], - ["分数含义", "overall、分项分数和变化分母分散说明，容易被误读为逐页通过率。", "用 Kitesurf 的五项实际分数和分母展示 overall 的完整算式，并直达评分流程。", "官方没有统一的逐页及格线；正文生成、内容质量和耗时继续分开报告。"], - ["共同低分", "低文字分缺少一页可以直接核对的人工答案与五方法输出。", "加入 Wikipedia 冻结页实例，展示人工正文、方法输出开头、五种长度和 text_edit 分数。", "35 页在五种方法上都低于 10%，说明整页输出与主内容答案存在共同范围偏差。"], - ["表格零分", "Moli 和 Obscura 的 0 分容易被误读为空响应。", "加入同一张两列表格的可解析 Markdown 与摊平文字对照，并列出五种方法的 TEDS。", "该例五种方法都有正文；零分来自无法还原表格、行和单元格结构。"], - ["异常处理", "九次无正文结果被笼统归为失败，无法决定修复方向。", "按已验证对照分成页面脚本冲突、冻结页主动回原站、服务复杂度边界三类，并链接逐页证据。", "未公开的 Cloudflare 内部阈值仍不猜测，只报告可重复触发的内容类型。"] + [ + "选型结论", + "旧页面罗列测量口径和方法画像，读者仍需自行拼出选择。", + "明确区分通用候选、低延迟文字与代码、以及严格按 overall 排名三种目标，并给出对应候选。", + "结论只适用于冻结 HTML 转 Markdown；真实 URL 仍需同时间窗复测。" + ], + [ + "分数含义", + "overall、分项分数和变化分母分散说明，容易被误读为逐页通过率。", + "用 Kitesurf 的五项实际分数和分母展示 overall 的完整算式，并直达评分流程。", + "官方没有统一的逐页及格线；正文生成、内容质量和耗时继续分开报告。" + ], + [ + "共同低分", + "低文字分缺少一页可以直接核对的人工答案与五方法输出。", + "加入 Wikipedia 冻结页实例，展示人工正文、方法输出开头、五种长度和 text_edit 分数。", + "35 页在五种方法上都低于 10%，说明整页输出与主内容答案存在共同范围偏差。" + ], + [ + "表格零分", + "Moli 和 Obscura 的 0 分容易被误读为空响应。", + "加入同一张两列表格的可解析 Markdown 与摊平文字对照，并列出五种方法的 TEDS。", + "该例五种方法都有正文；零分来自无法还原表格、行和单元格结构。" + ], + [ + "异常处理", + "九次无正文结果被笼统归为失败，无法决定修复方向。", + "按已验证对照分成页面脚本冲突、冻结页主动回原站、服务复杂度边界三类，并链接逐页证据。", + "未公开的 Cloudflare 内部阈值仍不猜测，只报告可重复触发的内容类型。" + ] ], "i18n": { "en": { @@ -2876,16 +3723,43 @@ { "id": "2026-09-16-public-webmainbench-score-reading-and-layout", "publication_state": "published", - "comparison_kinds": ["webmainbench_frozen_html_to_markdown"], + "comparison_kinds": [ + "webmainbench_frozen_html_to_markdown" + ], "date": "2026-09-16", "title": "澄清计分分支、复算身份与核心表排序", "summary": "成绩、分母、耗时、输出和正式调用均未改变。核心表默认按 overall 从高到低排列；报告明确修正后分数来自报告侧独立复算，不是对官方仓库或官方榜单的修改，并把空内容的跳过与零分条件写成确定规则。", "changes": [ - ["核心排序", "首页虽然展示 overall，默认排序却落在 text_edit，序号也随错误排序生成。", "默认按修正后 overall 降序排列，读者打开页面即可看到同一口径的主结果顺序。", "各列仍可交互排序；绿色只标记方向明确且唯一的领先值。"], - ["复算身份", "“本地修正版”容易被理解为修改了官方代码或官方榜单。", "统一改为“修正后复算”，说明官方仓库和榜单未改；报告侧沿用官方解析和代价规则，对保存输出独立复算。", "官方原始分和修正后复算分继续并列保留，可从核验页和逐页详情核对。"], - ["分母规则", "“可能不计分”和“通常跳过”没有说明官方程序的确定分支。", "明确两侧都没有代码、公式或表格时该项返回失败并退出均值；只有一侧有时以 0 分进入均值；两侧都有时计算相似度。", "分母是成功算出逐页分数的页数，不是通过数；官方没有统一逐页及格线。"], - ["表格结构", "“节点”和“树编辑距离”缺少用户可见的结构含义。", "用“表格—行—单元格”的还原过程解释 TEDS，并说明仅保留单元格文字、没有 Markdown 表格语法时不能还原结构。", "table_edit 仍比较表格文字；table_TEDS 比较可还原的表格结构。"], - ["宽表布局", "通用宽表把前七列写死后只给最后两列各留 36 像素，解读文字逐字换行，把一行撑到约 900–1,050 像素。", "方法画像表为耗时和解读列设置独立宽度，保留横向滚动并恢复正常行高。", "中英文、桌面和窄屏沿用同一语义表格。"] + [ + "核心排序", + "首页虽然展示 overall，默认排序却落在 text_edit，序号也随错误排序生成。", + "默认按修正后 overall 降序排列，读者打开页面即可看到同一口径的主结果顺序。", + "各列仍可交互排序；绿色只标记方向明确且唯一的领先值。" + ], + [ + "复算身份", + "“本地修正版”容易被理解为修改了官方代码或官方榜单。", + "统一改为“修正后复算”，说明官方仓库和榜单未改；报告侧沿用官方解析和代价规则，对保存输出独立复算。", + "官方原始分和修正后复算分继续并列保留，可从核验页和逐页详情核对。" + ], + [ + "分母规则", + "“可能不计分”和“通常跳过”没有说明官方程序的确定分支。", + "明确两侧都没有代码、公式或表格时该项返回失败并退出均值；只有一侧有时以 0 分进入均值；两侧都有时计算相似度。", + "分母是成功算出逐页分数的页数，不是通过数；官方没有统一逐页及格线。" + ], + [ + "表格结构", + "“节点”和“树编辑距离”缺少用户可见的结构含义。", + "用“表格—行—单元格”的还原过程解释 TEDS，并说明仅保留单元格文字、没有 Markdown 表格语法时不能还原结构。", + "table_edit 仍比较表格文字；table_TEDS 比较可还原的表格结构。" + ], + [ + "宽表布局", + "通用宽表把前七列写死后只给最后两列各留 36 像素，解读文字逐字换行，把一行撑到约 900–1,050 像素。", + "方法画像表为耗时和解读列设置独立宽度，保留横向滚动并恢复正常行高。", + "中英文、桌面和窄屏沿用同一语义表格。" + ] ], "i18n": { "en": { @@ -2896,16 +3770,43 @@ { "id": "2026-09-16-public-webmainbench-causal-interpretation", "publication_state": "published", - "comparison_kinds": ["webmainbench_frozen_html_to_markdown"], + "comparison_kinds": [ + "webmainbench_frozen_html_to_markdown" + ], "date": "2026-09-16", "title": "用全局证据重写公开基准解读", "summary": "成绩、分母、耗时和正式调用均未改变。报告解读改为围绕成功层级、整页范围偏差、结构损失、评分器额外识别、冻结页运行冲突和评分修正展开，并把 35 个共同低分页、分母构成及 6 个异常页面的根因放入同一条分析主线。", "changes": [ - ["解读主线", "原解读按测量、实例和配置分章，主要承担导航，读者难以回答高交付率为何仍只有中低质量分。", "以 991/1,000 次目标正文与 21.6%–32.4% overall 的反差开篇，依次解释成功层级、低分机制、分母、方法画像和适用边界。", "overall 是五个不同分母分项的平均，不是固定 200 页上的统一通过率。"], - ["低文字分", "零散案例能看到多余内容，但没有量化它是不是跨方法的共同问题。", "完整矩阵显示 35 页在五种方法上都低于 10%；这些低分页的输出长度中位数约为人工答案的 17–20 倍。", "该证据支持整页输出与主内容答案的范围错位，不等同于证明所有编辑距离差异都来自模板文字。"], - ["分母解释", "只说明分母会不同，没有展示差异由哪一侧的内容触发。", "逐方法列出代码、公式和表格的计分页，以及仅输出侧、仅答案侧数量；例如 Lightpanda 公式 169 页中有 127 页只在输出侧识别出公式。", "评分器按 Markdown 语法识别内容；额外识别不等于页面语义上真的含有该类内容。"], - ["异常根因", "正文缺失与低分散落在多个页面，容易被概括为请求或网络失败。", "把 9 次目标正文缺失归并为 6 个页面，展示脚本依赖、防框架循环、回源导航和 DOM / 表格复杂度等已验证原因。", "服务端未公开资源阈值的两例只定位到触发内容类型，不虚构内部阶段。"], - ["展示与校验", "长文缺少稳定的分析流程图，首次发布的 Mermaid 在分享页中未绘制。", "增加可见 HTML 流程作为默认内容，Mermaid 只作渐进增强；补充 0–10% 筛选并将结构、链接和回退内容纳入发布检查。", "中英文路由、章节、数据和证据链接保持一一对应。"] + [ + "解读主线", + "原解读按测量、实例和配置分章，主要承担导航，读者难以回答高交付率为何仍只有中低质量分。", + "以 991/1,000 次目标正文与 21.6%–32.4% overall 的反差开篇，依次解释成功层级、低分机制、分母、方法画像和适用边界。", + "overall 是五个不同分母分项的平均，不是固定 200 页上的统一通过率。" + ], + [ + "低文字分", + "零散案例能看到多余内容，但没有量化它是不是跨方法的共同问题。", + "完整矩阵显示 35 页在五种方法上都低于 10%；这些低分页的输出长度中位数约为人工答案的 17–20 倍。", + "该证据支持整页输出与主内容答案的范围错位，不等同于证明所有编辑距离差异都来自模板文字。" + ], + [ + "分母解释", + "只说明分母会不同，没有展示差异由哪一侧的内容触发。", + "逐方法列出代码、公式和表格的计分页，以及仅输出侧、仅答案侧数量；例如 Lightpanda 公式 169 页中有 127 页只在输出侧识别出公式。", + "评分器按 Markdown 语法识别内容；额外识别不等于页面语义上真的含有该类内容。" + ], + [ + "异常根因", + "正文缺失与低分散落在多个页面，容易被概括为请求或网络失败。", + "把 9 次目标正文缺失归并为 6 个页面，展示脚本依赖、防框架循环、回源导航和 DOM / 表格复杂度等已验证原因。", + "服务端未公开资源阈值的两例只定位到触发内容类型，不虚构内部阶段。" + ], + [ + "展示与校验", + "长文缺少稳定的分析流程图，首次发布的 Mermaid 在分享页中未绘制。", + "增加可见 HTML 流程作为默认内容，Mermaid 只作渐进增强；补充 0–10% 筛选并将结构、链接和回退内容纳入发布检查。", + "中英文路由、章节、数据和证据链接保持一一对应。" + ] ], "i18n": { "en": { @@ -2916,15 +3817,37 @@ { "id": "2026-09-16-public-webmainbench-anomaly-root-causes", "publication_state": "published", - "comparison_kinds": ["webmainbench_frozen_html_to_markdown"], + "comparison_kinds": [ + "webmainbench_frozen_html_to_markdown" + ], "date": "2026-09-16", "title": "补齐异常结果根因与对照证据", "summary": "对 1,000 条结果执行确定性异常扫描，30 条触发结果集中在 6 个页面。删减对照定位到页面脚本、原站导航和表格 DOM 复杂度；逐页详情与核验页展示直接观察和证据边界。正式调用、输出、分数、分母和耗时均未改变。", "changes": [ - ["异常范围", "缺正文、错误页、同页分歧和极低分仅有现象说明，无法判断是页面、方法、网络还是评分问题。", "以固定规则扫描完整 1,000 条矩阵，并把 30 条触发结果归并到 6 个页面逐一复核。", "规则用于发现可疑点，不把未触发结果宣称为逐条人工复核通过。"], - ["根因定位", "五条 Kitesurf 异常和一组本地浏览器异常缺少能重复验证的触发条件。", "用不计分的脚本、表格和导航删减对照定位递归错误处理器、错误防框架条件、降级跳转、原站跳转及累计表格复杂度。", "Cloudflare 未公开的内部资源阈值仍不推断为具体内存或转换阶段。"], - ["逐页证据", "浏览器错误页虽然有非空文字，容易与目标正文混淆；结果详情对根因覆盖不完整。", "错误页统一归入未生成目标正文；Track ID 与方法详情展示接口状态、最终内容、完整输出、对照观察和证据支持的原因。", "正文存在不是质量通过，官方仍没有统一逐页及格线。"], - ["结果处置", "诊断输入经过删减，不能作为正式任务替换原调用。", "发布版只携带绑定哈希的紧凑复核记录，保留原始 1,000 条结果及全部计分数据。", "诊断调用不进入成绩、分母、耗时或供应商成功率。"] + [ + "异常范围", + "缺正文、错误页、同页分歧和极低分仅有现象说明，无法判断是页面、方法、网络还是评分问题。", + "以固定规则扫描完整 1,000 条矩阵，并把 30 条触发结果归并到 6 个页面逐一复核。", + "规则用于发现可疑点，不把未触发结果宣称为逐条人工复核通过。" + ], + [ + "根因定位", + "五条 Kitesurf 异常和一组本地浏览器异常缺少能重复验证的触发条件。", + "用不计分的脚本、表格和导航删减对照定位递归错误处理器、错误防框架条件、降级跳转、原站跳转及累计表格复杂度。", + "Cloudflare 未公开的内部资源阈值仍不推断为具体内存或转换阶段。" + ], + [ + "逐页证据", + "浏览器错误页虽然有非空文字，容易与目标正文混淆；结果详情对根因覆盖不完整。", + "错误页统一归入未生成目标正文；Track ID 与方法详情展示接口状态、最终内容、完整输出、对照观察和证据支持的原因。", + "正文存在不是质量通过，官方仍没有统一逐页及格线。" + ], + [ + "结果处置", + "诊断输入经过删减，不能作为正式任务替换原调用。", + "发布版只携带绑定哈希的紧凑复核记录，保留原始 1,000 条结果及全部计分数据。", + "诊断调用不进入成绩、分母、耗时或供应商成功率。" + ] ], "i18n": { "en": { @@ -2935,16 +3858,43 @@ { "id": "2026-09-16-public-webmainbench-report-information-architecture", "publication_state": "published", - "comparison_kinds": ["webmainbench_frozen_html_to_markdown"], + "comparison_kinds": [ + "webmainbench_frozen_html_to_markdown" + ], "date": "2026-09-16", "title": "重整公开基准报告的信息架构与逐页诊断", "summary": "测量结果、评分值和样本均未改变。首页合并为一张核心比较表并补齐均值、P50、P95；评测集按独立分类维度筛选；方法页增加流程图；结果页把响应、正文状态和证据支持的原因放回逐页详情。", "changes": [ - ["首页比较", "质量、正文状态、配对耗时和分项分母分散在多张重复表中，排序列又被误当成绿色领先项。", "用一张横向核心表展示分项分数与全部 200 次逻辑结果的均值、P50、P95；分母置于分数下方，绿色只标唯一且方向明确的领先值。", "table_TEDS 与受其影响的 overall 继续只留档，不标领先者。"], - ["评测集", "五套独立分类混在同一张长表里，无法看出每套分类的样本数都合计为 200。", "按语言、难度、表格、代码和公式维度筛选分布表，并实时显示母集和样本合计。", "分类维度彼此独立，同一页面同时属于五套分类。"], - ["方法说明", "评测流程主要依赖连续长段文字，输入、响应、评分和聚合关系不直观。", "用 Mermaid 流程图先展示冻结输入、方法转换、正文判断、逐页评分和按实际页数聚合；运行细节折叠为按需证据。", "评分公式和已知 TEDS 缺陷仍保留可核对文字与源码链接。"], - ["逐页诊断", "“交付”和字符数无法区分正文、错误响应、错误页或无响应，失败原因又集中在重复汇总表。", "结果矩阵改为“是否生成正文”和“响应与正文”；每条失败详情显示直接观察、证据支持的原因范围、完整错误与评分字段。", "没有证据的远端内部根因不推断；原始 URL 仍不参与本轮抓取。"], - ["报告结构", "针对单条反馈不断增加模块，造成首页、分析页和解读页职责重叠。", "评测集只讲样本，方法页只讲流程，分析页保留同页比较、分类筛选和逐页矩阵，详情页承接响应与原因。", "中英文路由、数据、筛选和证据入口继续一一对应。"] + [ + "首页比较", + "质量、正文状态、配对耗时和分项分母分散在多张重复表中，排序列又被误当成绿色领先项。", + "用一张横向核心表展示分项分数与全部 200 次逻辑结果的均值、P50、P95；分母置于分数下方，绿色只标唯一且方向明确的领先值。", + "table_TEDS 与受其影响的 overall 继续只留档，不标领先者。" + ], + [ + "评测集", + "五套独立分类混在同一张长表里，无法看出每套分类的样本数都合计为 200。", + "按语言、难度、表格、代码和公式维度筛选分布表，并实时显示母集和样本合计。", + "分类维度彼此独立，同一页面同时属于五套分类。" + ], + [ + "方法说明", + "评测流程主要依赖连续长段文字，输入、响应、评分和聚合关系不直观。", + "用 Mermaid 流程图先展示冻结输入、方法转换、正文判断、逐页评分和按实际页数聚合；运行细节折叠为按需证据。", + "评分公式和已知 TEDS 缺陷仍保留可核对文字与源码链接。" + ], + [ + "逐页诊断", + "“交付”和字符数无法区分正文、错误响应、错误页或无响应，失败原因又集中在重复汇总表。", + "结果矩阵改为“是否生成正文”和“响应与正文”；每条失败详情显示直接观察、证据支持的原因范围、完整错误与评分字段。", + "没有证据的远端内部根因不推断；原始 URL 仍不参与本轮抓取。" + ], + [ + "报告结构", + "针对单条反馈不断增加模块，造成首页、分析页和解读页职责重叠。", + "评测集只讲样本，方法页只讲流程，分析页保留同页比较、分类筛选和逐页矩阵，详情页承接响应与原因。", + "中英文路由、数据、筛选和证据入口继续一一对应。" + ] ], "i18n": { "en": { @@ -2955,15 +3905,37 @@ { "id": "2026-09-15-public-webmainbench-dataset-and-result-clarity", "publication_state": "published", - "comparison_kinds": ["webmainbench_frozen_html_to_markdown"], + "comparison_kinds": [ + "webmainbench_frozen_html_to_markdown" + ], "date": "2026-09-15", "title": "公开基准页面与方法结果分开展示", "summary": "200 页评测集只展示冻结输入、人工参考与官方分类；五方法结果仍在分析和方法页。结果页明确区分输出交付与官方连续质量分，增加逐指标计分页数明细。方法成绩、评分公式、样本范围和耗时均未改变。", "changes": [ - ["评测集", "案例列表混入方法交付结果，冻结页面与人工参考难以独立查看。", "案例列表只保留页面分类；每个 Track ID 可查看完整参考和冻结 HTML。", "方法成绩在结果分析中继续提供。"], - ["评分分母", "分项页数随方法变化但缺少直接的数量拆解。", "逐方法逐指标列出固定页数、成功交付、实际计分和已交付未计分的页数。", "沿用官方 success=true 计入规则，不设置逐页及格线。"], - ["结果交互", "请求失败易被误读为原始 URL 请求失败，案例和方法详情入口不够清晰。", "结果筛选使用已交付/未交付；Track ID 指向页面和参考，方法名指向该次评分与完整输出。搜索预处理并延迟触发以减少千行表筛选卡顿。", "原始 URL 在本轮仅为来源元数据。"], - ["方法详情", "物理调用、长标识和双栏长文本占据主结果版面。", "主视图只展示最终采用的结果、官方分项评分和完整输出；调用明细、来源哈希可展开核验。", "物理调用证据和累计耗时继续保留。"] + [ + "评测集", + "案例列表混入方法交付结果，冻结页面与人工参考难以独立查看。", + "案例列表只保留页面分类；每个 Track ID 可查看完整参考和冻结 HTML。", + "方法成绩在结果分析中继续提供。" + ], + [ + "评分分母", + "分项页数随方法变化但缺少直接的数量拆解。", + "逐方法逐指标列出固定页数、成功交付、实际计分和已交付未计分的页数。", + "沿用官方 success=true 计入规则，不设置逐页及格线。" + ], + [ + "结果交互", + "请求失败易被误读为原始 URL 请求失败，案例和方法详情入口不够清晰。", + "结果筛选使用已交付/未交付；Track ID 指向页面和参考，方法名指向该次评分与完整输出。搜索预处理并延迟触发以减少千行表筛选卡顿。", + "原始 URL 在本轮仅为来源元数据。" + ], + [ + "方法详情", + "物理调用、长标识和双栏长文本占据主结果版面。", + "主视图只展示最终采用的结果、官方分项评分和完整输出；调用明细、来源哈希可展开核验。", + "物理调用证据和累计耗时继续保留。" + ] ], "i18n": { "en": { @@ -2974,14 +3946,31 @@ { "id": "2026-09-15-public-webmainbench-reading-and-inspection", "publication_state": "published", - "comparison_kinds": ["webmainbench_frozen_html_to_markdown"], + "comparison_kinds": [ + "webmainbench_frozen_html_to_markdown" + ], "date": "2026-09-15", "title": "公开网页基准的逐页对照与分类筛选", "summary": "WebMainBench-545 预选 200 页的五方法分数、调用和官方评分未变。评测集可打开每页人工真值、冻结 HTML 与五种实际输出；分类分数改为按分类轴、类别和方法筛选，长表横向展开。", "changes": [ - ["评测集", "候选基准表和分类表在窄屏拥挤，案例入口埋在页面末尾。", "证据型长表在容器内横向滚动；200 页案例按官方分类筛选，并可继续打开人工真值、逐项评分原因和完整输出。", "只说明已测 200 页，不把样本分数称作 545 页官方总榜。"], - ["结果分析", "五个分类轴顺次展开，页面过长。", "按分类轴、类别和方法筛选同一张结果表，保留官方分项分数和各项有效分母。", "分类覆盖不等于比例代表性；无分母显示为未计算。"], - ["解读", "单看 overall 容易掩盖正文、代码、公式和表格差异。", "以同页实例连接各分项分榜、方法结果与完整人工真值。", "请求交付、质量和端到端耗时仍分别报告。"] + [ + "评测集", + "候选基准表和分类表在窄屏拥挤，案例入口埋在页面末尾。", + "证据型长表在容器内横向滚动；200 页案例按官方分类筛选，并可继续打开人工真值、逐项评分原因和完整输出。", + "只说明已测 200 页，不把样本分数称作 545 页官方总榜。" + ], + [ + "结果分析", + "五个分类轴顺次展开，页面过长。", + "按分类轴、类别和方法筛选同一张结果表，保留官方分项分数和各项有效分母。", + "分类覆盖不等于比例代表性；无分母显示为未计算。" + ], + [ + "解读", + "单看 overall 容易掩盖正文、代码、公式和表格差异。", + "以同页实例连接各分项分榜、方法结果与完整人工真值。", + "请求交付、质量和端到端耗时仍分别报告。" + ] ], "i18n": { "en": { @@ -2992,24 +3981,56 @@ { "id": "2026-09-15-resource-winner-presentation", "publication_state": "published", - "comparison_kinds": ["headless_html", "headless_markdown"], + "comparison_kinds": [ + "headless_html", + "headless_markdown" + ], "date": "2026-09-15", "title": "资源对比按指标分组并明确领先者", "summary": "首页与报告解读将 CPU、内存和网络拆为三个指标组，直接标出可按低消耗判断的逐项最低值，并在摘要中说明 Lightpanda 是本轮资源效率领先者。评测证据、任务判定和分母不变。", "changes": [ - ["资源结论", "资源数据平铺在一张宽表中，读者需要自行寻找最低值。", "增加资源效率结论卡和 CPU、内存、网络三张关键结果卡，明确 Lightpanda 在本轮低消耗指标中的领先。", "资源领先与综合任务通过率分别判断，不合成总分。"], - ["逐项比较", "表格只突出排序列，没有稳定标记每项最低值。", "按计算、内存、网络分组，逐项标记最低消耗；CPU 利用率和传输速率只描述运行形态，不按低值评选。", "沿用四种本地方法共同通过的 15/20 个普通网页，不产生新的供应商调用。"], - ["发布门禁", "资源展示要求依赖人工记忆。", "仓库规则和报告检查同时要求首页、解读页及中英文版具备赢家摘要、三类分组和逐项赢家标记。", "以后新增或重建报告时缺失任一部分都会阻断发布。"] - ] - , + [ + "资源结论", + "资源数据平铺在一张宽表中，读者需要自行寻找最低值。", + "增加资源效率结论卡和 CPU、内存、网络三张关键结果卡，明确 Lightpanda 在本轮低消耗指标中的领先。", + "资源领先与综合任务通过率分别判断，不合成总分。" + ], + [ + "逐项比较", + "表格只突出排序列，没有稳定标记每项最低值。", + "按计算、内存、网络分组，逐项标记最低消耗；CPU 利用率和传输速率只描述运行形态，不按低值评选。", + "沿用四种本地方法共同通过的 15/20 个普通网页，不产生新的供应商调用。" + ], + [ + "发布门禁", + "资源展示要求依赖人工记忆。", + "仓库规则和报告检查同时要求首页、解读页及中英文版具备赢家摘要、三类分组和逐项赢家标记。", + "以后新增或重建报告时缺失任一部分都会阻断发布。" + ] + ], "i18n": { "en": { "title": "Group resource comparisons and name the leaders", "summary": "The home page and interpretation split CPU, memory, and network into three metric groups, mark each lower-is-better minimum directly, and identify Lightpanda as the resource-efficiency leader in this run. Evaluation evidence, task verdicts, and denominators are unchanged.", "changes": [ - ["Resource findings", "Resource data was spread across one wide table, requiring readers to find minima themselves.", "Add a resource-efficiency finding card and three key-result cards for CPU, memory, and network, making Lightpanda's lower-use lead explicit.", "Resource leadership and overall task pass rate are judged separately and are not combined into a composite score."], - ["Per-metric comparison", "The table highlighted only the sorted column and did not persistently mark each metric's minimum.", "Group metrics into compute, memory, and network and mark the lowest use in each; CPU utilization and transfer rates describe runtime behavior and are not ranked by lower values.", "Retain the 15/20 general pages passed by all four local methods, with no new provider calls."], - ["Release gate", "Resource-presentation requirements depended on manual memory.", "Repository rules and report checks now require a winner summary, three grouped sections, and per-metric winner markers on the home and interpretation pages in both languages.", "Future report creation or rebuilding is blocked if any of these elements is missing."] + [ + "Resource findings", + "Resource data was spread across one wide table, requiring readers to find minima themselves.", + "Add a resource-efficiency finding card and three key-result cards for CPU, memory, and network, making Lightpanda's lower-use lead explicit.", + "Resource leadership and overall task pass rate are judged separately and are not combined into a composite score." + ], + [ + "Per-metric comparison", + "The table highlighted only the sorted column and did not persistently mark each metric's minimum.", + "Group metrics into compute, memory, and network and mark the lowest use in each; CPU utilization and transfer rates describe runtime behavior and are not ranked by lower values.", + "Retain the 15/20 general pages passed by all four local methods, with no new provider calls." + ], + [ + "Release gate", + "Resource-presentation requirements depended on manual memory.", + "Repository rules and report checks now require a winner summary, three grouped sections, and per-metric winner markers on the home and interpretation pages in both languages.", + "Future report creation or rebuilding is blocked if any of these elements is missing." + ] ] } } @@ -3017,27 +4038,69 @@ { "id": "2026-09-14-five-method-resource-profile", "publication_state": "published", - "comparison_kinds": ["headless_html", "headless_markdown"], + "comparison_kinds": [ + "headless_html", + "headless_markdown" + ], "date": "2026-09-14", "scoring_revision_input_sha256": "8e8edd5c6e327010b480d98df95e949589491e5b5d91963641d58303bc735f98", "title": "增加 CPU、内存、网络实测并收敛 Moli 版本", "summary": "当前主报告只保留 Moli 1.1.1，并为四种可本地观测的方法增加统一隔离的 CPU、内存和网络比较。资源结果来自 20 题 × 4 方法的 80 次正式调用，汇总使用共同通过的 15 题。", "changes": [ - ["当前方法矩阵", "三个 Moli 版本同时展示会扩大后续全量重跑和复核范围，也偏离当前产品比较目标。", "主报告保留 Kitesurf、Lightpanda、Obscura、Moli 1.1.1、Chromium 五种方法；Moli 1.0.6 和 1.1.3 的历史证据继续保留，但不进入当前页面、分母和榜单。", "主矩阵复用已经完成并核验的 750 条五方法结果，没有重新调用供应商；同窗测速的共同成功分母由七配置的 8 题变为当前五配置的 9 题。"], - ["资源比较", "旧报告只有端到端耗时，无法区分浏览器客户端对 CPU、内存和网络的占用。", "预声明 20 个普通网页，单并发运行四种本地方法；每次调用限制为 2 CPU、2 GiB 内存和 512 PID，并保存累计 CPU、内存采样及网络收发计数。", "只在四种方法共同通过的 15 题上汇总；失败结果仍保留在逐次证据中但不用于资源排名。"], - ["隔离与复用", "宿主机进程采样会混入其他任务，旧版成品镜像的 Moli 和 Chromium 版本又与当前口径不一致。", "抽出共享的 Linux/cgroup 资源计量与容器限制组件，Web Fetch 每次使用一次性容器和专用网络；Browser Use 复用相同基础计量。固定资源镜像构建一次并按镜像 ID 复用。", "现有 Docker 下载层可复用；旧 Moli 1.1.3 和 Chromium 151 成品镜像不冒充当前 Moli 1.1.1 与 Chromium 152。"], - ["适用边界", "远程 Kitesurf 的客户端请求流量不能代表其浏览器服务端的 CPU、内存和上游网络。", "Kitesurf 继续参加正确率和端到端耗时比较，但从资源消耗表中排除并说明原因。", "50 ms 采样可能遗漏更短峰值；本轮只能解释 arm64 Linux 隔离容器中的客户端成本。"] - ] - , + [ + "当前方法矩阵", + "三个 Moli 版本同时展示会扩大后续全量重跑和复核范围，也偏离当前产品比较目标。", + "主报告保留 Kitesurf、Lightpanda、Obscura、Moli 1.1.1、Chromium 五种方法；Moli 1.0.6 和 1.1.3 的历史证据继续保留，但不进入当前页面、分母和榜单。", + "主矩阵复用已经完成并核验的 750 条五方法结果，没有重新调用供应商；同窗测速的共同成功分母由七配置的 8 题变为当前五配置的 9 题。" + ], + [ + "资源比较", + "旧报告只有端到端耗时，无法区分浏览器客户端对 CPU、内存和网络的占用。", + "预声明 20 个普通网页，单并发运行四种本地方法；每次调用限制为 2 CPU、2 GiB 内存和 512 PID，并保存累计 CPU、内存采样及网络收发计数。", + "只在四种方法共同通过的 15 题上汇总；失败结果仍保留在逐次证据中但不用于资源排名。" + ], + [ + "隔离与复用", + "宿主机进程采样会混入其他任务，旧版成品镜像的 Moli 和 Chromium 版本又与当前口径不一致。", + "抽出共享的 Linux/cgroup 资源计量与容器限制组件，Web Fetch 每次使用一次性容器和专用网络；Browser Use 复用相同基础计量。固定资源镜像构建一次并按镜像 ID 复用。", + "现有 Docker 下载层可复用；旧 Moli 1.1.3 和 Chromium 151 成品镜像不冒充当前 Moli 1.1.1 与 Chromium 152。" + ], + [ + "适用边界", + "远程 Kitesurf 的客户端请求流量不能代表其浏览器服务端的 CPU、内存和上游网络。", + "Kitesurf 继续参加正确率和端到端耗时比较，但从资源消耗表中排除并说明原因。", + "50 ms 采样可能遗漏更短峰值；本轮只能解释 arm64 Linux 隔离容器中的客户端成本。" + ] + ], "i18n": { "en": { "title": "Add measured CPU, memory, and network use and narrow the Moli version", "summary": "The current primary report retains only Moli 1.1.1 and adds a uniformly isolated CPU, memory, and network comparison for four locally observable methods. Resource results come from 80 formal calls across 20 tasks × 4 methods, aggregated over the 15 shared passes.", "changes": [ - ["Current method matrix", "Showing three Moli versions would expand every future rerun and review and distract from the current product comparison.", "The primary report retains Kitesurf, Lightpanda, Obscura, Moli 1.1.1, and Chromium. Historical evidence for Moli 1.0.6 and 1.1.3 remains archived but is excluded from current pages, denominators, and rankings.", "The primary matrix reuses 750 completed and reviewed five-method results without new provider calls. Removing the two historical configurations changes the same-window shared-pass denominator from 8 tasks across seven configurations to 9 tasks across the current five."], - ["Resource comparison", "The previous report had only end-to-end latency and could not distinguish browser-client CPU, memory, and network use.", "20 general pages were predeclared and run one at a time across four local methods. Each call was limited to 2 CPUs, 2 GiB of memory, and 512 PIDs, with cumulative CPU, sampled memory, and network byte counters retained.", "Aggregates use only the 15 tasks passed by all four methods. Failed results remain in per-attempt evidence but do not enter the resource ranking."], - ["Isolation and reuse", "Host-process sampling can include unrelated work, while the existing finished images use Moli and Chromium versions that do not match the current contract.", "Shared Linux/cgroup accounting and container-limit components now serve both evaluations. Web Fetch uses a disposable container and dedicated network per call, while Browser Use reuses the same accounting foundation. The fixed resource image is built once and reused by image ID.", "Existing Docker download layers are reusable, but finished images containing Moli 1.1.3 and Chromium 151 are not presented as the current Moli 1.1.1 and Chromium 152 environment."], - ["Scope limits", "Kitesurf's local request traffic cannot represent CPU, memory, or upstream network use in its remote browser service.", "Kitesurf remains in correctness and end-to-end latency comparisons but is excluded from the resource-use table with the reason disclosed.", "Sampling every 50 ms may miss shorter peaks. This run describes only client cost in isolated arm64 Linux containers."] + [ + "Current method matrix", + "Showing three Moli versions would expand every future rerun and review and distract from the current product comparison.", + "The primary report retains Kitesurf, Lightpanda, Obscura, Moli 1.1.1, and Chromium. Historical evidence for Moli 1.0.6 and 1.1.3 remains archived but is excluded from current pages, denominators, and rankings.", + "The primary matrix reuses 750 completed and reviewed five-method results without new provider calls. Removing the two historical configurations changes the same-window shared-pass denominator from 8 tasks across seven configurations to 9 tasks across the current five." + ], + [ + "Resource comparison", + "The previous report had only end-to-end latency and could not distinguish browser-client CPU, memory, and network use.", + "20 general pages were predeclared and run one at a time across four local methods. Each call was limited to 2 CPUs, 2 GiB of memory, and 512 PIDs, with cumulative CPU, sampled memory, and network byte counters retained.", + "Aggregates use only the 15 tasks passed by all four methods. Failed results remain in per-attempt evidence but do not enter the resource ranking." + ], + [ + "Isolation and reuse", + "Host-process sampling can include unrelated work, while the existing finished images use Moli and Chromium versions that do not match the current contract.", + "Shared Linux/cgroup accounting and container-limit components now serve both evaluations. Web Fetch uses a disposable container and dedicated network per call, while Browser Use reuses the same accounting foundation. The fixed resource image is built once and reused by image ID.", + "Existing Docker download layers are reusable, but finished images containing Moli 1.1.3 and Chromium 151 are not presented as the current Moli 1.1.1 and Chromium 152 environment." + ], + [ + "Scope limits", + "Kitesurf's local request traffic cannot represent CPU, memory, or upstream network use in its remote browser service.", + "Kitesurf remains in correctness and end-to-end latency comparisons but is excluded from the resource-use table with the reason disclosed.", + "Sampling every 50 ms may miss shorter peaks. This run describes only client cost in isolated arm64 Linux containers." + ] ] } } @@ -3046,7 +4109,9 @@ "id": "2026-09-15-official-route-and-call-drilldown", "publication_state": "published", "date": "2026-09-15", - "comparison_kinds": ["web_fetch_methods"], + "comparison_kinds": [ + "web_fetch_methods" + ], "scoring_revision_input_sha256": [ "596f8005399be74d73dd14b8a77f5f775b3e2dd36f037643c421259bb5d8ff53", "b8b7adec934761ed374191912c05ef2da449cb1a7cbda104f671db08d033a5ee" @@ -3054,25 +4119,84 @@ "title": "按任务修正官方接口并补齐逐次调用下钻", "summary": "十种方法仍执行同一组 150 题，保留 1,500 条权威结果。官方接口修正认证重跑 Cloudflare 23 条、Firecrawl 17 条，共 40 次调用；通过数由 1,024 增至 1,029。document-05 的公共格式归一化零调用增加 1 条通过；Cloudflare 两条视频链接任务又因适配器纠错增加 4 次认证调用。累计 44 次新物理调用，当前通过数 1,032/1,500；Moli 仍为 1.1.4。", "changes": [ - ["Cloudflare 任务接口", "原 /markdown 适合网页正文；部分直接 PDF/DOCX 出现接口成功但正文为空，页面图片和视频属性也不能稳定从 Markdown 取得。", "网页正文与动态题保留 Browser Run /markdown；直接 PDF/DOCX 上传原文件给 Workers AI /ai/tomarkdown；页面图片和视频地址使用 Browser Run /scrape 的 DOM 属性。下载原文件与转换共用一次端到端预算。", "仅替换 Cloudflare 的 7 条直接文档、9 条页面图片和 7 条页面视频结果；/pdf 生成网页 PDF，/links 只列普通链接，/crawl 扩展到其他页面，均不替代这些题的交付物。"], - ["Cloudflare 视频地址适配", "/scrape 已返回 Blender 的 .mp4.zip 地址和 Bilibili 的 BV 页面入口，旧适配器只接纳常见裸媒体扩展名，因而把目标地址丢掉；放宽后又把普通导航和工具 iframe 误列为视频。", "下载包保留 .mp4.zip 形态；视频详情入口仅请求 a[href]，再按视频页面路由筛选，不用标准答案挑选输出。两条受影响任务均按纠正后的接口认证重跑。", "video-blender-release 与 video-bilibili-current 最终均通过；Bilibili 输出只含站内 BV 详情地址。中间调用和旧物理证据保留，但不择优进入权威矩阵。"], - ["Firecrawl 任务接口", "动态题的固定 waitFor 延迟不能执行已声明的顶层选择器条件；页面视频题漏取 Scrape 的 video 字段。", "动态题预选 Scrape Actions 的 selector wait；视频资源题同时请求原生 video 格式，并保留正文和资源地址。匹配数量与 iframe 子文档完成条件仍按接口边界披露。", "仅替换 10 条动态与 7 条页面视频结果。video 下载入口不等于媒体原字节或画面理解；Search、Crawl 与 Agent 不用于给定 URL 的单页任务。"], - ["成绩与同口径分母", "只改结果表会掩盖接口和评分入口变化，也会让读者把重新调用与离线复算混为一谈。", "40 个方法—题目组合换用官方接口；两条视频链接组合又因适配错误增加 4 次物理调用。44 次新调用完整保留，最终权威矩阵仍只含 40 条新调用结果与 1,460 条原调用结果。document-05 的 Markdown 表格分隔行在公共格式归一化中按排版处理，保存输出离线复评分。", "通过数依次为 1,024/1,500、接口纠正后 1,029/1,500、零调用归一化后 1,030/1,500、两条视频适配纠错后 1,032/1,500。分母、150 题、十方法和 Moli 1.1.4 均不变。"], - ["核验与资源别名", "缩略图可由同一资产的另一 Wikimedia 域名交付，单看主机名会错判 image-04；反过来，无缩略图标识的同名地址也不能任意放行。", "40 条新结果和 1,460 条旧结果均有逐条独立核验；最后两条 Cloudflare 输出重新盲审，严格复用身份不变的 1,498 条核验。image-04 按同资产缩略图域名迁移裁决，公共资源规则同时拒绝缺少缩略图标识的近似地址。", "当前 1,500/1,500 条权威结果有有效逐条判定核验，未决判定 0。image-04 修正未改变成绩；异常根因调查另有独立门禁。"], - ["逐次调用日志", "旧详情能看到最终输出，但输出为空时，读者看不到接口状态、最后错误或每次重试的过程。", "新物理调用保存脱敏且哈希绑定的请求摘要、接口/进程状态、最后输出、错误与重试记录；详情页按需打开弹窗。完整原始供应商响应仍留在私有证据，不随公开包发布。", "本轮全部 40 次新调用有可下钻安全日志；历史调用只展示实际留存字段，并明确标示没有保存的日志，不补造历史 stdout、stderr 或原始信封。" ] - ] - , + [ + "Cloudflare 任务接口", + "原 /markdown 适合网页正文；部分直接 PDF/DOCX 出现接口成功但正文为空，页面图片和视频属性也不能稳定从 Markdown 取得。", + "网页正文与动态题保留 Browser Run /markdown；直接 PDF/DOCX 上传原文件给 Workers AI /ai/tomarkdown；页面图片和视频地址使用 Browser Run /scrape 的 DOM 属性。下载原文件与转换共用一次端到端预算。", + "仅替换 Cloudflare 的 7 条直接文档、9 条页面图片和 7 条页面视频结果；/pdf 生成网页 PDF，/links 只列普通链接，/crawl 扩展到其他页面，均不替代这些题的交付物。" + ], + [ + "Cloudflare 视频地址适配", + "/scrape 已返回 Blender 的 .mp4.zip 地址和 Bilibili 的 BV 页面入口，旧适配器只接纳常见裸媒体扩展名，因而把目标地址丢掉；放宽后又把普通导航和工具 iframe 误列为视频。", + "下载包保留 .mp4.zip 形态；视频详情入口仅请求 a[href]，再按视频页面路由筛选，不用标准答案挑选输出。两条受影响任务均按纠正后的接口认证重跑。", + "video-blender-release 与 video-bilibili-current 最终均通过；Bilibili 输出只含站内 BV 详情地址。中间调用和旧物理证据保留，但不择优进入权威矩阵。" + ], + [ + "Firecrawl 任务接口", + "动态题的固定 waitFor 延迟不能执行已声明的顶层选择器条件；页面视频题漏取 Scrape 的 video 字段。", + "动态题预选 Scrape Actions 的 selector wait；视频资源题同时请求原生 video 格式，并保留正文和资源地址。匹配数量与 iframe 子文档完成条件仍按接口边界披露。", + "仅替换 10 条动态与 7 条页面视频结果。video 下载入口不等于媒体原字节或画面理解；Search、Crawl 与 Agent 不用于给定 URL 的单页任务。" + ], + [ + "成绩与同口径分母", + "只改结果表会掩盖接口和评分入口变化，也会让读者把重新调用与离线复算混为一谈。", + "40 个方法—题目组合换用官方接口；两条视频链接组合又因适配错误增加 4 次物理调用。44 次新调用完整保留，最终权威矩阵仍只含 40 条新调用结果与 1,460 条原调用结果。document-05 的 Markdown 表格分隔行在公共格式归一化中按排版处理，保存输出离线复评分。", + "通过数依次为 1,024/1,500、接口纠正后 1,029/1,500、零调用归一化后 1,030/1,500、两条视频适配纠错后 1,032/1,500。分母、150 题、十方法和 Moli 1.1.4 均不变。" + ], + [ + "核验与资源别名", + "缩略图可由同一资产的另一 Wikimedia 域名交付，单看主机名会错判 image-04；反过来，无缩略图标识的同名地址也不能任意放行。", + "40 条新结果和 1,460 条旧结果均有逐条独立核验；最后两条 Cloudflare 输出重新盲审，严格复用身份不变的 1,498 条核验。image-04 按同资产缩略图域名迁移裁决，公共资源规则同时拒绝缺少缩略图标识的近似地址。", + "当前 1,500/1,500 条权威结果有有效逐条判定核验，未决判定 0。image-04 修正未改变成绩；异常根因调查另有独立门禁。" + ], + [ + "逐次调用日志", + "旧详情能看到最终输出，但输出为空时，读者看不到接口状态、最后错误或每次重试的过程。", + "新物理调用保存脱敏且哈希绑定的请求摘要、接口/进程状态、最后输出、错误与重试记录；详情页按需打开弹窗。完整原始供应商响应仍留在私有证据，不随公开包发布。", + "本轮全部 40 次新调用有可下钻安全日志；历史调用只展示实际留存字段，并明确标示没有保存的日志，不补造历史 stdout、stderr 或原始信封。" + ] + ], "i18n": { "en": { "title": "Correct official routes by task and add per-call drilldown", "summary": "The ten methods still run the same 150 tasks, with 1,500 authoritative results. Official-route corrections certified 23 Cloudflare and 17 Firecrawl calls, 40 in total, raising passes from 1,024 to 1,029. A shared format-normalization correction added 1 document-05 pass without a provider call. Two Cloudflare video-link tasks then required 4 additional certified adapter-correction calls. The release retains 44 new physical calls and now passes 1,032/1,500 tasks; Moli remains at 1.1.4.", "changes": [ - ["Cloudflare routes by task", "The former /markdown route suits webpage text. Some direct PDF/DOCX requests returned a successful interface response with empty body text, and page image and video attributes could not be reliably recovered from Markdown.", "Web text and dynamic tasks retain Browser Run /markdown. Direct PDF/DOCX files are uploaded to Workers AI /ai/tomarkdown; page image and video URLs come from Browser Run /scrape DOM attributes. Original-file download and conversion share one end-to-end deadline.", "Only Cloudflare's 7 direct-document, 9 page-image, and 7 page-video results were replaced. /pdf generates a webpage PDF, /links lists ordinary links, and /crawl expands to other pages; none substitutes for the deliverables in these tasks."], - ["Cloudflare video-URL adaptation", "/scrape had returned Blender's .mp4.zip download URL and Bilibili BV detail links, but the old adapter discarded them by requiring a bare media extension. A broad relaxation then mislabeled ordinary navigation and utility iframes as video entries.", "The archive route keeps .mp4.zip downloads. Video-detail tasks request only a[href] and filter by video-page URL route, without selecting output against the reference answer. Both affected tasks were certified again under the corrected interface.", "video-blender-release and video-bilibili-current both pass in the final matrix; the Bilibili output contains only on-site BV detail URLs. Intermediate and earlier physical calls remain retained but are not selected by score."], - ["Firecrawl routes by task", "A fixed waitFor delay could not enforce the declared top-level selector for dynamic tasks; page-video tasks omitted Scrape's video field.", "Dynamic tasks preselect Scrape Actions selector waits. Video-resource tasks also request the native video format while retaining text and resource URLs. Match-count and iframe-child completion conditions remain disclosed as interface limits.", "Only 10 dynamic and 7 page-video results were replaced. A video download URL is neither original media bytes nor visual understanding. Search, Crawl, and Agent were not used for single-page tasks with a given URL."], - ["Results on the same denominator", "Editing only a results table would conceal changes to provider routes and the shared scorer, and blur the distinction between new calls and offline rescoring.", "40 method–task pairs use corrected official routes; two video-link pairs added 4 physical adapter-correction calls. All 44 new calls are retained, while the authoritative matrix contains 40 final replacement results and 1,460 inherited results. document-05's Markdown table-divider row is treated as layout by shared normalization and its saved output was rescored offline.", "Passes moved from 1,024/1,500 to 1,029/1,500 after route corrections, 1,030/1,500 after zero-call normalization, and 1,032/1,500 after video-adapter corrections. The denominator, 150 tasks, ten methods, and Moli 1.1.4 are unchanged."], - ["Review and resource aliases", "A thumbnail of the same asset may be served from another Wikimedia host; comparing hostnames alone would misjudge image-04. A similarly named URL without a thumbnail marker must not be accepted indiscriminately.", "All 40 new and 1,460 inherited results have per-attempt independent review. The final two Cloudflare outputs were blindly reviewed again, strictly reusing 1,498 unchanged review identities. image-04 was adjudicated as a same-asset thumbnail-host migration, while the shared resource rule rejects a lookalike URL without the thumbnail path.", "All 1,500/1,500 authoritative results have valid per-attempt verdict review, with 0 unresolved verdicts. The image-04 correction did not change scores; root-cause anomaly investigation has a separate gate."], - ["Per-call logs", "Earlier detail pages showed final output, but an empty answer did not reveal interface status, the final error, or each retry.", "New physical calls retain redacted, hash-bound request summaries, interface/process status, final output, errors, and retry records. The detail page opens this evidence on demand in a dialog. Full raw provider responses remain private and are not included in the public bundle.", "All 40 new calls in this release have safe drilldown logs. Older calls show only fields actually retained and explicitly identify unavailable logs; historical stdout, stderr, and raw envelopes are not fabricated."] + [ + "Cloudflare routes by task", + "The former /markdown route suits webpage text. Some direct PDF/DOCX requests returned a successful interface response with empty body text, and page image and video attributes could not be reliably recovered from Markdown.", + "Web text and dynamic tasks retain Browser Run /markdown. Direct PDF/DOCX files are uploaded to Workers AI /ai/tomarkdown; page image and video URLs come from Browser Run /scrape DOM attributes. Original-file download and conversion share one end-to-end deadline.", + "Only Cloudflare's 7 direct-document, 9 page-image, and 7 page-video results were replaced. /pdf generates a webpage PDF, /links lists ordinary links, and /crawl expands to other pages; none substitutes for the deliverables in these tasks." + ], + [ + "Cloudflare video-URL adaptation", + "/scrape had returned Blender's .mp4.zip download URL and Bilibili BV detail links, but the old adapter discarded them by requiring a bare media extension. A broad relaxation then mislabeled ordinary navigation and utility iframes as video entries.", + "The archive route keeps .mp4.zip downloads. Video-detail tasks request only a[href] and filter by video-page URL route, without selecting output against the reference answer. Both affected tasks were certified again under the corrected interface.", + "video-blender-release and video-bilibili-current both pass in the final matrix; the Bilibili output contains only on-site BV detail URLs. Intermediate and earlier physical calls remain retained but are not selected by score." + ], + [ + "Firecrawl routes by task", + "A fixed waitFor delay could not enforce the declared top-level selector for dynamic tasks; page-video tasks omitted Scrape's video field.", + "Dynamic tasks preselect Scrape Actions selector waits. Video-resource tasks also request the native video format while retaining text and resource URLs. Match-count and iframe-child completion conditions remain disclosed as interface limits.", + "Only 10 dynamic and 7 page-video results were replaced. A video download URL is neither original media bytes nor visual understanding. Search, Crawl, and Agent were not used for single-page tasks with a given URL." + ], + [ + "Results on the same denominator", + "Editing only a results table would conceal changes to provider routes and the shared scorer, and blur the distinction between new calls and offline rescoring.", + "40 method–task pairs use corrected official routes; two video-link pairs added 4 physical adapter-correction calls. All 44 new calls are retained, while the authoritative matrix contains 40 final replacement results and 1,460 inherited results. document-05's Markdown table-divider row is treated as layout by shared normalization and its saved output was rescored offline.", + "Passes moved from 1,024/1,500 to 1,029/1,500 after route corrections, 1,030/1,500 after zero-call normalization, and 1,032/1,500 after video-adapter corrections. The denominator, 150 tasks, ten methods, and Moli 1.1.4 are unchanged." + ], + [ + "Review and resource aliases", + "A thumbnail of the same asset may be served from another Wikimedia host; comparing hostnames alone would misjudge image-04. A similarly named URL without a thumbnail marker must not be accepted indiscriminately.", + "All 40 new and 1,460 inherited results have per-attempt independent review. The final two Cloudflare outputs were blindly reviewed again, strictly reusing 1,498 unchanged review identities. image-04 was adjudicated as a same-asset thumbnail-host migration, while the shared resource rule rejects a lookalike URL without the thumbnail path.", + "All 1,500/1,500 authoritative results have valid per-attempt verdict review, with 0 unresolved verdicts. The image-04 correction did not change scores; root-cause anomaly investigation has a separate gate." + ], + [ + "Per-call logs", + "Earlier detail pages showed final output, but an empty answer did not reveal interface status, the final error, or each retry.", + "New physical calls retain redacted, hash-bound request summaries, interface/process status, final output, errors, and retry records. The detail page opens this evidence on demand in a dialog. Full raw provider responses remain private and are not included in the public bundle.", + "All 40 new calls in this release have safe drilldown logs. Older calls show only fields actually retained and explicitly identify unavailable logs; historical stdout, stderr, and raw envelopes are not fabricated." + ] ] } } @@ -3081,7 +4205,9 @@ "id": "2026-09-11-full-matrix-root-cause-audit", "publication_state": "published", "date": "2026-09-14", - "comparison_kinds": ["web_fetch_methods"], + "comparison_kinds": [ + "web_fetch_methods" + ], "scoring_revision_input_sha256": [ "9b85443f840063956126bc233584e4cbe1d884bde1eb0c59571ce02fdd419697", "3b49615c9eda1503e0952d64187fc39b13207478d40e03effa7100ba6ddc0d41", @@ -3103,40 +4229,159 @@ "7aaa2d5ce43729b2d351666fb4e346ed14222cde152fe82f044e11cfc78e9c7d", "5202c38ad76b3605094f539ae01bc168aac53b585d41a92b38b1aafc09487c45" ], - "title": "修复评测全链路根因并设置全矩阵核验门禁", - "summary": "当前十种方法的 1,500 条主结果已全部独立核验，未决为 0；其中 309 条异常触发结果进一步完成输入、参考、请求、原始响应、适配器、评分器和最终判定七层审查。已修复正文分段、具名链接、脚本状态、文档地标、请求状态、动态卡片边界、公开路径泄漏和参考元数据等共因；仅认证重跑接口或产物身份受影响的声明范围，其余修正复用原始产物离线完成。Moli 保持 1.1.4。", - "changes": [ - ["全量结果核验", "上一版只独立审查机器失败，机器通过尚未逐条核验，无法发现被宽松规则放行的系统性假阳性。", "机器通过与失败统一使用隐藏机器结论的审查包；任务和审查可见证据哈希均未变化的审查继续有效，只重审发生变化的结果。", "当前十方法矩阵审查覆盖 1,500/1,500。修正前审查提出 56 条代理与机器分歧，其中 2 条确认评分错误并先修根因；其余均依据实际产物、标准答案和任务合同裁决，未决为 0。同窗测速只作时延测量，不在结果审查范围。"], - ["正文、动态与挑战页评分", "旧评分实现可能在代码更新后继续被报告读取；分段正文按参考顺序贪心匹配会让较长实际片段先占用错误区域；序列化脚本状态含目标字符串时又可能被当成可读正文。", "报告构建拒绝旧评分实现；正文片段改为全局容量分配且不受参考区域顺序影响；脚本或应用状态壳不能满足正文任务。", "两类修正均通过公共 score_attempt 与任务判定入口执行正反向回归；保存输出统一离线重算，不按方法或单条结果增加特例。"], - ["Exa 具名链接接口", "旧适配器只读取裸 links 数组，既无法证明名称与 URL 的归属，又漏用了官方返回明确 anchor—URL 配对的 richLinks。", "仅具名链接任务请求 extras.richLinks，并只接纳 anchor 与 URL 均非空的明确配对；正文标题与裸 links 永不按位置猜配。", "接口合同变化只影响 Exa 的 10 个挑战任务，因此通过认证流程重跑该范围；3 条通过、7 条仍因实际产物不满足合同而失败。"], - ["DOCX 完整内容", "document-05 只要求标题和章节名，导致返回正文叙述却漏掉表格单元格、日历与最终尾注的结果仍可通过。", "任务合同增加来自冻结源文档的表格、日历和尾注内容地标；Markdown 表格分隔符按排版处理，不能把完整表格误判失败。", "保存产物统一离线复评分：CowAgent 的不完整结果改为失败，Firecrawl 与 Tavily 的完整 Markdown 表格保持通过；不需要重新调用供应商。"], - ["图片资源范围", "图片地址题一度被拟议成 target-only 输出，可能把已经命中目标、但同时保留页面 Logo 或推荐图的有效抓取误判失败；部分响应式目标别名又需要独立源证据。", "资源验收只要求命中任务声明的目标或有证据的合法别名；额外页面图片保留为纯净度诊断，不参与二元通过判定。目标清单、别名、来源观察和资源合同均进入哈希绑定。", "避免把页面级抓取强行等同于只返回一个 URL，同时仍拒绝没有命中目标的长文本或大量无关图片；契约变化后旧分数不能发布。"], - ["防复发与证据修订", "评分、参考、数据集和审查收据可分别更新，若生命周期只校验部分字段，旧核验可能被错误沿用；只改最终判定也无法阻止同类错误复发。", "每类根因登记正反向可执行回归；参考修订和评分修订分别串联完整输入输出哈希、根因与回归编号，报告构建校验评分实现、任务内容、资源合同和来源证据。", "评分与参考修订不重发 provider 请求；只有接口、版本、输入或请求合同改变的预声明范围才重跑，并保留原调用与新调用的可追溯关系。"], - ["Firecrawl 动态接口", "旧适配器使用当前账户地区不允许的 actions，10 个动态任务都在服务入口失败，错误地形成 0/10 的能力结论。", "改用当前公开接口支持的 waitFor，并通过调用前后参考冻结的认证流程只重跑这 10 个任务；其余 140 条 Firecrawl 结果不变。", "新结果 6/10 通过；新增 10 条全部由隐藏机器结论的 Codex CLI 盲审，0 分歧、0 未决。剩余 4 条因实际输出不满足任务合同而失败。"], - ["报告阅读与耗时", "全部方法共同成功任务的跨批次耗时表默认展开，占据总榜下方较大篇幅；综合总榜没有同时给出完整 150 次调用的时延分布。", "共同成功任务耗时改为按需展开；综合总榜新增每种方法全部 150 次调用的均值、P50、P95，和通过率使用相同固定分母。", "时延列不参与通过率排序，也不选择成功子集；失败调用耗时保留在全调用分布中。"], - ["请求状态与空输出", "旧适配器把供应商调用成功但没有可验收正文，与供应商或目标请求失败混成同一种失败，导致请求成功率与任务通过率语义串线。", "Cloudflare 依据保存的 success、result 与 origin status 信封归一化；Moli 依据 1.1.4 的退出状态和完整响应错误信封归一化。空的成功响应仍是请求成功、任务失败；明确的目标非 2xx 仍是请求失败。", "14 条状态修正均由哈希绑定的原始证据离线重建，任务判定和分数不变，无 provider 调用。"], - ["动态提取与公开产物", "Crawl4AI 动态购物车请求声明了限定提取，但 worker 未传入对应配置并丢失卡片边界；OpenClaw 错误堆栈和 CowAgent 保存提示又会把本机路径带入公开证据。", "Crawl4AI worker 传入已声明的提取配置并保留卡片间边界；OpenClaw/DeepSeek 错误面只保留稳定消息，CowAgent 仅删除本机保存路径而保留抓取正文。", "Crawl4AI 该题认证重跑后通过，方法总分增至 119/150；OpenClaw 15 条和 CowAgent 11 条受影响产物认证重跑，任务总分不变，公开产物不再包含真实主机路径。"], - ["参考证据一致性", "四个完整正文样本的来源说明曾错误指向其他网站；NOAA 样本已有当日独立 200 响应、main 区域和完整正文哈希，却保留了“全文不可用”的旧备注。", "完整正文来源必须与题目 URL 同站；NOAA 元数据改为明确记录完整参考可用，稳定锚点继续决定二元验收，全文快照只提供诊断指标。新增数据集门禁拒绝有效全文与不可用声明并存。", "来源与备注修正不改变 provider 输出或任务分数；相关结果重新绑定并独立复核。"], - ["异常根因门禁", "仅比较标准答案与最终判定无法主动发现空输出、跨站相同错误、集中零分、接口误用、参考冲突和公开产物泄漏；按批次序号缓存又会在小修后错配审查结果。", "逐条盲审之后，对确定性异常信号执行七层调查；异常缓存按精确 attempt 范围而不是批次位置复用，范围变化时只重审变化组。", "最终矩阵 1,500/1,500 已核验，309 条异常触发结果全部完成调查，未决 0；缓存范围、参考修订和最终审查均有哈希绑定。"] - ] - , + "title": "修复评测全链路根因并设置全矩阵核验门禁", + "summary": "当前十种方法的 1,500 条主结果已全部独立核验，未决为 0；其中 309 条异常触发结果进一步完成输入、参考、请求、原始响应、适配器、评分器和最终判定七层审查。已修复正文分段、具名链接、脚本状态、文档地标、请求状态、动态卡片边界、公开路径泄漏和参考元数据等共因；仅认证重跑接口或产物身份受影响的声明范围，其余修正复用原始产物离线完成。Moli 保持 1.1.4。", + "changes": [ + [ + "全量结果核验", + "上一版只独立审查机器失败，机器通过尚未逐条核验，无法发现被宽松规则放行的系统性假阳性。", + "机器通过与失败统一使用隐藏机器结论的审查包；任务和审查可见证据哈希均未变化的审查继续有效，只重审发生变化的结果。", + "当前十方法矩阵审查覆盖 1,500/1,500。修正前审查提出 56 条代理与机器分歧，其中 2 条确认评分错误并先修根因；其余均依据实际产物、标准答案和任务合同裁决，未决为 0。同窗测速只作时延测量，不在结果审查范围。" + ], + [ + "正文、动态与挑战页评分", + "旧评分实现可能在代码更新后继续被报告读取；分段正文按参考顺序贪心匹配会让较长实际片段先占用错误区域；序列化脚本状态含目标字符串时又可能被当成可读正文。", + "报告构建拒绝旧评分实现；正文片段改为全局容量分配且不受参考区域顺序影响；脚本或应用状态壳不能满足正文任务。", + "两类修正均通过公共 score_attempt 与任务判定入口执行正反向回归；保存输出统一离线重算，不按方法或单条结果增加特例。" + ], + [ + "Exa 具名链接接口", + "旧适配器只读取裸 links 数组，既无法证明名称与 URL 的归属，又漏用了官方返回明确 anchor—URL 配对的 richLinks。", + "仅具名链接任务请求 extras.richLinks，并只接纳 anchor 与 URL 均非空的明确配对；正文标题与裸 links 永不按位置猜配。", + "接口合同变化只影响 Exa 的 10 个挑战任务，因此通过认证流程重跑该范围；3 条通过、7 条仍因实际产物不满足合同而失败。" + ], + [ + "DOCX 完整内容", + "document-05 只要求标题和章节名，导致返回正文叙述却漏掉表格单元格、日历与最终尾注的结果仍可通过。", + "任务合同增加来自冻结源文档的表格、日历和尾注内容地标；Markdown 表格分隔符按排版处理，不能把完整表格误判失败。", + "保存产物统一离线复评分：CowAgent 的不完整结果改为失败，Firecrawl 与 Tavily 的完整 Markdown 表格保持通过；不需要重新调用供应商。" + ], + [ + "图片资源范围", + "图片地址题一度被拟议成 target-only 输出，可能把已经命中目标、但同时保留页面 Logo 或推荐图的有效抓取误判失败；部分响应式目标别名又需要独立源证据。", + "资源验收只要求命中任务声明的目标或有证据的合法别名；额外页面图片保留为纯净度诊断，不参与二元通过判定。目标清单、别名、来源观察和资源合同均进入哈希绑定。", + "避免把页面级抓取强行等同于只返回一个 URL，同时仍拒绝没有命中目标的长文本或大量无关图片；契约变化后旧分数不能发布。" + ], + [ + "防复发与证据修订", + "评分、参考、数据集和审查收据可分别更新，若生命周期只校验部分字段，旧核验可能被错误沿用；只改最终判定也无法阻止同类错误复发。", + "每类根因登记正反向可执行回归；参考修订和评分修订分别串联完整输入输出哈希、根因与回归编号，报告构建校验评分实现、任务内容、资源合同和来源证据。", + "评分与参考修订不重发 provider 请求；只有接口、版本、输入或请求合同改变的预声明范围才重跑，并保留原调用与新调用的可追溯关系。" + ], + [ + "Firecrawl 动态接口", + "旧适配器使用当前账户地区不允许的 actions，10 个动态任务都在服务入口失败，错误地形成 0/10 的能力结论。", + "改用当前公开接口支持的 waitFor，并通过调用前后参考冻结的认证流程只重跑这 10 个任务；其余 140 条 Firecrawl 结果不变。", + "新结果 6/10 通过；新增 10 条全部由隐藏机器结论的 Codex CLI 盲审，0 分歧、0 未决。剩余 4 条因实际输出不满足任务合同而失败。" + ], + [ + "报告阅读与耗时", + "全部方法共同成功任务的跨批次耗时表默认展开，占据总榜下方较大篇幅；综合总榜没有同时给出完整 150 次调用的时延分布。", + "共同成功任务耗时改为按需展开；综合总榜新增每种方法全部 150 次调用的均值、P50、P95，和通过率使用相同固定分母。", + "时延列不参与通过率排序，也不选择成功子集；失败调用耗时保留在全调用分布中。" + ], + [ + "请求状态与空输出", + "旧适配器把供应商调用成功但没有可验收正文，与供应商或目标请求失败混成同一种失败，导致请求成功率与任务通过率语义串线。", + "Cloudflare 依据保存的 success、result 与 origin status 信封归一化；Moli 依据 1.1.4 的退出状态和完整响应错误信封归一化。空的成功响应仍是请求成功、任务失败；明确的目标非 2xx 仍是请求失败。", + "14 条状态修正均由哈希绑定的原始证据离线重建，任务判定和分数不变，无 provider 调用。" + ], + [ + "动态提取与公开产物", + "Crawl4AI 动态购物车请求声明了限定提取，但 worker 未传入对应配置并丢失卡片边界；OpenClaw 错误堆栈和 CowAgent 保存提示又会把本机路径带入公开证据。", + "Crawl4AI worker 传入已声明的提取配置并保留卡片间边界；OpenClaw/DeepSeek 错误面只保留稳定消息，CowAgent 仅删除本机保存路径而保留抓取正文。", + "Crawl4AI 该题认证重跑后通过，方法总分增至 119/150；OpenClaw 15 条和 CowAgent 11 条受影响产物认证重跑，任务总分不变，公开产物不再包含真实主机路径。" + ], + [ + "参考证据一致性", + "四个完整正文样本的来源说明曾错误指向其他网站；NOAA 样本已有当日独立 200 响应、main 区域和完整正文哈希，却保留了“全文不可用”的旧备注。", + "完整正文来源必须与题目 URL 同站；NOAA 元数据改为明确记录完整参考可用，稳定锚点继续决定二元验收，全文快照只提供诊断指标。新增数据集门禁拒绝有效全文与不可用声明并存。", + "来源与备注修正不改变 provider 输出或任务分数；相关结果重新绑定并独立复核。" + ], + [ + "异常根因门禁", + "仅比较标准答案与最终判定无法主动发现空输出、跨站相同错误、集中零分、接口误用、参考冲突和公开产物泄漏；按批次序号缓存又会在小修后错配审查结果。", + "逐条盲审之后，对确定性异常信号执行七层调查；异常缓存按精确 attempt 范围而不是批次位置复用，范围变化时只重审变化组。", + "最终矩阵 1,500/1,500 已核验，309 条异常触发结果全部完成调查，未决 0；缓存范围、参考修订和最终审查均有哈希绑定。" + ] + ], "i18n": { "en": { "title": "Fix root causes across the evaluation chain and require full-matrix review", "summary": "All 1,500 primary results across ten methods were independently reviewed with 0 unresolved decisions. The 309 anomaly-triggered results also completed seven-layer review covering input, reference, request, raw response, adapter, scorer, and final verdict. Shared fixes cover body segmentation, named links, script-state text, document landmarks, request status, dynamic-card boundaries, public path leakage, and reference metadata. Only declared scopes whose interface or artifact identity changed were rerun; all other corrections reused saved outputs offline. Moli remains on 1.1.4.", "changes": [ - ["Full result review", "The previous release independently reviewed machine failures only. Machine passes had not been checked individually, leaving systematic false positives from permissive rules undetectable.", "Machine passes and failures use the same packets with machine judgments hidden. Prior reviews remain valid only when both the task and reviewer-visible evidence hash are unchanged; only changed results are reviewed again.", "Review covers 1,500/1,500 results in the current ten-method matrix. Before correction, the reviewer raised 56 disagreements with machine judgments; 2 confirmed scoring errors whose root causes were fixed first, and the remainder were adjudicated against actual artifacts, reference answers, and task contracts. There are 0 unresolved items. The same-window cohort remains latency-only and is outside result review."], - ["Body, dynamic-task, and challenge-page scoring", "A report could keep reading stale scorer output after code changes; greedy body matching in reference order could let longer returned fragments consume the wrong region first; and serialized script state containing target strings could be mistaken for readable body text.", "Report builds reject stale scorer output; body fragments now use global capacity allocation independent of reference-region order; and script or application-state shells cannot satisfy body tasks.", "Both corrections have positive and negative regressions executed through the shared score_attempt and task-decision paths. Saved output is consistently rescored offline without provider-specific or row-specific exceptions."], - ["Exa named-link interface", "The old adapter read only the bare links array, which could not prove name-to-URL ownership and omitted official richLinks with explicit anchor–URL pairs.", "Only named-link tasks request extras.richLinks, and only explicit pairs with nonempty anchors and URLs are accepted. Body headings and bare links are never paired by position.", "The interface-contract change affects only Exa's 10 challenge tasks, so the certified rerun was limited to that scope. 3 pass; 7 still fail because the actual artifacts do not satisfy the contract."], - ["Complete DOCX content", "document-05 required only the title and section names, allowing narrative output that omitted table cells, calendar content, and the final notes to pass.", "The task contract now includes table, calendar, and note landmarks from the frozen source document. Markdown table separators are treated as layout so complete tables are not falsely rejected.", "Saved artifacts were consistently rescored offline: CowAgent's incomplete result now fails, while Firecrawl and Tavily retain passes for complete Markdown tables. No provider calls were needed."], - ["Image-resource scope", "Image-URL tasks were briefly proposed as target-only output, which could fail a valid page fetch that found the target while retaining logos or recommendation images. Some responsive target aliases also require independent source evidence.", "Resource acceptance requires a declared target or an evidenced valid alias. Additional page images remain a precision diagnostic and do not affect binary acceptance. Target lists, aliases, source observations, and resource contracts are all hash-bound.", "This avoids treating a page-level fetch as if it had to return only one URL, while still rejecting long text or many unrelated images that never match the target. Stale scores cannot be published after a contract change."], - ["Recurrence prevention and evidence revision", "Scores, references, datasets, and review receipts can change independently. Partial lifecycle validation could wrongly reuse an old review, while patching only a final verdict cannot prevent recurrence.", "Each root cause has executable controls in both directions. Reference and scoring revisions separately chain complete input and output hashes, root causes, and regression IDs; report builds validate scorer identity, task content, resource contracts, and source evidence.", "Scoring and reference revisions do not resend provider requests. Only predeclared scopes affected by an interface, version, input, or request-contract change are rerun, with traceability preserved between old and new calls."], - ["Firecrawl dynamic interface", "The old adapter used actions, which the current account region does not allow. All 10 dynamic tasks failed at the service boundary, incorrectly producing a 0/10 capability result.", "The adapter now uses waitFor, supported by the current public interface. A certified run with references frozen before and after calls reran only these 10 tasks; the other 140 Firecrawl results are unchanged.", "The new results pass 6/10. Codex CLI blindly reviewed all 10 additions with machine judgments hidden, finding 0 disagreements and 0 unresolved items. The remaining 4 fail because their actual output does not satisfy the task contract."], - ["Report reading and latency", "The cross-batch latency table for tasks passed by every method was expanded by default and occupied substantial space below the ranking; the overall ranking did not show latency over all 150 calls.", "Shared-success latency is now collapsed on demand. The overall ranking adds mean, P50, and P95 over all 150 calls per method, using the same fixed denominator as coverage.", "Latency columns do not affect coverage ordering or select a successful subset; failed-call elapsed time remains in the all-call distribution."], - ["Request status and empty output", "The old adapters conflated a successful provider call with no acceptable body and a provider or target request failure, mixing request-success and task-pass semantics.", "Cloudflare now normalizes saved success, result, and origin-status envelopes; Moli uses the 1.1.4 exit status and whole-response error envelope. An empty successful response remains request success but task failure, while an explicit target non-2xx remains request failure.", "All 14 status corrections were reconstructed offline from hash-bound raw evidence. Task verdicts and scores did not change, and no provider was called."], - ["Dynamic extraction and public artifacts", "The Crawl4AI dynamic-cart request declared bounded extraction, but the worker did not pass the configuration and lost card boundaries. OpenClaw error stacks and CowAgent save notices could also expose host-local paths in public evidence.", "The Crawl4AI worker now passes the declared extraction configuration and preserves card boundaries. OpenClaw and DeepSeek error surfaces retain only stable messages, while CowAgent removes only host-local save paths and keeps fetched text.", "The certified Crawl4AI rerun passes and raises its total to 119/150. Certified reruns covered 15 affected OpenClaw artifacts and 11 CowAgent artifacts without changing their task totals; public artifacts no longer contain real host paths."], - ["Reference-evidence consistency", "Four complete-body samples had source descriptions pointing to unrelated sites. The NOAA sample had a same-day independent 200 response, main-region extraction, and complete-body hash but retained a stale note saying the full text was unavailable.", "Complete-body evidence must use the task URL's site. NOAA metadata now explicitly records the complete reference as available; stable anchors still drive binary acceptance and the full snapshot supplies diagnostic metrics. A dataset guard rejects valid full text paired with an unavailable declaration.", "Source and note corrections do not change provider output or task scores; affected results were rebound and independently reviewed."], - ["Anomaly root-cause gate", "Comparing only a reference and final verdict cannot proactively find empty output, repeated cross-site errors, concentrated zero scores, interface misuse, reference conflicts, or public-artifact leakage. Caching by batch number also misaligns reviews after small revisions.", "After row-level blind review, deterministic anomaly signals receive a seven-layer investigation. Anomaly caches are reused by exact attempt scope rather than batch position, so only changed groups are reviewed again.", "The final matrix is reviewed 1,500/1,500; all 309 anomaly-triggered results completed investigation with 0 unresolved items. Cache scope, reference revisions, and final review are hash-bound."] + [ + "Full result review", + "The previous release independently reviewed machine failures only. Machine passes had not been checked individually, leaving systematic false positives from permissive rules undetectable.", + "Machine passes and failures use the same packets with machine judgments hidden. Prior reviews remain valid only when both the task and reviewer-visible evidence hash are unchanged; only changed results are reviewed again.", + "Review covers 1,500/1,500 results in the current ten-method matrix. Before correction, the reviewer raised 56 disagreements with machine judgments; 2 confirmed scoring errors whose root causes were fixed first, and the remainder were adjudicated against actual artifacts, reference answers, and task contracts. There are 0 unresolved items. The same-window cohort remains latency-only and is outside result review." + ], + [ + "Body, dynamic-task, and challenge-page scoring", + "A report could keep reading stale scorer output after code changes; greedy body matching in reference order could let longer returned fragments consume the wrong region first; and serialized script state containing target strings could be mistaken for readable body text.", + "Report builds reject stale scorer output; body fragments now use global capacity allocation independent of reference-region order; and script or application-state shells cannot satisfy body tasks.", + "Both corrections have positive and negative regressions executed through the shared score_attempt and task-decision paths. Saved output is consistently rescored offline without provider-specific or row-specific exceptions." + ], + [ + "Exa named-link interface", + "The old adapter read only the bare links array, which could not prove name-to-URL ownership and omitted official richLinks with explicit anchor–URL pairs.", + "Only named-link tasks request extras.richLinks, and only explicit pairs with nonempty anchors and URLs are accepted. Body headings and bare links are never paired by position.", + "The interface-contract change affects only Exa's 10 challenge tasks, so the certified rerun was limited to that scope. 3 pass; 7 still fail because the actual artifacts do not satisfy the contract." + ], + [ + "Complete DOCX content", + "document-05 required only the title and section names, allowing narrative output that omitted table cells, calendar content, and the final notes to pass.", + "The task contract now includes table, calendar, and note landmarks from the frozen source document. Markdown table separators are treated as layout so complete tables are not falsely rejected.", + "Saved artifacts were consistently rescored offline: CowAgent's incomplete result now fails, while Firecrawl and Tavily retain passes for complete Markdown tables. No provider calls were needed." + ], + [ + "Image-resource scope", + "Image-URL tasks were briefly proposed as target-only output, which could fail a valid page fetch that found the target while retaining logos or recommendation images. Some responsive target aliases also require independent source evidence.", + "Resource acceptance requires a declared target or an evidenced valid alias. Additional page images remain a precision diagnostic and do not affect binary acceptance. Target lists, aliases, source observations, and resource contracts are all hash-bound.", + "This avoids treating a page-level fetch as if it had to return only one URL, while still rejecting long text or many unrelated images that never match the target. Stale scores cannot be published after a contract change." + ], + [ + "Recurrence prevention and evidence revision", + "Scores, references, datasets, and review receipts can change independently. Partial lifecycle validation could wrongly reuse an old review, while patching only a final verdict cannot prevent recurrence.", + "Each root cause has executable controls in both directions. Reference and scoring revisions separately chain complete input and output hashes, root causes, and regression IDs; report builds validate scorer identity, task content, resource contracts, and source evidence.", + "Scoring and reference revisions do not resend provider requests. Only predeclared scopes affected by an interface, version, input, or request-contract change are rerun, with traceability preserved between old and new calls." + ], + [ + "Firecrawl dynamic interface", + "The old adapter used actions, which the current account region does not allow. All 10 dynamic tasks failed at the service boundary, incorrectly producing a 0/10 capability result.", + "The adapter now uses waitFor, supported by the current public interface. A certified run with references frozen before and after calls reran only these 10 tasks; the other 140 Firecrawl results are unchanged.", + "The new results pass 6/10. Codex CLI blindly reviewed all 10 additions with machine judgments hidden, finding 0 disagreements and 0 unresolved items. The remaining 4 fail because their actual output does not satisfy the task contract." + ], + [ + "Report reading and latency", + "The cross-batch latency table for tasks passed by every method was expanded by default and occupied substantial space below the ranking; the overall ranking did not show latency over all 150 calls.", + "Shared-success latency is now collapsed on demand. The overall ranking adds mean, P50, and P95 over all 150 calls per method, using the same fixed denominator as coverage.", + "Latency columns do not affect coverage ordering or select a successful subset; failed-call elapsed time remains in the all-call distribution." + ], + [ + "Request status and empty output", + "The old adapters conflated a successful provider call with no acceptable body and a provider or target request failure, mixing request-success and task-pass semantics.", + "Cloudflare now normalizes saved success, result, and origin-status envelopes; Moli uses the 1.1.4 exit status and whole-response error envelope. An empty successful response remains request success but task failure, while an explicit target non-2xx remains request failure.", + "All 14 status corrections were reconstructed offline from hash-bound raw evidence. Task verdicts and scores did not change, and no provider was called." + ], + [ + "Dynamic extraction and public artifacts", + "The Crawl4AI dynamic-cart request declared bounded extraction, but the worker did not pass the configuration and lost card boundaries. OpenClaw error stacks and CowAgent save notices could also expose host-local paths in public evidence.", + "The Crawl4AI worker now passes the declared extraction configuration and preserves card boundaries. OpenClaw and DeepSeek error surfaces retain only stable messages, while CowAgent removes only host-local save paths and keeps fetched text.", + "The certified Crawl4AI rerun passes and raises its total to 119/150. Certified reruns covered 15 affected OpenClaw artifacts and 11 CowAgent artifacts without changing their task totals; public artifacts no longer contain real host paths." + ], + [ + "Reference-evidence consistency", + "Four complete-body samples had source descriptions pointing to unrelated sites. The NOAA sample had a same-day independent 200 response, main-region extraction, and complete-body hash but retained a stale note saying the full text was unavailable.", + "Complete-body evidence must use the task URL's site. NOAA metadata now explicitly records the complete reference as available; stable anchors still drive binary acceptance and the full snapshot supplies diagnostic metrics. A dataset guard rejects valid full text paired with an unavailable declaration.", + "Source and note corrections do not change provider output or task scores; affected results were rebound and independently reviewed." + ], + [ + "Anomaly root-cause gate", + "Comparing only a reference and final verdict cannot proactively find empty output, repeated cross-site errors, concentrated zero scores, interface misuse, reference conflicts, or public-artifact leakage. Caching by batch number also misaligns reviews after small revisions.", + "After row-level blind review, deterministic anomaly signals receive a seven-layer investigation. Anomaly caches are reused by exact attempt scope rather than batch position, so only changed groups are reviewed again.", + "The final matrix is reviewed 1,500/1,500; all 309 anomaly-triggered results completed investigation with 0 unresolved items. Cache scope, reference revisions, and final review are hash-bound." + ] ] } } @@ -3145,7 +4390,9 @@ "id": "2026-09-10-twelve-method-refresh-contract", "publication_state": "published", "date": "2026-09-10", - "comparison_kinds": ["web_fetch_methods"], + "comparison_kinds": [ + "web_fetch_methods" + ], "scoring_revision_input_sha256": [ "f167652c1225f33829123a7ef9f75280a610d104e76ff4752f982c3ec3b89e0b", "0362d27e8bf8418c79a255e144fd146d7450699a753f4eb00b8f38963ab8b526", @@ -3154,29 +4401,108 @@ "title": "完成十二方法升级、影响范围重跑与失败项核验", "summary": "十二方法报告已重新绑定当前评测集、实现身份和独立发布目标。主矩阵按预声明影响范围新增 432 次调用，同窗测速新增 240 次；主矩阵 705 条机器失败结果完成独立核验，1,095 条机器通过结果未逐条审查，分歧逐条裁决，未决为 0。", "changes": [ - ["评测对象", "默认报告一度指向七种浏览器方法，旧十二方法页面又保留了已替换的易变正文样本，无法作为本轮十二方法权威矩阵。", "当前 profile 恢复十二种 Web Fetch 方法和 150 个任务；八个易变全文样本改为 RFC、固定 Git 提交、W3C 定版文档或 Wikipedia 固定修订，并保留明确的替换关系。", "最终权威矩阵仍为 1,800 个方法—任务逻辑结果；被替换题目及另外五个受参考或实现影响的题目须按当前合同重跑，旧结果不直接沿用。"], - ["实现身份与前置门禁", "旧证据没有完整固定 CowAgent、OpenClaw、DeepSeek Harness、LangChain 与 Crawl4AI 的源码或依赖身份，空路径或错误版本还可能在证据目录创建后才失败。", "源码型方法固定仓库修订、Node 版本和必需文件；LangChain WebBaseLoader 0.3.30、Crawl4AI 0.9.3 及文档解析依赖进入 uv.lock。所有方法在创建运行证据前完成身份、命令和路径校验，身份摘要写入 manifest。", "LangChain 与 Crawl4AI 因旧权威证据未记录库版本，各自完整重跑 150 题；源码型方法只有精确且干净的绑定实现进入本轮。"], - ["直接文档路线", "通用 Moli 适配器把直接 PDF 或 DOCX 正文任务当作普通网页 Markdown，可能在取得原始文档前失败或把二进制响应序列化。", "直接 PDF、DOCX 与文件任务预先选择 Moli 自动原始响应；正文任务在同一个 45 秒预算内解析文字并保留原始字节哈希，文件任务继续按字节验收，不增加跨方法下载后备。", "受路线影响的文档结果已进入声明的重跑范围；评分与盲审均使用新保存输出。"], - ["偶发下载证据", "正文抓取若同时触发浏览器下载，旧 runner 因二进制没有文件任务验收记录而中止整批，导致后续结果缺失。", "调用边界现在会保存并哈希所有返回字节；只有文件任务可用这些字节通过验收，正文任务仍必须交付合格文字。触发故障的 Crawl4AI 批次已整批作废并重跑。", "修复只保证证据完整和运行可继续，不把偶发下载当成正文能力，也不复用崩溃前的部分批次。"], - ["影响范围与测速", "仅补跑旧失败或只重建页面会混入未知实现版本，也无法公平比较来自不同批次的十二方法耗时。", "主矩阵预先声明 13 题 × 12 方法、LangChain 与 Crawl4AI 的其余完整方法 cohort，以及两个超时边界修正，共完成 432 次新调用；另以 provider-blind、域名去重的 20 题完成 240 次十二方法同窗测速。", "主矩阵固定每个方法—任务一个权威逻辑结果，不按最好或最快挑选。测速保留完整预声明矩阵：全调用延迟按同一 20 题分母统计，成功调用延迟另作条件统计；十二方法共同通过为 1/20，只作为诊断，不据此筛样本或排名。"], - ["证据与失败项核验", "旧组装器可能把 worker 工作树和符号链接带入汇总，又会漏掉重试物理调用返回的辅助文件；审查包若遗漏原生答案字段，也会制造系统性假分歧。", "汇总只复制哈希绑定的公开文件，拒绝符号链接和工作树路径，并绑定主返回、辅助文件与每次物理重试。Codex gpt-5.6-sol 对主矩阵 705 条机器失败结果盲审，审查包同时提供白名单化的原生答案 metadata；2 条分歧逐条裁决。", "最终证据为 12 × 150 的唯一主矩阵和 12 × 20 的唯一测速矩阵；主矩阵失败项核验 705/705，机器通过未逐条审查 1,095 条，测速结果不在本次审查范围，未决 0。裁决只确定审计结论，不回写或择优替换 provider 输出。"], - ["报告与发布隔离", "旧生成器混用了 150 题、六类和七浏览器专属叙述，方法页重复 Exa、Tavily、Firecrawl 配置，通用报告还可能显示 headless 修订史、复现链接和中文延迟加载提示。", "同一 renderer 改由当前 dataset、records 和实际收据生成数量、方法路线、修订说明与链接；十二方法与 headless 历史按 comparison kind 隔离，中英文虚拟路由分别显示对应加载及无障碍文案。当前十二方法构建绑定新的 dev 域站点，旧十二方法站点与 app 域浏览器参考报告保持不变。", "发布器逐文件核验公开文件；服务端允许的 HTML 外壳注入与匿名字节一致性分层披露，不再把两者混称为全站字节相同。"], - ["环境导入", "托管方法凭据已存在于用户级 ~/.env，但旧流程只读取仓库 .env，导致新会话会误报缺少变量。", "新增显式 import-env：仅将九个允许变量从用户级文件补入仓库 .env，保留已有非空值、隐藏变量内容并强制 0600 权限；文档和测试固定该入口。", "凭据值仍不进入 Git、证据、日志或报告；导入后六个托管方法的前置校验通过。"] - ] - , + [ + "评测对象", + "默认报告一度指向七种浏览器方法，旧十二方法页面又保留了已替换的易变正文样本，无法作为本轮十二方法权威矩阵。", + "当前 profile 恢复十二种 Web Fetch 方法和 150 个任务；八个易变全文样本改为 RFC、固定 Git 提交、W3C 定版文档或 Wikipedia 固定修订，并保留明确的替换关系。", + "最终权威矩阵仍为 1,800 个方法—任务逻辑结果；被替换题目及另外五个受参考或实现影响的题目须按当前合同重跑，旧结果不直接沿用。" + ], + [ + "实现身份与前置门禁", + "旧证据没有完整固定 CowAgent、OpenClaw、DeepSeek Harness、LangChain 与 Crawl4AI 的源码或依赖身份，空路径或错误版本还可能在证据目录创建后才失败。", + "源码型方法固定仓库修订、Node 版本和必需文件；LangChain WebBaseLoader 0.3.30、Crawl4AI 0.9.3 及文档解析依赖进入 uv.lock。所有方法在创建运行证据前完成身份、命令和路径校验，身份摘要写入 manifest。", + "LangChain 与 Crawl4AI 因旧权威证据未记录库版本，各自完整重跑 150 题；源码型方法只有精确且干净的绑定实现进入本轮。" + ], + [ + "直接文档路线", + "通用 Moli 适配器把直接 PDF 或 DOCX 正文任务当作普通网页 Markdown，可能在取得原始文档前失败或把二进制响应序列化。", + "直接 PDF、DOCX 与文件任务预先选择 Moli 自动原始响应；正文任务在同一个 45 秒预算内解析文字并保留原始字节哈希，文件任务继续按字节验收，不增加跨方法下载后备。", + "受路线影响的文档结果已进入声明的重跑范围；评分与盲审均使用新保存输出。" + ], + [ + "偶发下载证据", + "正文抓取若同时触发浏览器下载，旧 runner 因二进制没有文件任务验收记录而中止整批，导致后续结果缺失。", + "调用边界现在会保存并哈希所有返回字节；只有文件任务可用这些字节通过验收，正文任务仍必须交付合格文字。触发故障的 Crawl4AI 批次已整批作废并重跑。", + "修复只保证证据完整和运行可继续，不把偶发下载当成正文能力，也不复用崩溃前的部分批次。" + ], + [ + "影响范围与测速", + "仅补跑旧失败或只重建页面会混入未知实现版本，也无法公平比较来自不同批次的十二方法耗时。", + "主矩阵预先声明 13 题 × 12 方法、LangChain 与 Crawl4AI 的其余完整方法 cohort，以及两个超时边界修正，共完成 432 次新调用；另以 provider-blind、域名去重的 20 题完成 240 次十二方法同窗测速。", + "主矩阵固定每个方法—任务一个权威逻辑结果，不按最好或最快挑选。测速保留完整预声明矩阵：全调用延迟按同一 20 题分母统计，成功调用延迟另作条件统计；十二方法共同通过为 1/20，只作为诊断，不据此筛样本或排名。" + ], + [ + "证据与失败项核验", + "旧组装器可能把 worker 工作树和符号链接带入汇总，又会漏掉重试物理调用返回的辅助文件；审查包若遗漏原生答案字段，也会制造系统性假分歧。", + "汇总只复制哈希绑定的公开文件，拒绝符号链接和工作树路径，并绑定主返回、辅助文件与每次物理重试。Codex gpt-5.6-sol 对主矩阵 705 条机器失败结果盲审，审查包同时提供白名单化的原生答案 metadata；2 条分歧逐条裁决。", + "最终证据为 12 × 150 的唯一主矩阵和 12 × 20 的唯一测速矩阵；主矩阵失败项核验 705/705，机器通过未逐条审查 1,095 条，测速结果不在本次审查范围，未决 0。裁决只确定审计结论，不回写或择优替换 provider 输出。" + ], + [ + "报告与发布隔离", + "旧生成器混用了 150 题、六类和七浏览器专属叙述，方法页重复 Exa、Tavily、Firecrawl 配置，通用报告还可能显示 headless 修订史、复现链接和中文延迟加载提示。", + "同一 renderer 改由当前 dataset、records 和实际收据生成数量、方法路线、修订说明与链接；十二方法与 headless 历史按 comparison kind 隔离，中英文虚拟路由分别显示对应加载及无障碍文案。当前十二方法构建绑定新的 dev 域站点，旧十二方法站点与 app 域浏览器参考报告保持不变。", + "发布器逐文件核验公开文件；服务端允许的 HTML 外壳注入与匿名字节一致性分层披露，不再把两者混称为全站字节相同。" + ], + [ + "环境导入", + "托管方法凭据已存在于用户级 ~/.env，但旧流程只读取仓库 .env，导致新会话会误报缺少变量。", + "新增显式 import-env：仅将九个允许变量从用户级文件补入仓库 .env，保留已有非空值、隐藏变量内容并强制 0600 权限；文档和测试固定该入口。", + "凭据值仍不进入 Git、证据、日志或报告；导入后六个托管方法的前置校验通过。" + ] + ], "i18n": { "en": { "title": "Complete the twelve-method upgrade, impact-scoped reruns, and failure review", "summary": "The twelve-method report is now bound to the current evaluation set, implementation identities, and an independent publication target. The primary matrix adds 432 calls within the predeclared impact scope and the same-window latency cohort adds 240 calls. All 705 machine failures in the primary matrix received independent review; the 1,095 machine passes were not reviewed individually. Every disagreement was adjudicated, with 0 unresolved.", "changes": [ - ["Evaluation scope", "The default report had temporarily pointed to the seven browser methods, while the old twelve-method site still contained volatile body-text tasks that have since been replaced. Neither could serve as the authoritative twelve-method matrix for this run.", "The current profile restores twelve Web Fetch methods and 150 tasks. Eight volatile full-text tasks are replaced with RFCs, pinned Git commits, final W3C publications, or a fixed Wikipedia revision, with each replacement recorded explicitly.", "The final authoritative matrix remains 1,800 logical method–task results. The replaced tasks and five additional tasks affected by reference or implementation changes must be rerun under the current contract; their old results are not carried forward directly."], - ["Implementation identity and preflight gates", "The old evidence did not fully pin the source or dependency identities for CowAgent, OpenClaw, DeepSeek Harness, LangChain, and Crawl4AI. Empty paths or incorrect versions could also fail only after an evidence directory had been created.", "Source-backed methods now pin repository revisions, Node versions, and required files. LangChain WebBaseLoader 0.3.30, Crawl4AI 0.9.3, and document-parsing dependencies are locked in uv.lock. Every method validates its identity, command, and paths before run evidence is created, and an identity summary is written to the manifest.", "LangChain and Crawl4AI each reran all 150 tasks because the previous authoritative evidence did not record their library versions. For source-based methods, only exact, clean, bound implementations entered this run."], - ["Direct-document path", "The general Moli adapter treated direct PDF and DOCX body-text tasks as ordinary web Markdown, which could fail before retrieving the original document or serialize a binary response.", "Direct PDF, DOCX, and file tasks now preselect Moli's automatic raw response. Body-text tasks parse text within the same 45-second budget and retain the raw-byte hash; file tasks continue to be accepted by byte validation, with no cross-method download fallback.", "Document results affected by the route change were included in the declared rerun scope; scoring and blind review both use the newly saved outputs."], - ["Incidental-download evidence", "When a body fetch also triggered a browser download, the old runner aborted the whole batch because the binary had no file-task evaluation record, leaving later results missing.", "The call boundary now persists and hashes every returned binary. Only file tasks may pass on those bytes; body tasks must still deliver acceptable text. The affected Crawl4AI batch was invalidated in full and rerun.", "This fix preserves evidence and lets execution continue; it neither treats an incidental download as body capability nor reuses the partial pre-crash batch."], - ["Impact scope and latency cohort", "Rerunning only old failures or merely rebuilding the pages would mix unknown implementation versions and would not support a fair latency comparison across twelve methods measured in different batches.", "The primary matrix predeclared 13 tasks × 12 methods, the remaining full-method cohorts for LangChain and Crawl4AI, and two deadline-boundary corrections, for 432 new calls in total. A separate provider-blind, domain-deduplicated set of 20 tasks produced 240 same-window latency calls across all twelve methods.", "The primary matrix fixes one authoritative logical result for every method–task pair and never selects the best or fastest result. The latency cohort retains its complete predeclared matrix: all-call latency uses the same 20-task denominator, while successful-call latency is reported separately as a conditional statistic. All twelve methods jointly passed 1/20 tasks; this is diagnostic only and is not used to select samples or rank methods."], - ["Evidence and failure review", "The old assembler could pull worker trees and symbolic links into the aggregate while omitting auxiliary files returned by physical retry calls. A review packet that omitted native answer fields could also create systematic false disagreements.", "Aggregation now copies only hash-bound public files, rejects symbolic links and worker-tree paths, and binds the primary response, auxiliary files, and every physical retry. Codex gpt-5.6-sol blindly reviewed the 705 machine failures in the primary matrix, with allowlisted native answer metadata included in the review packet; the 2 disagreements were adjudicated individually.", "The final evidence consists of one 12 × 150 primary matrix and one 12 × 20 latency matrix. Failure review covers 705/705 primary-matrix failures; 1,095 machine passes were not reviewed individually, and latency results were outside this review scope. There are 0 unresolved items. Adjudication determines only the audit conclusion and never rewrites or selectively replaces provider output."], - ["Report and publication isolation", "The old generator mixed fixed 150-task and six-family copy with seven-browser-specific explanations. Method pages repeated Exa, Tavily, and Firecrawl configuration, while the general report could also show headless changelog entries, reproduction links, and Chinese lazy-loading messages.", "The shared renderer now derives counts, method paths, revision explanations, and links from the current dataset, records, and available receipts. Twelve-method and headless histories are separated by comparison kind, and Chinese and English virtual routes show localized loading and accessibility copy. The current twelve-method build is bound to a new dev-domain site, while the old twelve-method site and the app-domain browser reference report remain unchanged.", "The publisher verifies public files individually. Server-permitted HTML shell injection and anonymous byte equality are disclosed as separate layers rather than being conflated as site-wide byte identity."], - ["Environment import", "Credentials for hosted methods already existed in the user-level ~/.env, but the old workflow read only the repository .env, causing new sessions to report variables as missing incorrectly.", "An explicit import-env command now copies only nine allowlisted variables from the user-level file into the repository .env, preserves existing non-empty values, hides variable contents, and enforces 0600 permissions. Documentation and tests make this the stable entry point.", "Credential values still never enter Git, evidence, logs, or the report. Preflight validation passes for all six hosted methods after import."] + [ + "Evaluation scope", + "The default report had temporarily pointed to the seven browser methods, while the old twelve-method site still contained volatile body-text tasks that have since been replaced. Neither could serve as the authoritative twelve-method matrix for this run.", + "The current profile restores twelve Web Fetch methods and 150 tasks. Eight volatile full-text tasks are replaced with RFCs, pinned Git commits, final W3C publications, or a fixed Wikipedia revision, with each replacement recorded explicitly.", + "The final authoritative matrix remains 1,800 logical method–task results. The replaced tasks and five additional tasks affected by reference or implementation changes must be rerun under the current contract; their old results are not carried forward directly." + ], + [ + "Implementation identity and preflight gates", + "The old evidence did not fully pin the source or dependency identities for CowAgent, OpenClaw, DeepSeek Harness, LangChain, and Crawl4AI. Empty paths or incorrect versions could also fail only after an evidence directory had been created.", + "Source-backed methods now pin repository revisions, Node versions, and required files. LangChain WebBaseLoader 0.3.30, Crawl4AI 0.9.3, and document-parsing dependencies are locked in uv.lock. Every method validates its identity, command, and paths before run evidence is created, and an identity summary is written to the manifest.", + "LangChain and Crawl4AI each reran all 150 tasks because the previous authoritative evidence did not record their library versions. For source-based methods, only exact, clean, bound implementations entered this run." + ], + [ + "Direct-document path", + "The general Moli adapter treated direct PDF and DOCX body-text tasks as ordinary web Markdown, which could fail before retrieving the original document or serialize a binary response.", + "Direct PDF, DOCX, and file tasks now preselect Moli's automatic raw response. Body-text tasks parse text within the same 45-second budget and retain the raw-byte hash; file tasks continue to be accepted by byte validation, with no cross-method download fallback.", + "Document results affected by the route change were included in the declared rerun scope; scoring and blind review both use the newly saved outputs." + ], + [ + "Incidental-download evidence", + "When a body fetch also triggered a browser download, the old runner aborted the whole batch because the binary had no file-task evaluation record, leaving later results missing.", + "The call boundary now persists and hashes every returned binary. Only file tasks may pass on those bytes; body tasks must still deliver acceptable text. The affected Crawl4AI batch was invalidated in full and rerun.", + "This fix preserves evidence and lets execution continue; it neither treats an incidental download as body capability nor reuses the partial pre-crash batch." + ], + [ + "Impact scope and latency cohort", + "Rerunning only old failures or merely rebuilding the pages would mix unknown implementation versions and would not support a fair latency comparison across twelve methods measured in different batches.", + "The primary matrix predeclared 13 tasks × 12 methods, the remaining full-method cohorts for LangChain and Crawl4AI, and two deadline-boundary corrections, for 432 new calls in total. A separate provider-blind, domain-deduplicated set of 20 tasks produced 240 same-window latency calls across all twelve methods.", + "The primary matrix fixes one authoritative logical result for every method–task pair and never selects the best or fastest result. The latency cohort retains its complete predeclared matrix: all-call latency uses the same 20-task denominator, while successful-call latency is reported separately as a conditional statistic. All twelve methods jointly passed 1/20 tasks; this is diagnostic only and is not used to select samples or rank methods." + ], + [ + "Evidence and failure review", + "The old assembler could pull worker trees and symbolic links into the aggregate while omitting auxiliary files returned by physical retry calls. A review packet that omitted native answer fields could also create systematic false disagreements.", + "Aggregation now copies only hash-bound public files, rejects symbolic links and worker-tree paths, and binds the primary response, auxiliary files, and every physical retry. Codex gpt-5.6-sol blindly reviewed the 705 machine failures in the primary matrix, with allowlisted native answer metadata included in the review packet; the 2 disagreements were adjudicated individually.", + "The final evidence consists of one 12 × 150 primary matrix and one 12 × 20 latency matrix. Failure review covers 705/705 primary-matrix failures; 1,095 machine passes were not reviewed individually, and latency results were outside this review scope. There are 0 unresolved items. Adjudication determines only the audit conclusion and never rewrites or selectively replaces provider output." + ], + [ + "Report and publication isolation", + "The old generator mixed fixed 150-task and six-family copy with seven-browser-specific explanations. Method pages repeated Exa, Tavily, and Firecrawl configuration, while the general report could also show headless changelog entries, reproduction links, and Chinese lazy-loading messages.", + "The shared renderer now derives counts, method paths, revision explanations, and links from the current dataset, records, and available receipts. Twelve-method and headless histories are separated by comparison kind, and Chinese and English virtual routes show localized loading and accessibility copy. The current twelve-method build is bound to a new dev-domain site, while the old twelve-method site and the app-domain browser reference report remain unchanged.", + "The publisher verifies public files individually. Server-permitted HTML shell injection and anonymous byte equality are disclosed as separate layers rather than being conflated as site-wide byte identity." + ], + [ + "Environment import", + "Credentials for hosted methods already existed in the user-level ~/.env, but the old workflow read only the repository .env, causing new sessions to report variables as missing incorrectly.", + "An explicit import-env command now copies only nine allowlisted variables from the user-level file into the repository .env, preserves existing non-empty values, hides variable contents, and enforces 0600 permissions. Documentation and tests make this the stable entry point.", + "Credential values still never enter Git, evidence, logs, or the report. Preflight validation passes for all six hosted methods after import." + ] ] } } @@ -3188,17 +4514,36 @@ "title": "解读突出耗时减少比例并增加三张信息图", "summary": "首屏以 Chromium 为基准展示均值、P50、P95 的耗时减少比例；以结果总览和两张流程图替代重复总结与部分流程文字，标题改为直接的内容名称。", "changes": [ - ["耗时比较", "首屏强调秒数，读者需要自行计算差距。", "用未舍入数据计算相对 Chromium 的耗时减少比例：均值 42.8%、P50 46.7%、P95 44.3%，秒数作为次要依据保留。", "仍为跨批次的 98 道共同成功任务；统计样本与测量边界不变。"], - ["信息图", "概念流程与总结主要依靠文字。", "增加结果总览、Web Fetch 与 Browser Use 对照、五种交付物流程图；提供完整图片下载，英文版使用对应英文总览。", "流程图为说明性示意，六组案例仍展示原始证据。图片逐项核对并绑定哈希，不参与评分。"] - ] - , + [ + "耗时比较", + "首屏强调秒数，读者需要自行计算差距。", + "用未舍入数据计算相对 Chromium 的耗时减少比例：均值 42.8%、P50 46.7%、P95 44.3%，秒数作为次要依据保留。", + "仍为跨批次的 98 道共同成功任务；统计样本与测量边界不变。" + ], + [ + "信息图", + "概念流程与总结主要依靠文字。", + "增加结果总览、Web Fetch 与 Browser Use 对照、五种交付物流程图；提供完整图片下载，英文版使用对应英文总览。", + "流程图为说明性示意，六组案例仍展示原始证据。图片逐项核对并绑定哈希，不参与评分。" + ] + ], "i18n": { "en": { "title": "Emphasize latency reduction and add three explanatory figures", "summary": "The opening compares mean, P50, and P95 latency reductions against Chromium. A result overview and two flow diagrams replace repeated summaries and some process prose, with direct descriptive titles.", "changes": [ - ["Latency comparison", "The opening emphasized seconds and required readers to calculate the differences themselves.", "Use unrounded values to calculate reductions relative to Chromium: 42.8% for mean latency, 46.7% for P50, and 44.3% for P95, while retaining seconds as secondary context.", "The population remains 98 shared successful tasks collected across batches; the sample and measurement boundary are unchanged."], - ["Explanatory figures", "Conceptual flows and summaries relied mostly on prose.", "Add a result overview, a Web Fetch versus Browser Use comparison, and a flow diagram for five deliverable types. Full-size images are downloadable, with a corresponding English overview.", "The diagrams are explanatory. Six case groups still show original evidence; images are reviewed and hash-bound but do not affect scoring."] + [ + "Latency comparison", + "The opening emphasized seconds and required readers to calculate the differences themselves.", + "Use unrounded values to calculate reductions relative to Chromium: 42.8% for mean latency, 46.7% for P50, and 44.3% for P95, while retaining seconds as secondary context.", + "The population remains 98 shared successful tasks collected across batches; the sample and measurement boundary are unchanged." + ], + [ + "Explanatory figures", + "Conceptual flows and summaries relied mostly on prose.", + "Add a result overview, a Web Fetch versus Browser Use comparison, and a flow diagram for five deliverable types. Full-size images are downloadable, with a corresponding English overview.", + "The diagrams are explanatory. Six case groups still show original evidence; images are reviewed and hash-bound but do not affect scoring." + ] ] } } @@ -3210,19 +4555,48 @@ "title": "解读开篇增加综合通过率与六类结果", "summary": "五浏览器解读先展示综合通过率与六类任务成绩，正文解释差异，结尾归纳结论。浏览器配置改为对照表，章节统一编号。", "changes": [ - ["结果展示", "总通过率与分组结果分散在正文中。", "开篇集中展示五款浏览器的总通过率与六类通过数，随后解释文档、媒体和网页的差异。", "沿用同一组 150 道任务和 Moli 1.1.1，评分与通过数不变。"], - ["耗时展示", "解读同时展开两组不同条件的耗时比较，阅读主线分散。", "解读集中展示全部 98 道共同成功主任务的均值、P50、P95，并保留跨批次条件。", "原报告的同窗测速章节与证据继续保留，未删除或改写调用记录。"], - ["阅读结构", "章节眉题、副标题和浏览器长段落增加阅读负担。", "统一编号标题和字号层级，以对照表介绍五款浏览器，增加量化总结与有序后续计划。", "保留 Web Fetch、Browser Use 背景、五类流程和六组真实案例。"] - ] - , + [ + "结果展示", + "总通过率与分组结果分散在正文中。", + "开篇集中展示五款浏览器的总通过率与六类通过数，随后解释文档、媒体和网页的差异。", + "沿用同一组 150 道任务和 Moli 1.1.1，评分与通过数不变。" + ], + [ + "耗时展示", + "解读同时展开两组不同条件的耗时比较，阅读主线分散。", + "解读集中展示全部 98 道共同成功主任务的均值、P50、P95，并保留跨批次条件。", + "原报告的同窗测速章节与证据继续保留，未删除或改写调用记录。" + ], + [ + "阅读结构", + "章节眉题、副标题和浏览器长段落增加阅读负担。", + "统一编号标题和字号层级，以对照表介绍五款浏览器，增加量化总结与有序后续计划。", + "保留 Web Fetch、Browser Use 背景、五类流程和六组真实案例。" + ] + ], "i18n": { "en": { "title": "Interpretation opens with overall pass rates and six task groups", "summary": "The five-browser article opens with overall pass rates and results for six task groups, explains the differences in the body, and closes with findings. Browser configurations use a comparison table, with consistently numbered sections.", "changes": [ - ["Result presentation", "Overall pass rates and group results were spread throughout the article.", "The opening brings together overall pass rates and six group pass counts for the five browsers, followed by explanations of document, media, and web differences.", "The same 150 tasks and Moli 1.1.1 are retained; scores and pass counts are unchanged."], - ["Latency presentation", "The article expanded two latency comparisons under different conditions, dividing the reading flow.", "The article focuses on mean latency, P50, and P95 for all 98 shared successful main tasks, retaining the collection-across-batches condition.", "The original report retains its same-window latency section and evidence; call records are neither deleted nor rewritten."], - ["Reading structure", "Section eyebrows, subtitles, and long browser paragraphs added reading overhead.", "Use consistently numbered headings and type sizes, introduce the five browsers in a comparison table, and add quantified findings and an ordered follow-up plan.", "Retain Web Fetch and Browser Use background, five workflow types, and real examples from six groups."] + [ + "Result presentation", + "Overall pass rates and group results were spread throughout the article.", + "The opening brings together overall pass rates and six group pass counts for the five browsers, followed by explanations of document, media, and web differences.", + "The same 150 tasks and Moli 1.1.1 are retained; scores and pass counts are unchanged." + ], + [ + "Latency presentation", + "The article expanded two latency comparisons under different conditions, dividing the reading flow.", + "The article focuses on mean latency, P50, and P95 for all 98 shared successful main tasks, retaining the collection-across-batches condition.", + "The original report retains its same-window latency section and evidence; call records are neither deleted nor rewritten." + ], + [ + "Reading structure", + "Section eyebrows, subtitles, and long browser paragraphs added reading overhead.", + "Use consistently numbered headings and type sizes, introduce the five browsers in a comparison table, and add quantified findings and an ordered follow-up plan.", + "Retain Web Fetch and Browser Use background, five workflow types, and real examples from six groups." + ] ] } } @@ -3234,7 +4608,12 @@ "title": "增加五浏览器 Web Fetch 图文解读", "summary": "文章对比 Moli 1.1.1 与另外四款浏览器，明确 150 题中通过 135 题，完整展示均值、P50、P95，并用六个评测集的可切换案例并列标准答案与真实输出。原始调用、评分及榜单分母不变。", "changes": [ - ["加载恢复", "短暂网络失败后，失败缓存使页面内重试无法重新请求。", "清除失败缓存，为页面和证据内容提供重新加载入口。", "仅修复报告读取行为，原始输出与评分不变。"], + [ + "加载恢复", + "短暂网络失败后，失败缓存使页面内重试无法重新请求。", + "清除失败缓存，为页面和证据内容提供重新加载入口。", + "仅修复报告读取行为，原始输出与评分不变。" + ], [ "文章范围", "多版本结果与案例细节分散，难以独立理解 Moli 1.1.1 的表现。", @@ -3259,18 +4638,42 @@ "普通网页、动态网页、热门站点、文档、图片、视频各取一个案例，逐项核对独立参考与真实返回；保留原始 PNG，关键数字直接排版。", "明确结构检查不核验商品名称事实、文件长度门槛为 32 字节，以及正文规则经过统一离线修订。" ] - ] - , + ], "i18n": { "en": { "title": "Add an illustrated five-browser Web Fetch interpretation", "summary": "The article compares Moli 1.1.1 with four other browsers, states that it passed 135 of 150 tasks, presents mean latency, P50, and P95, and provides selectable examples from six dataset groups pairing references with actual outputs. Original calls, scores, and ranking denominators are unchanged.", "changes": [ - ["Loading recovery", "After a temporary network failure, cached failures prevented in-page retries from making a new request.", "Clear cached failures and provide reload controls for pages and evidence content.", "Only report loading behavior changes; original outputs and scores are unchanged."], - ["Article scope", "Results across versions and scattered case details made it difficult to understand Moli 1.1.1's performance independently.", "Generate five-browser charts, a 120/30-task breakdown, a PDF path, and media examples from the same evidence; disclose that Moli 1.1.1 was selected after seeing version results.", "The article presents 750 results from five methods; all 1,050 records across seven paths remain available, without task-by-task version selection."], - ["Basis for conclusions", "The total pass count does not explain which input types contribute to the difference.", "Show that, across 120 web tasks, Moli and Chromium each complete 111; across 30 document and media tasks, they complete 24 and 14 respectively.", "PDF results include the local parser; file checks do not assess media semantics. Latency for the 98 shared successful main tasks spans batches. The five-browser same-window comparison uses all 9 shared successes, without changing the original seven-configuration latency results."], - ["Reading format", "Report tables were not convenient for long-form reading.", "The integrated article and standalone HTML share body text, inline graphics, and styles, with mobile and print layouts.", "Presentation and explanation changes only: no new provider calls and no score or latency changes."], - ["Acceptance explained", "The five task types previously had only conceptual flows, making the reasons for passing difficult to check.", "One example each from general pages, dynamic pages, popular sites, documents, images, and videos compares independent references with actual outputs check by check. The original PNG is retained and key figures are rendered directly as text.", "Clarify that structural checks do not verify product-name facts, the file-length threshold is 32 bytes, and body rules received shared offline revisions."] + [ + "Loading recovery", + "After a temporary network failure, cached failures prevented in-page retries from making a new request.", + "Clear cached failures and provide reload controls for pages and evidence content.", + "Only report loading behavior changes; original outputs and scores are unchanged." + ], + [ + "Article scope", + "Results across versions and scattered case details made it difficult to understand Moli 1.1.1's performance independently.", + "Generate five-browser charts, a 120/30-task breakdown, a PDF path, and media examples from the same evidence; disclose that Moli 1.1.1 was selected after seeing version results.", + "The article presents 750 results from five methods; all 1,050 records across seven paths remain available, without task-by-task version selection." + ], + [ + "Basis for conclusions", + "The total pass count does not explain which input types contribute to the difference.", + "Show that, across 120 web tasks, Moli and Chromium each complete 111; across 30 document and media tasks, they complete 24 and 14 respectively.", + "PDF results include the local parser; file checks do not assess media semantics. Latency for the 98 shared successful main tasks spans batches. The five-browser same-window comparison uses all 9 shared successes, without changing the original seven-configuration latency results." + ], + [ + "Reading format", + "Report tables were not convenient for long-form reading.", + "The integrated article and standalone HTML share body text, inline graphics, and styles, with mobile and print layouts.", + "Presentation and explanation changes only: no new provider calls and no score or latency changes." + ], + [ + "Acceptance explained", + "The five task types previously had only conceptual flows, making the reasons for passing difficult to check.", + "One example each from general pages, dynamic pages, popular sites, documents, images, and videos compares independent references with actual outputs check by check. The original PNG is retained and key figures are rendered directly as text.", + "Clarify that structural checks do not verify product-name facts, the file-length threshold is 32 bytes, and body rules received shared offline revisions." + ] ] } } @@ -3318,19 +4721,48 @@ "对同一四题、三个版本做三轮轮换，统一采用第三轮的 12 个耗时并重算均值、P50、P95；不按结果择优。", "全部通过任务中，1.1.3 的均值由 4,000 降至 3,706 ms、P50 由 3,444 降至 3,340 ms、P95 由 9,833 降至 7,983 ms；1.0.6 与 1.1.1 也按同一规则更新。共保留 36 次物理调用；正文、评分和任务判定未改变。" ] - ] - , + ], "i18n": { "en": { "title": "Document on-demand publishing, complete setup guidance, and strengthen the public-release security gate", "summary": "The report now distinguishes logical routes from deployed physical files and adds route-aligned bilingual access, setup instructions for all seven methods, and a stronger public-release security gate. It also corrects 12 anomalous latency values across four tasks and three Moli versions using one fixed final round. Task outputs, verdicts, and scores are unchanged; latency aggregates were recomputed.", "changes": [ - ["Publishing architecture", "Calling 1,218 auditable routes deployed HTML files suggested duplicated deployment and eager loading.", "Clarified that the local build produces logical routes, while the Sites bundle uses one HTML shell, a shared runtime, and content-addressed compressed blocks; pages and large evidence load on demand.", "The publication manifest records logical routes, source files, compressed blocks, and per-file hashes. Evidence is not reduced and download bytes do not change."], - ["Access & setup", "The old page showed only Cloudflare, the sole provider account requirement, making acquisition and binary configuration for local methods appear to be missing.", "Changed the page to an access and installation checklist: Cloudflare lists account and credential variables, while the six local methods list official acquisition sources and binary variables.", "Publish only variable names and official sources; never retain values, local paths, cookies, or personal sessions."], - ["Review attribution", "The ranking said only that results were reviewed, without identifying the reviewer.", "Each method's review column now shows 100/100 · reviewed by Codex and continues to link to the shared review record.", "Presentation copy only; review bindings, coverage, and unresolved counts are unchanged."], - ["Chinese and English editions", "The former English entry was a one-page summary whose routes, sections, and per-attempt evidence explanations did not align with the Chinese report.", "Generate route-aligned Chinese and English sites from the same report model. Navigation follows dataset, methodology, analysis, review, interpretation, methods, reproduction, changelog, and setup; switching language stays on the current route.", "Validate one-to-one parity for routes, sections, tables, values, filters, and evidence links. UI and explanations are rewritten for natural English, while task names, IDs, URLs, reference answers, and raw outputs remain unchanged and are not rescored."], - ["Public-release security gate", "The previous release scan covered common Google, Artifact Hub, GitHub, and OpenAI key patterns, but did not explicitly block Authorization values, local user-directory paths, or AWS and Slack credentials.", "Every rebuilt public file is now scanned uniformly, adding checks for Authorization values, user-directory paths, AWS and Slack credentials, and private-key patterns. Existing redaction of Google-shaped secrets in third-party HTML retains both hashes.", "Pattern scanning cannot prove that every unknown secret is absent. Structured credential values remain prohibited in evidence and reports, and published files still require byte-level verification."], - ["Moli latency correction", "A few anomalous legacy latency values made 1.1.3 appear to have an unproven overall performance regression.", "Run three order-rotated rounds for the same four tasks and three versions, use the 12 third-round latency values uniformly, and recompute mean, P50, and P95 without selecting favorable results.", "Across passing tasks, the 1.1.3 mean fell from 4,000 to 3,706 ms, P50 from 3,444 to 3,340 ms, and P95 from 9,833 to 7,983 ms. Versions 1.0.6 and 1.1.1 were updated under the same rule. All 36 physical calls are retained; body text, scores, and task verdicts are unchanged."] + [ + "Publishing architecture", + "Calling 1,218 auditable routes deployed HTML files suggested duplicated deployment and eager loading.", + "Clarified that the local build produces logical routes, while the Sites bundle uses one HTML shell, a shared runtime, and content-addressed compressed blocks; pages and large evidence load on demand.", + "The publication manifest records logical routes, source files, compressed blocks, and per-file hashes. Evidence is not reduced and download bytes do not change." + ], + [ + "Access & setup", + "The old page showed only Cloudflare, the sole provider account requirement, making acquisition and binary configuration for local methods appear to be missing.", + "Changed the page to an access and installation checklist: Cloudflare lists account and credential variables, while the six local methods list official acquisition sources and binary variables.", + "Publish only variable names and official sources; never retain values, local paths, cookies, or personal sessions." + ], + [ + "Review attribution", + "The ranking said only that results were reviewed, without identifying the reviewer.", + "Each method's review column now shows 100/100 · reviewed by Codex and continues to link to the shared review record.", + "Presentation copy only; review bindings, coverage, and unresolved counts are unchanged." + ], + [ + "Chinese and English editions", + "The former English entry was a one-page summary whose routes, sections, and per-attempt evidence explanations did not align with the Chinese report.", + "Generate route-aligned Chinese and English sites from the same report model. Navigation follows dataset, methodology, analysis, review, interpretation, methods, reproduction, changelog, and setup; switching language stays on the current route.", + "Validate one-to-one parity for routes, sections, tables, values, filters, and evidence links. UI and explanations are rewritten for natural English, while task names, IDs, URLs, reference answers, and raw outputs remain unchanged and are not rescored." + ], + [ + "Public-release security gate", + "The previous release scan covered common Google, Artifact Hub, GitHub, and OpenAI key patterns, but did not explicitly block Authorization values, local user-directory paths, or AWS and Slack credentials.", + "Every rebuilt public file is now scanned uniformly, adding checks for Authorization values, user-directory paths, AWS and Slack credentials, and private-key patterns. Existing redaction of Google-shaped secrets in third-party HTML retains both hashes.", + "Pattern scanning cannot prove that every unknown secret is absent. Structured credential values remain prohibited in evidence and reports, and published files still require byte-level verification." + ], + [ + "Moli latency correction", + "A few anomalous legacy latency values made 1.1.3 appear to have an unproven overall performance regression.", + "Run three order-rotated rounds for the same four tasks and three versions, use the 12 third-round latency values uniformly, and recompute mean, P50, and P95 without selecting favorable results.", + "Across passing tasks, the 1.1.3 mean fell from 4,000 to 3,706 ms, P50 from 3,444 to 3,340 ms, and P95 from 9,833 to 7,983 ms. Versions 1.0.6 and 1.1.1 were updated under the same rule. All 36 physical calls are retained; body text, scores, and task verdicts are unchanged." + ] ] } } @@ -3366,17 +4798,36 @@ "区分两条路径：错误命令、版本、裁判或参考按明确范围 supersede 并重测；普通失败可在测量前声明最多重试一次，保留两次物理证据且最后一次无条件生效。", "当前 1,050 条结果没有启用重试，issue #66 的诊断复测也不替换正式结果；后续协议可按需启用。" ] - ] - , + ], "i18n": { "en": { "title": "Unify method-combination comparisons and add Moli latency diagnostics", "summary": "The analysis page now uses one comparator for both two-method and multi-method comparisons, with a clearer task-and-result layout. After rotating request order on the public web and checking against a fixed local page, the regression signal in issue #66 remains a long-tail effect that cannot be attributed to a single cause. Official tasks, scores, and latency values are unchanged.", "changes": [ - ["Unified comparator", "Two-method and multi-method comparisons were split across two large cards and two result tables, duplicating controls and crowding the page.", "Keep one method A selector, a multi-select for comparison methods, and one shared set of relationship filters; render only one matched-task result table.", "Supports disagreements, A passing or failing, all methods passing or failing, and only the selected methods passing; no new score is produced."], - ["Result layout", "The two-column layout squeezed the task-family and method columns, wrapping Chinese family names one character at a time and making horizontal reading difficult.", "The component now uses the full width. Task family appears as a badge inside the task cell, and passing and failing methods appear as wrapping status labels.", "The table can still scroll horizontally on narrow screens, while task names, IDs, and evidence links remain complete."], - ["Moli latency signal", "The official single-run results show 1.1.3 markedly slower than earlier versions on a few tasks, but cannot separate version effects from network and page-wait effects.", "Retest four high-difference tasks on distinct paths in three rotating rounds, then use five rounds per version on a fixed local page to measure fixed overhead.", "Per-task medians on the public web move in both directions and have overlapping ranges. Fixed local overhead shows no 1.1.3 regression, so issue #66 is not promoted to a confirmed regression."], - ["Corrections and failure retries", "Banning every later call would prevent correction of proven errors; selecting a better response after failure would instead create best-of bias.", "Separate two paths: an incorrect command, version, judge, or reference supersedes the declared scope and is rerun; an ordinary failure may use at most one retry declared before measurement, retaining both physical calls and unconditionally using the final one.", "Retries are not enabled for the current 1,050 results, and the diagnostic reruns for issue #66 do not replace official results; future protocols may enable retries when needed."] + [ + "Unified comparator", + "Two-method and multi-method comparisons were split across two large cards and two result tables, duplicating controls and crowding the page.", + "Keep one method A selector, a multi-select for comparison methods, and one shared set of relationship filters; render only one matched-task result table.", + "Supports disagreements, A passing or failing, all methods passing or failing, and only the selected methods passing; no new score is produced." + ], + [ + "Result layout", + "The two-column layout squeezed the task-family and method columns, wrapping Chinese family names one character at a time and making horizontal reading difficult.", + "The component now uses the full width. Task family appears as a badge inside the task cell, and passing and failing methods appear as wrapping status labels.", + "The table can still scroll horizontally on narrow screens, while task names, IDs, and evidence links remain complete." + ], + [ + "Moli latency signal", + "The official single-run results show 1.1.3 markedly slower than earlier versions on a few tasks, but cannot separate version effects from network and page-wait effects.", + "Retest four high-difference tasks on distinct paths in three rotating rounds, then use five rounds per version on a fixed local page to measure fixed overhead.", + "Per-task medians on the public web move in both directions and have overlapping ranges. Fixed local overhead shows no 1.1.3 regression, so issue #66 is not promoted to a confirmed regression." + ], + [ + "Corrections and failure retries", + "Banning every later call would prevent correction of proven errors; selecting a better response after failure would instead create best-of bias.", + "Separate two paths: an incorrect command, version, judge, or reference supersedes the declared scope and is rerun; an ordinary failure may use at most one retry declared before measurement, retaining both physical calls and unconditionally using the final one.", + "Retries are not enabled for the current 1,050 results, and the diagnostic reruns for issue #66 do not replace official results; future protocols may enable retries when needed." + ] ] } } @@ -3406,16 +4857,30 @@ "三轮配对诊断将瞬时请求、页面不稳定和当前主榜成绩分开；京东保留为网络与会话敏感的挑战样本。", "当前主榜未启用失败重试，也不用诊断调用挑选更好响应；后续可预先声明失败重试，已证实的错误可按完整范围纠正并替换权威逻辑结果。" ] - ] - , + ], "i18n": { "en": { "title": "Add matched-task method comparisons and fix stalls on large outputs", "summary": "The analysis page now includes two-method differences and multi-method intersections. Complete evidence larger than 256 KiB is no longer inserted automatically and instead loads on demand or downloads directly. Official tasks, calls, verdicts, scores, and latency values are unchanged.", "changes": [ - ["Matched-task method comparison", "The old page could only filter 1,050 calls by method or result, making it difficult to find tasks where A passed and B failed, both passed, both failed, or several methods disagreed.", "Add two-method differences and multi-method intersections. Each row reuses the reviewed primary verdict for the same task and links to every reference answer, actual output, and failure reason.", "This is set analysis: it produces no new score and does not mix latency from different passing samples."], - ["Large-evidence loading", "Automatically downloading an output of more than a million characters and inserting it into a pre element occupied the main thread for a long time, making the page appear frozen.", "Text larger than 256 KiB first shows an on-demand control. After a click, the complete content is placed in a read-only text box; the original-file download is always retained.", "Only browser presentation timing and controls change; evidence bytes and checksums in the release bundle are unchanged."], - ["Version and sample boundaries", "Moli 1.1.3 passing two fewer tasks than 1.1.1 could be read directly as a capability regression, while JD.com occasionally opening could be used to dismiss the official failure.", "Three paired diagnostic rounds separate transient requests and page instability from the current primary ranking. JD.com remains a challenge sample sensitive to network and session state.", "The current primary ranking does not enable failure retries or use diagnostic calls to select better responses. Future runs may predeclare failure retries, while proven errors may be corrected across their complete scope and replace the authoritative logical results."] + [ + "Matched-task method comparison", + "The old page could only filter 1,050 calls by method or result, making it difficult to find tasks where A passed and B failed, both passed, both failed, or several methods disagreed.", + "Add two-method differences and multi-method intersections. Each row reuses the reviewed primary verdict for the same task and links to every reference answer, actual output, and failure reason.", + "This is set analysis: it produces no new score and does not mix latency from different passing samples." + ], + [ + "Large-evidence loading", + "Automatically downloading an output of more than a million characters and inserting it into a pre element occupied the main thread for a long time, making the page appear frozen.", + "Text larger than 256 KiB first shows an on-demand control. After a click, the complete content is placed in a read-only text box; the original-file download is always retained.", + "Only browser presentation timing and controls change; evidence bytes and checksums in the release bundle are unchanged." + ], + [ + "Version and sample boundaries", + "Moli 1.1.3 passing two fewer tasks than 1.1.1 could be read directly as a capability regression, while JD.com occasionally opening could be used to dismiss the official failure.", + "Three paired diagnostic rounds separate transient requests and page instability from the current primary ranking. JD.com remains a challenge sample sensitive to network and session state.", + "The current primary ranking does not enable failure retries or use diagnostic calls to select better responses. Future runs may predeclare failure retries, while proven errors may be corrected across their complete scope and replace the authoritative logical results." + ] ] } } @@ -3445,16 +4910,30 @@ "新增 benchmark-report Skill，并由根 AGENTS.md 强制路由；规则适用于 Web Fetch、Browser Use 及其他系统对比，不绑定本报告页面结构。", "这是流程和维护门禁；本轮主结果仍为 1,050 条，同窗测速仍为 140 条，未重新调用 provider。" ] - ] - , + ], "i18n": { "en": { "title": "Migrate to Sites and formalize the benchmark-report workflow", "summary": "The report moved to a new Sites domain while retaining the original sites. The repository now includes a report skill spanning benchmark types and covering evaluation design, evidence, independent review, impact-scoped reruns, and publication gates. Tasks, outputs, scores, and latency values are unchanged.", "changes": [ - ["Publication target", "The old publication script hard-coded the artifact-hub.dev domain, so a new session could continue updating an old site.", "The current report configuration binds both the Sites service address and site ID. Publishing, verification, and body links read the same configuration, and an update is rejected if the targets disagree.", "The new site is UUAhhQMnrGwm; neither old site, bWkRP157EtF6 or UeZE1vfcEOrM, was overwritten."], - ["Publishing identity", "An old publication token in the repository .env could take precedence over the new user-level Sites token, causing 401 responses even after authorization.", "Publishing and verification scripts no longer load the evaluation .env; they read only the explicit process environment or user-level token, and device authorization backs up a differing old token.", "Only the source of publication credentials changes; no provider call or report data changes."], - ["Cross-benchmark workflow", "The existing benchmark-editorial skill covers only formal interpretation articles and cannot govern evaluation design, score review, selective reruns, or report publication.", "Add the benchmark-report skill and require routing through it from the root AGENTS.md. Its rules apply to Web Fetch, Browser Use, and other system comparisons without depending on this report's page structure.", "This is a workflow and maintenance gate. The run still contains 1,050 primary results and 140 same-window latency results; no provider was called again."] + [ + "Publication target", + "The old publication script hard-coded the artifact-hub.dev domain, so a new session could continue updating an old site.", + "The current report configuration binds both the Sites service address and site ID. Publishing, verification, and body links read the same configuration, and an update is rejected if the targets disagree.", + "The new site is UUAhhQMnrGwm; neither old site, bWkRP157EtF6 or UeZE1vfcEOrM, was overwritten." + ], + [ + "Publishing identity", + "An old publication token in the repository .env could take precedence over the new user-level Sites token, causing 401 responses even after authorization.", + "Publishing and verification scripts no longer load the evaluation .env; they read only the explicit process environment or user-level token, and device authorization backs up a differing old token.", + "Only the source of publication credentials changes; no provider call or report data changes." + ], + [ + "Cross-benchmark workflow", + "The existing benchmark-editorial skill covers only formal interpretation articles and cannot govern evaluation design, score review, selective reruns, or report publication.", + "Add the benchmark-report skill and require routing through it from the root AGENTS.md. Its rules apply to Web Fetch, Browser Use, and other system comparisons without depending on this report's page structure.", + "This is a workflow and maintenance gate. The run still contains 1,050 primary results and 140 same-window latency results; no provider was called again." + ] ] } } @@ -3485,16 +4964,30 @@ "核验轮次改为独立的结构化记录并在构建时校验；Moli 直接文件统一使用裸 fetch；文档命令统一使用锁定的 uv 运行入口。", "页面只展示每轮范围、发现、纠正和未完成项；不增加核验结果下载。" ] - ] - , + ], "i18n": { "en": { "title": "Pin the default report evidence and measured routes", "summary": "Default rebuilds now accept only the current evidence for seven methods, 150 tasks, and 1,050 primary calls. Machine-readable routes now reflect the native Markdown, Moli automatic raw responses, and node-html-markdown 2.0.0 actually used in this run. Scores, call outputs, and latency values are unchanged.", "changes": [ - ["Default rebuild", "The default evidence directory still pointed to the old twelve-method evaluation, so a new session could silently generate a different report.", "Set the reviewed seven-method evidence as current and add a separate current-report configuration file. Build and evidence checks reject evidence whose methods, counts, deadlines, or routes do not match.", "Primary-result review remains 1,050/1,050 and same-window latency review remains 140/140; old evidence is retained under the research directory."], - ["Route metadata", "Download data still identified Chromium's converter as the retired pyhtml2md 1.7.0 and did not state how the three Moli versions handled direct documents and media.", "Route descriptions now share the runner's current definitions: ordinary pages use each method's native body capability, Moli uses automatic raw responses for direct documents and media, and Chromium uses node-html-markdown 2.0.0.", "Only non-verdict metadata is corrected; tasks, references, outputs, scores, and latency values are unchanged."], - ["Regression-prevention gate", "Review rounds were attached to one release record, so adding a release could empty the review page; the generic Moli file adapter could also still append --dump html to binary responses.", "Review rounds are now independent structured records validated at build time. Moli direct files consistently use bare fetch, and document commands consistently use the locked uv runtime entry point.", "The page shows only each round's scope, findings, corrections, and outstanding items; no review-result download is added."] + [ + "Default rebuild", + "The default evidence directory still pointed to the old twelve-method evaluation, so a new session could silently generate a different report.", + "Set the reviewed seven-method evidence as current and add a separate current-report configuration file. Build and evidence checks reject evidence whose methods, counts, deadlines, or routes do not match.", + "Primary-result review remains 1,050/1,050 and same-window latency review remains 140/140; old evidence is retained under the research directory." + ], + [ + "Route metadata", + "Download data still identified Chromium's converter as the retired pyhtml2md 1.7.0 and did not state how the three Moli versions handled direct documents and media.", + "Route descriptions now share the runner's current definitions: ordinary pages use each method's native body capability, Moli uses automatic raw responses for direct documents and media, and Chromium uses node-html-markdown 2.0.0.", + "Only non-verdict metadata is corrected; tasks, references, outputs, scores, and latency values are unchanged." + ], + [ + "Regression-prevention gate", + "Review rounds were attached to one release record, so adding a release could empty the review page; the generic Moli file adapter could also still append --dump html to binary responses.", + "Review rounds are now independent structured records validated at build time. Moli direct files consistently use bare fetch, and document commands consistently use the locked uv runtime entry point.", + "The page shows only each round's scope, findings, corrections, and outstanding items; no review-result download is added." + ] ] } } @@ -3555,20 +5048,54 @@ "环境和复现合并，只保留版本、二进制哈希、部署边界、配置文件和锁定命令；账号页仅列所需账号、变量名和官方获取入口。", "正式密钥只从仓库本地 .env 读取，不进入 HTML、Git 或发布包；Kitesurf 的托管后端版本和本地部署不可复现边界单独披露。" ] - ] - , + ], "i18n": { "en": { "title": "Complete seven-method measurement, full review, and same-window latency testing", "summary": "All seven methods completed the same 150 tasks for 1,050 primary calls, plus 20 tasks × 7 methods for 140 same-window latency calls. A Codex task agent reviewed 1,190/1,190 outputs without seeing machine verdicts or scores; the final decisions agree with current scoring, with 0 unresolved.", "changes": [ - ["Complete seven-method matrix", "After Moli 1.0.6 was added, the old six-method denominator and version comparison were incomplete.", "Each of the seven methods runs the same 150 tasks, retaining one authoritative logical result per method-task pair; Moli 1.0.6, 1.1.1, and 1.1.3 are listed separately.", "Pass counts are Moli 1.1.1 135, Moli 1.0.6 134, Moli 1.1.3 133, Chromium 125, Lightpanda 119, Obscura 117, and Kitesurf 112. The totals show coverage of this dataset only and do not replace the six family rankings."], - ["Independent-review ledger", "Reviewing only failures would miss false positives, and old detail pages also mistook layout differences for missing content.", "A Codex task agent read every task contract, reference answer, and complete actual output. Primary-result review covers 1,050/1,050 and same-window latency review covers 140/140; machine verdicts and scores were hidden until after the independent decision.", "Current automated false negatives are 0, false positives are 0, and unresolved items are 0. Historical issues including challenge-08, dynamic-quote field boundaries, the Feishu table, and direct-document routes were rescored or rerun under the current shared rules."], - ["Same-window latency test", "The primary matrix spans several execution batches, so remote networking, queuing, and local resource contention prevent fair cross-batch latency ranking.", "Use 20 predeclared pages on distinct domains and run all seven methods in parallel within the same window. Latency is compared only on the same 8 tasks passed by every method.", "Moli 1.0.6 has the lowest sample P50. The 8 tasks are a small exploratory sample, and P95 is only a sample diagnostic, not an estimate of stable tail latency or an SLA."], - ["Reference and route corrections", "live-025 incorrectly used the initial HTTP bytes as the reference for a dynamic page. For direct PDFs and media, forcing --dump on Moli fails before the artifact is retrieved.", "live-025 now uses visible body text rendered by an independent browser, and all seven methods were rerun. The three Moli versions use bare fetch for direct PDF, DOCX, and media URLs; body tasks include local parsing, while file tasks validate bytes directly.", "References, input-output targets, and the scoring entry point are identical for every method; each method may still use its fastest official native-artifact route."], - ["document-03 reference replacement", "The old W3C PDF URL became a Cloudflare verification page during this run; continuing to use it would mistake site blocking for PDF-parsing capability.", "Replace it with a three-page PDF from a pinned Mozilla PDF.js Git commit and call all seven methods again; the unchanged 1,043 primary results and their reviews remain intact.", "Moli 1.0.6 and 1.1.1 retrieved all three pages of text and passed. Moli 1.1.3 had a single request failure, while the other methods did not retrieve readable body text. This single result cannot by itself establish a version regression."], - ["Title diagnostics", "The title metric previously showed a definition followed by an empty table, leaving its inputs, alignment, and aggregation process invisible.", "Show four steps: read records, align by detail URL, compare each title, and aggregate comparable results. When no titles can be compared, explicitly show not applicable instead of generating an empty table.", "Title diagnostics still do not determine task acceptance and do not represent ranking recall, freshness, or whole-page body quality."], - ["Environment and reproduction", "The old environment page repeated every batch, while the account page could imply that production credentials should be stored in the report.", "Combine environment and reproduction, retaining only versions, binary hashes, deployment boundaries, configuration files, and locked commands. The account page lists only required accounts, variable names, and official acquisition sources.", "Production credentials are read only from the repository-local .env and never enter HTML, Git, or the release bundle. Kitesurf's hosted backend version and the non-reproducibility of local deployment are disclosed separately."] + [ + "Complete seven-method matrix", + "After Moli 1.0.6 was added, the old six-method denominator and version comparison were incomplete.", + "Each of the seven methods runs the same 150 tasks, retaining one authoritative logical result per method-task pair; Moli 1.0.6, 1.1.1, and 1.1.3 are listed separately.", + "Pass counts are Moli 1.1.1 135, Moli 1.0.6 134, Moli 1.1.3 133, Chromium 125, Lightpanda 119, Obscura 117, and Kitesurf 112. The totals show coverage of this dataset only and do not replace the six family rankings." + ], + [ + "Independent-review ledger", + "Reviewing only failures would miss false positives, and old detail pages also mistook layout differences for missing content.", + "A Codex task agent read every task contract, reference answer, and complete actual output. Primary-result review covers 1,050/1,050 and same-window latency review covers 140/140; machine verdicts and scores were hidden until after the independent decision.", + "Current automated false negatives are 0, false positives are 0, and unresolved items are 0. Historical issues including challenge-08, dynamic-quote field boundaries, the Feishu table, and direct-document routes were rescored or rerun under the current shared rules." + ], + [ + "Same-window latency test", + "The primary matrix spans several execution batches, so remote networking, queuing, and local resource contention prevent fair cross-batch latency ranking.", + "Use 20 predeclared pages on distinct domains and run all seven methods in parallel within the same window. Latency is compared only on the same 8 tasks passed by every method.", + "Moli 1.0.6 has the lowest sample P50. The 8 tasks are a small exploratory sample, and P95 is only a sample diagnostic, not an estimate of stable tail latency or an SLA." + ], + [ + "Reference and route corrections", + "live-025 incorrectly used the initial HTTP bytes as the reference for a dynamic page. For direct PDFs and media, forcing --dump on Moli fails before the artifact is retrieved.", + "live-025 now uses visible body text rendered by an independent browser, and all seven methods were rerun. The three Moli versions use bare fetch for direct PDF, DOCX, and media URLs; body tasks include local parsing, while file tasks validate bytes directly.", + "References, input-output targets, and the scoring entry point are identical for every method; each method may still use its fastest official native-artifact route." + ], + [ + "document-03 reference replacement", + "The old W3C PDF URL became a Cloudflare verification page during this run; continuing to use it would mistake site blocking for PDF-parsing capability.", + "Replace it with a three-page PDF from a pinned Mozilla PDF.js Git commit and call all seven methods again; the unchanged 1,043 primary results and their reviews remain intact.", + "Moli 1.0.6 and 1.1.1 retrieved all three pages of text and passed. Moli 1.1.3 had a single request failure, while the other methods did not retrieve readable body text. This single result cannot by itself establish a version regression." + ], + [ + "Title diagnostics", + "The title metric previously showed a definition followed by an empty table, leaving its inputs, alignment, and aggregation process invisible.", + "Show four steps: read records, align by detail URL, compare each title, and aggregate comparable results. When no titles can be compared, explicitly show not applicable instead of generating an empty table.", + "Title diagnostics still do not determine task acceptance and do not represent ranking recall, freshness, or whole-page body quality." + ], + [ + "Environment and reproduction", + "The old environment page repeated every batch, while the account page could imply that production credentials should be stored in the report.", + "Combine environment and reproduction, retaining only versions, binary hashes, deployment boundaries, configuration files, and locked commands. The account page lists only required accounts, variable names, and official acquisition sources.", + "Production credentials are read only from the repository-local .env and never enter HTML, Git, or the release bundle. Kitesurf's hosted backend version and the non-reproducibility of local deployment are disclosed separately." + ] ] } } @@ -3598,16 +5125,30 @@ "主结果 900/900 全量盲审，包括 Moli 1.1.1 和 1.1.3 各 150 条；184 个机器失败项全部有非空独立审查证据。", "各方法失败核验覆盖均为 100%，无未裁定结果。" ] - ] - , + ], "i18n": { "en": { "title": "Distinguish text presence from intact field boundaries", "summary": "Review two Moli outputs for the asynchronous Quotes list: all 10 quotations and authors were retrieved, but adjacent fields and multiple tags lacked readable separators. The task remains failed, while the detail page now presents quotations, authors, and tags field by field instead of describing lost boundaries broadly as missing text.", "changes": [ - ["Field-level evidence", "The old detail page marked an entire stable record as unmatched even when its quotation and author appeared in full, obscuring that the actual failure was the tag-list boundary.", "For each record, show the reference, parsed value, and verdict for quotation, author, and tags; when all tag characters are present but separators disappear, explicitly mark the field boundary invalid.", "Only the failure explanation and classification are corrected; the task contract, raw output, latency, and score do not change."], - ["Review of two Moli versions", "Looking only at Moli 1.1.3 would miss whether the earlier version had the same problem.", "The outputs from Moli 1.1.1 and 1.1.3 for this task are byte-for-byte identical, each with 10 complete unique records. In 8 records, multiple tags run together, and the last record also includes Loading text.", "Both versions remain failed. Single-tag record 9 matches normally, showing that the rule checks tag boundaries rather than whether the quotation appears."], - ["Failure-review coverage", "Readers need to distinguish full review from checks of only selected versions.", "All 900/900 primary results were blindly reviewed, including Moli 1.1.1 and 1.1.3 with 150 each. Every one of the 184 machine failures has nonempty independent review evidence.", "Failure-review coverage is 100% for every method, with no unadjudicated results."] + [ + "Field-level evidence", + "The old detail page marked an entire stable record as unmatched even when its quotation and author appeared in full, obscuring that the actual failure was the tag-list boundary.", + "For each record, show the reference, parsed value, and verdict for quotation, author, and tags; when all tag characters are present but separators disappear, explicitly mark the field boundary invalid.", + "Only the failure explanation and classification are corrected; the task contract, raw output, latency, and score do not change." + ], + [ + "Review of two Moli versions", + "Looking only at Moli 1.1.3 would miss whether the earlier version had the same problem.", + "The outputs from Moli 1.1.1 and 1.1.3 for this task are byte-for-byte identical, each with 10 complete unique records. In 8 records, multiple tags run together, and the last record also includes Loading text.", + "Both versions remain failed. Single-tag record 9 matches normally, showing that the rule checks tag boundaries rather than whether the quotation appears." + ], + [ + "Failure-review coverage", + "Readers need to distinguish full review from checks of only selected versions.", + "All 900/900 primary results were blindly reviewed, including Moli 1.1.1 and 1.1.3 with 150 each. Every one of the 184 machine failures has nonempty independent review evidence.", + "Failure-review coverage is 100% for every method, with no unadjudicated results." + ] ] } } @@ -3650,18 +5191,42 @@ "删除与目标无关的表格语法限制；继续用 3 个稳定事实、五段正文覆盖和完整文本质量验收，同一规则作用于全部方法。", "沿用原始输出和耗时离线重评分，不重跑 provider；3 次结果由失败改为通过。" ] - ] - , + ], "i18n": { "en": { "title": "Audit document references and correct equivalent-table misclassification", "summary": "Audit all 10 document tasks across their frozen sources, complete body references, body-region contracts, and 60 actual outputs, and recheck the Feishu help-document table verdict. No document reference was truncated. The Feishu task had incorrectly treated Markdown table syntax as a content requirement, causing 3 misclassifications that are now corrected under the same input-output target.", "changes": [ - ["Coverage presentation", "Region details placed a short inspection snippet beside the long reference body, making it easy to mistake the snippet for the only text used in machine scoring.", "The result page now explains the denominator, numerator, and complete input in place: coverage is computed from the complete actual output above, while the locator snippet only helps readers inspect one match.", "The scoring implementation, saved output, latency, and score are unchanged."], - ["Minimal PDF scope", "The source body for document-03 contains only Dummy PDF file; reusing the long-document description would make the reference look incomplete.", "The report explicitly labels this a minimal document-parsing sample: the complete extractable body of the frozen source is 14 characters, and the task checks only whether short PDF text can be obtained.", "This task cannot establish support for multiple pages, tables, or appendices. None of the six methods retrieved these 14 characters in this run, so all results remain failed."], - ["Document-reference audit", "When an embedded-document task retains only the surrounding HTML, readers cannot trace which PDF supplied the complete body.", "document-07 records the PDF actually loaded by the viewer, the existing frozen-file path, and two hashes. document-08 records the embedded URL in the frozen HTML, the independent-download hash, and the extracted-body hash.", "All 10 document references and body-region contracts passed review; no score changed because of a truncated reference."], - ["Moli DOCX description", "The report said the direct DOCX was obtained as a raw response, but Moli 1.1.3 actually serialized that DOCX as HTML containing package bytes in this run.", "The method page now describes automatic selection accurately: the PDF became a raw response and was parsed, while the DOCX did not become a parseable original file and failed.", "No external fallback such as curl is added, and another downloader's success is not credited to Moli."], - ["Equivalent table output", "The original rule for the Feishu help document accepted only a Markdown pipe table or an HTML table. Obscura and two Moli versions returned all three column headings and the complete table content, but expanded it as plain text in reading order and were therefore misclassified as failures.", "Remove the table-syntax restriction, which is unrelated to the target, while retaining acceptance based on 3 stable facts, five-region body coverage, and complete-text quality. The same rule applies to every method.", "Rescore the existing raw outputs and latency offline without rerunning the provider; 3 results change from fail to pass."] + [ + "Coverage presentation", + "Region details placed a short inspection snippet beside the long reference body, making it easy to mistake the snippet for the only text used in machine scoring.", + "The result page now explains the denominator, numerator, and complete input in place: coverage is computed from the complete actual output above, while the locator snippet only helps readers inspect one match.", + "The scoring implementation, saved output, latency, and score are unchanged." + ], + [ + "Minimal PDF scope", + "The source body for document-03 contains only Dummy PDF file; reusing the long-document description would make the reference look incomplete.", + "The report explicitly labels this a minimal document-parsing sample: the complete extractable body of the frozen source is 14 characters, and the task checks only whether short PDF text can be obtained.", + "This task cannot establish support for multiple pages, tables, or appendices. None of the six methods retrieved these 14 characters in this run, so all results remain failed." + ], + [ + "Document-reference audit", + "When an embedded-document task retains only the surrounding HTML, readers cannot trace which PDF supplied the complete body.", + "document-07 records the PDF actually loaded by the viewer, the existing frozen-file path, and two hashes. document-08 records the embedded URL in the frozen HTML, the independent-download hash, and the extracted-body hash.", + "All 10 document references and body-region contracts passed review; no score changed because of a truncated reference." + ], + [ + "Moli DOCX description", + "The report said the direct DOCX was obtained as a raw response, but Moli 1.1.3 actually serialized that DOCX as HTML containing package bytes in this run.", + "The method page now describes automatic selection accurately: the PDF became a raw response and was parsed, while the DOCX did not become a parseable original file and failed.", + "No external fallback such as curl is added, and another downloader's success is not credited to Moli." + ], + [ + "Equivalent table output", + "The original rule for the Feishu help document accepted only a Markdown pipe table or an HTML table. Obscura and two Moli versions returned all three column headings and the complete table content, but expanded it as plain text in reading order and were therefore misclassified as failures.", + "Remove the table-syntax restriction, which is unrelated to the target, while retaining acceptance based on 3 stable facts, five-region body coverage, and complete-text quality. The same rule applies to every method.", + "Rescore the existing raw outputs and latency offline without rerunning the provider; 3 results change from fail to pass." + ] ] } } @@ -3727,21 +5292,60 @@ "组装器保留源证据顺序，并将参考修订、评分修订和结果核验记录随报告发布。", "900/900 指标可从保存输出复现；150 个任务和 6 种方法均无缺项。" ] - ] - , + ], "i18n": { "en": { "title": "Correct document and resource routes and independently review all 900 results", "summary": "Across 150 tasks × 6 methods, all 900 calls have explicit results. Moli 1.1.3 now uses automatic raw responses for direct documents and media. Codex gpt-5.6-sol blindly reviewed all 900 saved outputs; every disagreement was adjudicated and none remain unresolved. Task detail pages show the reference answer, actual output, and verdict evidence directly.", "changes": [ - ["Correct route for Moli 1.1.3", "Using --dump html/markdown for a direct PDF errors before body retrieval and cannot measure the raw-response capability of 1.1.3 accurately.", "Ordinary pages continue to use native Markdown. Direct documents and media use bare fetch, allowing the CLI to select a raw response or serialized page automatically. PDF, PNG, WebM, and MP4 returned original bytes in this run; DOCX was serialized as HTML and failed.", "Only the 7 direct-document calls and 3 direct-file calls affected by the implementation are replaced. Moli 1.1.3 ultimately passes 133/150, while the older Moli 1.1.1 passes 123/150."], - ["Separate document-body and file tracks", "Downloading a PDF or DOCX is not the same as obtaining its body text; conversely, a body task should not force every method through the same intermediate format.", "Body tasks uniformly require readable text: direct body output is accepted, while an original file is parsed locally with parsing time included. Original-file tasks validate format, length, and target SHA-256 without an unrelated Markdown conversion.", "Current coverage includes PDF, DOCX, TXT, and embedded documents; it does not claim support for legacy DOC, OCR, table semantics, or arbitrary office formats."], - ["Independent review of every result", "Reviewing only automated failures would miss false positives. Giving the agent only an empty text placeholder for file tasks would also create false failures.", "Codex gpt-5.6-sol reviewed 900/900 results without seeing machine verdicts or scores. File tasks also supplied real binary evidence, and the 3 batches affected by a protocol defect were reviewed separately.", "Every disagreement has observable evidence and a final adjudication, with 0 unresolved; all 120/120 same-window latency results were reviewed separately."], - ["Shared scoring correction", "Markdown link boundaries could make an existing Reddit community description appear missing, and percent-encoded relative download URLs could also be missed.", "Restore visible link text at layout boundaries. Resolve and URL-decode only angle-bracketed relative targets with file extensions against the source page; do not treat ordinary filename prose as a URL.", "False negatives for two Moli versions on challenge-08 are corrected, and Chromium's Blender download-package URL is changed to pass. Raw output, call status, latency, and content hashes remain unchanged."], - ["Corrupted-text detection", "A successful request and long output do not establish readable text; run-9b35581eb0fdbbdea4a6 contains many replacement characters.", "Consistently measure the count and share of U+FFFD, classify the body as decode-corrupted at a material threshold, and show the count, ratio, and raw output on the detail page.", "The call remains failed, but its reason changes from generic missing content to verifiable decode corruption; garbled text is not treated as valid body text."], - ["Task-detail redesign", "The old page labeled five regions from one answer as body-region, while hiding the reference answer and actual output behind numerous tables and disclosure controls.", "The first screen shows task ID, method, run ID, and conclusion, followed by the complete reference answer and actual output side by side. Region checks retain only a summary table, source evidence expands on demand, and run fields are grouped in a disclosure at the end.", "Only evidence presentation changes; no task, output, score, or latency value is modified."], - ["Version traceability", "When the runtime version string is missing, a product name alone cannot establish which build was tested.", "Local implementations are pinned by binary SHA-256: Moli 1.1.1 was matched byte for byte to the official archive afterward; Moli 1.1.3 and Chromium retain runtime versions and hashes; Lightpanda is identified by its exact nightly hash.", "Kitesurf is a hosted service whose server build is unpublished; the report marks it unavailable instead of inventing a version."], - ["Evidence packaging", "Changing JSONL row order while reassembling the same records invalidates independent-review hashes.", "The assembler preserves source-evidence order and publishes reference revisions, scoring revisions, and result-review records with the report.", "The 900/900 metric is reproducible from saved outputs; none of the 150 tasks or 6 methods is missing."] + [ + "Correct route for Moli 1.1.3", + "Using --dump html/markdown for a direct PDF errors before body retrieval and cannot measure the raw-response capability of 1.1.3 accurately.", + "Ordinary pages continue to use native Markdown. Direct documents and media use bare fetch, allowing the CLI to select a raw response or serialized page automatically. PDF, PNG, WebM, and MP4 returned original bytes in this run; DOCX was serialized as HTML and failed.", + "Only the 7 direct-document calls and 3 direct-file calls affected by the implementation are replaced. Moli 1.1.3 ultimately passes 133/150, while the older Moli 1.1.1 passes 123/150." + ], + [ + "Separate document-body and file tracks", + "Downloading a PDF or DOCX is not the same as obtaining its body text; conversely, a body task should not force every method through the same intermediate format.", + "Body tasks uniformly require readable text: direct body output is accepted, while an original file is parsed locally with parsing time included. Original-file tasks validate format, length, and target SHA-256 without an unrelated Markdown conversion.", + "Current coverage includes PDF, DOCX, TXT, and embedded documents; it does not claim support for legacy DOC, OCR, table semantics, or arbitrary office formats." + ], + [ + "Independent review of every result", + "Reviewing only automated failures would miss false positives. Giving the agent only an empty text placeholder for file tasks would also create false failures.", + "Codex gpt-5.6-sol reviewed 900/900 results without seeing machine verdicts or scores. File tasks also supplied real binary evidence, and the 3 batches affected by a protocol defect were reviewed separately.", + "Every disagreement has observable evidence and a final adjudication, with 0 unresolved; all 120/120 same-window latency results were reviewed separately." + ], + [ + "Shared scoring correction", + "Markdown link boundaries could make an existing Reddit community description appear missing, and percent-encoded relative download URLs could also be missed.", + "Restore visible link text at layout boundaries. Resolve and URL-decode only angle-bracketed relative targets with file extensions against the source page; do not treat ordinary filename prose as a URL.", + "False negatives for two Moli versions on challenge-08 are corrected, and Chromium's Blender download-package URL is changed to pass. Raw output, call status, latency, and content hashes remain unchanged." + ], + [ + "Corrupted-text detection", + "A successful request and long output do not establish readable text; run-9b35581eb0fdbbdea4a6 contains many replacement characters.", + "Consistently measure the count and share of U+FFFD, classify the body as decode-corrupted at a material threshold, and show the count, ratio, and raw output on the detail page.", + "The call remains failed, but its reason changes from generic missing content to verifiable decode corruption; garbled text is not treated as valid body text." + ], + [ + "Task-detail redesign", + "The old page labeled five regions from one answer as body-region, while hiding the reference answer and actual output behind numerous tables and disclosure controls.", + "The first screen shows task ID, method, run ID, and conclusion, followed by the complete reference answer and actual output side by side. Region checks retain only a summary table, source evidence expands on demand, and run fields are grouped in a disclosure at the end.", + "Only evidence presentation changes; no task, output, score, or latency value is modified." + ], + [ + "Version traceability", + "When the runtime version string is missing, a product name alone cannot establish which build was tested.", + "Local implementations are pinned by binary SHA-256: Moli 1.1.1 was matched byte for byte to the official archive afterward; Moli 1.1.3 and Chromium retain runtime versions and hashes; Lightpanda is identified by its exact nightly hash.", + "Kitesurf is a hosted service whose server build is unpublished; the report marks it unavailable instead of inventing a version." + ], + [ + "Evidence packaging", + "Changing JSONL row order while reassembling the same records invalidates independent-review hashes.", + "The assembler preserves source-evidence order and publishes reference revisions, scoring revisions, and result-review records with the report.", + "The 900/900 metric is reproducible from saved outputs; none of the 150 tasks or 6 methods is missing." + ] ] } } @@ -3795,20 +5399,54 @@ "发布目标固定为独立站点 bWkRP157EtF6。", "旧站点 UeZE1vfcEOrM 保持不变。" ] - ] - , + ], "i18n": { "en": { "title": "Complete 750 measured calls on the fastest Markdown routes of five browsers", "summary": "Kitesurf, Lightpanda, Obscura, and Moli use native Markdown, while Chromium uses a rendered DOM with node-html-markdown 2.0.0. All 150 tasks × 5 methods were executed, and the independent-reference lifecycle and offline rescoring both passed.", "changes": [ - ["Method contract", "The old draft forced all five methods through the same HTML conversion chain, obscuring the advantage of native interfaces.", "Align URLs, Markdown output, task set, acceptance rules, and the 45-second deadline; each method uses its fastest official or built-in Markdown route.", "Kitesurf uses the account-level Cloudflare /markdown route; three local CLIs use native Markdown; only Chromium, which lacks native Markdown, converts the DOM."], - ["Chromium route", "The old pyhtml2md route converted large pages slowly and passed one fewer ordinary web task.", "Rescore candidate converters on the same saved DOM batch, adopt node-html-markdown 2.0.0, and retain the HTML, version, hash, and conversion latency.", "Chromium passes 98/100 ordinary web tasks in this run. Conversion is included in client end-to-end latency and is not presented as a native browser capability."], - ["Historical-verdict review", "The old rules missed line breaks across Markdown links, native-interface link layout, layout whitespace, and some dynamic expressions, and had also accepted incomplete body text.", "Compare all 750 saved outputs individually with their reference answers and task contracts, correcting 43 false negatives and 1 false positive under shared rules; run-f4e6f2ee641e8baa9e3d is confirmed as misclassified.", "The corrected rules apply to every method without provider-specific exceptions; raw outputs, call identities, and latency values do not change."], - ["Sync with main", "Upstream task-rule and report-defect corrections can affect earlier conclusions.", "After merging the latest main, review the affected scope. Because the Django tutorial reference changed, rerun the matched task for all five methods; reuse other results only when reference bytes and evaluation contracts are equivalent.", "The new Moli call for this task exits the browser, leaving its ordinary-web result at 93/100; the old pass is not retained."], - ["Fairness", "Remote Kitesurf, local processes, differing readiness conditions, and conversion time could be misread as pure engine performance.", "Pair speed only on identical URLs passed by both methods and state the client end-to-end boundary explicitly.", "The results cannot establish same-machine CPU/RSS, network-excluded engine speed, sustained throughput, cost, or an SLA."], - ["Task identification", "Task detail pages showed only titles, so readers could not confirm task IDs from the page.", "Both task pages and per-call pages now show stable task IDs immediately; run IDs, rules, references, and outputs remain traceable.", "The presentation change does not alter tasks, results, or scores."], - ["Publication isolation", "This report serves a different purpose from the old report, and overwriting the old site would break historical links.", "The publication target is pinned to the separate site bWkRP157EtF6.", "The old site UeZE1vfcEOrM remains unchanged."] + [ + "Method contract", + "The old draft forced all five methods through the same HTML conversion chain, obscuring the advantage of native interfaces.", + "Align URLs, Markdown output, task set, acceptance rules, and the 45-second deadline; each method uses its fastest official or built-in Markdown route.", + "Kitesurf uses the account-level Cloudflare /markdown route; three local CLIs use native Markdown; only Chromium, which lacks native Markdown, converts the DOM." + ], + [ + "Chromium route", + "The old pyhtml2md route converted large pages slowly and passed one fewer ordinary web task.", + "Rescore candidate converters on the same saved DOM batch, adopt node-html-markdown 2.0.0, and retain the HTML, version, hash, and conversion latency.", + "Chromium passes 98/100 ordinary web tasks in this run. Conversion is included in client end-to-end latency and is not presented as a native browser capability." + ], + [ + "Historical-verdict review", + "The old rules missed line breaks across Markdown links, native-interface link layout, layout whitespace, and some dynamic expressions, and had also accepted incomplete body text.", + "Compare all 750 saved outputs individually with their reference answers and task contracts, correcting 43 false negatives and 1 false positive under shared rules; run-f4e6f2ee641e8baa9e3d is confirmed as misclassified.", + "The corrected rules apply to every method without provider-specific exceptions; raw outputs, call identities, and latency values do not change." + ], + [ + "Sync with main", + "Upstream task-rule and report-defect corrections can affect earlier conclusions.", + "After merging the latest main, review the affected scope. Because the Django tutorial reference changed, rerun the matched task for all five methods; reuse other results only when reference bytes and evaluation contracts are equivalent.", + "The new Moli call for this task exits the browser, leaving its ordinary-web result at 93/100; the old pass is not retained." + ], + [ + "Fairness", + "Remote Kitesurf, local processes, differing readiness conditions, and conversion time could be misread as pure engine performance.", + "Pair speed only on identical URLs passed by both methods and state the client end-to-end boundary explicitly.", + "The results cannot establish same-machine CPU/RSS, network-excluded engine speed, sustained throughput, cost, or an SLA." + ], + [ + "Task identification", + "Task detail pages showed only titles, so readers could not confirm task IDs from the page.", + "Both task pages and per-call pages now show stable task IDs immediately; run IDs, rules, references, and outputs remain traceable.", + "The presentation change does not alter tasks, results, or scores." + ], + [ + "Publication isolation", + "This report serves a different purpose from the old report, and overwriting the old site would break historical links.", + "The publication target is pinned to the separate site bWkRP157EtF6.", + "The old site UeZE1vfcEOrM remains unchanged." + ] ] } } @@ -3856,19 +5494,48 @@ "六类任务概览前置，直接列出样本数、完整判分的方法数、最高任务通过率及对应方法；详细榜单留在后方。", "不同类别任务要求不同，最高分不能跨类别直接比较。" ] - ] - , + ], "i18n": { "en": { "title": "Complete file-task evaluation and establish a report-completeness publication gate", "summary": "All 36 file-task calls were actually made and judged: Moli and Crawl4AI each pass 3, for 6 passes, 30 failures, and 0 unmeasured. Every one of the report's 1,800 planned calls has a unique result and explicit verdict.", "changes": [ - ["File tasks", "Earlier records did not retain original-file bytes, so retrieval of direct PNG, WebM, and MP4 files could not be judged.", "Each of the twelve methods runs 3 file tasks; all 36 calls in the final configuration are validated by format, length, and target SHA-256.", "Moli and Crawl4AI each pass 3 calls, while the other 30 fail. Failures remain real results, and valid difficult tasks are not removed."], - ["Native-interface scope", "Testing only body-text routes would miss publicly offered native file capabilities and mistake a partial configuration for product capability.", "Moli file tasks output original-file bytes. Crawl4AI file tasks preselect its native HTTP crawler, while body tasks retain the browser route.", "Evaluate the public interface fixed before the task begins, without post-failure fallback or presenting an ordinary HTTP download as a provider result."], - ["Timing boundary", "Latency did not clearly distinguish server time, downloading, rendering, local processing, and report processing.", "Use one client end-to-end timing boundary: from starting the provider call until a scoreable final answer exists. Downloading, rendering, extraction, input conversion, and file hashing are included.", "Scoring, evidence persistence, rate-limit waits, and report generation are excluded; provider server-side latency is not ranked."], - ["Completeness gate", "Using actual calls as the denominator could make 8/8 appear complete despite missing planned calls; two coverage columns were also redundant.", "Change to “judgement coverage = explicitly judged calls / planned calls” and require exactly one current result for every provider × case.", "Any missing, duplicate, or unjudgeable result marks the entire report unreliable; completeness checks, report validation, and release packaging all block publication of incomplete results."], - ["Evidence presentation", "Successful files were retained in the bundle but difficult to verify directly from call pages, while the changelog stacked several same-day revisions.", "Successful-call pages provide file downloads, format, byte count, and SHA-256. Same-day revisions are merged into one official release, with earlier releases collapsed.", "Only audit access and reading structure improve; original files, scores, and latency values do not change."], - ["Overview structure", "The family overview was hidden after detailed rankings, forcing readers through large tables first.", "Move the six-family overview forward and list sample count, methods with complete judgements, highest task pass rate, and the corresponding methods directly; detailed rankings remain later.", "Task requirements differ by family, so top scores cannot be compared directly across families."] + [ + "File tasks", + "Earlier records did not retain original-file bytes, so retrieval of direct PNG, WebM, and MP4 files could not be judged.", + "Each of the twelve methods runs 3 file tasks; all 36 calls in the final configuration are validated by format, length, and target SHA-256.", + "Moli and Crawl4AI each pass 3 calls, while the other 30 fail. Failures remain real results, and valid difficult tasks are not removed." + ], + [ + "Native-interface scope", + "Testing only body-text routes would miss publicly offered native file capabilities and mistake a partial configuration for product capability.", + "Moli file tasks output original-file bytes. Crawl4AI file tasks preselect its native HTTP crawler, while body tasks retain the browser route.", + "Evaluate the public interface fixed before the task begins, without post-failure fallback or presenting an ordinary HTTP download as a provider result." + ], + [ + "Timing boundary", + "Latency did not clearly distinguish server time, downloading, rendering, local processing, and report processing.", + "Use one client end-to-end timing boundary: from starting the provider call until a scoreable final answer exists. Downloading, rendering, extraction, input conversion, and file hashing are included.", + "Scoring, evidence persistence, rate-limit waits, and report generation are excluded; provider server-side latency is not ranked." + ], + [ + "Completeness gate", + "Using actual calls as the denominator could make 8/8 appear complete despite missing planned calls; two coverage columns were also redundant.", + "Change to “judgement coverage = explicitly judged calls / planned calls” and require exactly one current result for every provider × case.", + "Any missing, duplicate, or unjudgeable result marks the entire report unreliable; completeness checks, report validation, and release packaging all block publication of incomplete results." + ], + [ + "Evidence presentation", + "Successful files were retained in the bundle but difficult to verify directly from call pages, while the changelog stacked several same-day revisions.", + "Successful-call pages provide file downloads, format, byte count, and SHA-256. Same-day revisions are merged into one official release, with earlier releases collapsed.", + "Only audit access and reading structure improve; original files, scores, and latency values do not change." + ], + [ + "Overview structure", + "The family overview was hidden after detailed rankings, forcing readers through large tables first.", + "Move the six-family overview forward and list sample count, methods with complete judgements, highest task pass rate, and the corresponding methods directly; detailed rankings remain later.", + "Task requirements differ by family, so top scores cannot be compared directly across families." + ] ] } } @@ -3923,20 +5590,54 @@ "增加共同通过子集诊断，并明确单次、跨批次与参考范围限制。", "不能从本轮样本外推全网能力、稳定性或生产 SLA。" ] - ] - , + ], "i18n": { "en": { "title": "Unify acceptance criteria and correct references, scores, and report conclusions", "summary": "Retain 150 tasks and 1,800 original calls. Verdicts change for 92 tasks, and 36 historical records lacking file-byte evidence change from incorrectly recorded failures to unmeasured. Raw outputs and latency values are unchanged.", "changes": [ - ["Body references", "The CSS reference body's table of contents was incorrectly removed, while the Django document included surrounding site content.", "Redetermine body boundaries from independently saved source pages and record provenance and reference hashes.", "Full-text F1 changes with the corrected references; earlier low scores cannot be attributed directly to provider body-text noise."], - ["Shared scoring", "Explicit name-link fields and valid Markdown soft line breaks were not handled consistently in acceptance.", "All methods share one answer transformation and scoring entry point, excluding request echoes and diagnostic logs.", "Correct format-induced false failures without scoring branches keyed by provider name."], - ["Body coverage", "Checking only opening keywords and minimum length could let a very short excerpt pass a long-document task.", "For tasks with a complete independent reference, add token-coverage checks across up to five contiguous regions.", "This is a minimum content-coverage requirement, not a replacement for full-text F1 or a guarantee of semantic or factual correctness."], - ["Dynamic fields", "Required installment and delivery fields in the product task were not fully validated.", "Check field ownership, count, type, and uniqueness against independent references; volatile prices and ordering are not frozen as permanent answers.", "Complete fields do not establish that volatile values are accurate or current."], - ["Deadline", "A result exceeding the registered budget could still count as successful.", "New calls use a shared end-to-end deadline; historical records are rejudged only from their registered latency and budget.", "The report does not claim that historical calls were originally controlled by the new hard deadline."], - ["Unmeasured files", "36 file records without byte evidence were treated as failures.", "First mark them unmeasured and remove them from the pass-rate denominator, then fill them with real provider calls on 2026-09-04.", "Do not substitute a URL, text, or another method's download for the original method's file evidence."], - ["Interpreting results", "Conditional latency from different passing subsets was treated as matched-task speed, and some article conclusions exceeded the evidence.", "Add diagnostics for the jointly passing subset and state single-run, cross-batch, and reference-scope limitations explicitly.", "This sample cannot establish internet-wide capability, stability, or a production SLA."] + [ + "Body references", + "The CSS reference body's table of contents was incorrectly removed, while the Django document included surrounding site content.", + "Redetermine body boundaries from independently saved source pages and record provenance and reference hashes.", + "Full-text F1 changes with the corrected references; earlier low scores cannot be attributed directly to provider body-text noise." + ], + [ + "Shared scoring", + "Explicit name-link fields and valid Markdown soft line breaks were not handled consistently in acceptance.", + "All methods share one answer transformation and scoring entry point, excluding request echoes and diagnostic logs.", + "Correct format-induced false failures without scoring branches keyed by provider name." + ], + [ + "Body coverage", + "Checking only opening keywords and minimum length could let a very short excerpt pass a long-document task.", + "For tasks with a complete independent reference, add token-coverage checks across up to five contiguous regions.", + "This is a minimum content-coverage requirement, not a replacement for full-text F1 or a guarantee of semantic or factual correctness." + ], + [ + "Dynamic fields", + "Required installment and delivery fields in the product task were not fully validated.", + "Check field ownership, count, type, and uniqueness against independent references; volatile prices and ordering are not frozen as permanent answers.", + "Complete fields do not establish that volatile values are accurate or current." + ], + [ + "Deadline", + "A result exceeding the registered budget could still count as successful.", + "New calls use a shared end-to-end deadline; historical records are rejudged only from their registered latency and budget.", + "The report does not claim that historical calls were originally controlled by the new hard deadline." + ], + [ + "Unmeasured files", + "36 file records without byte evidence were treated as failures.", + "First mark them unmeasured and remove them from the pass-rate denominator, then fill them with real provider calls on 2026-09-04.", + "Do not substitute a URL, text, or another method's download for the original method's file evidence." + ], + [ + "Interpreting results", + "Conditional latency from different passing subsets was treated as matched-task speed, and some article conclusions exceeded the evidence.", + "Add diagnostics for the jointly passing subset and state single-run, cross-batch, and reference-scope limitations explicitly.", + "This sample cannot establish internet-wide capability, stability, or a production SLA." + ] ] } } diff --git a/tests/test_assembly_evidence.py b/tests/test_assembly_evidence.py index c657844dd..27664f71d 100644 --- a/tests/test_assembly_evidence.py +++ b/tests/test_assembly_evidence.py @@ -288,3 +288,97 @@ def test_assembly_rejects_changed_missing_or_linked_evidence(source, tmp_path, c assemble(root, tmp_path / 'certificates', [], target) assert not target.exists() assert not list(tmp_path.glob('.rejected-*')) + + +@pytest.mark.parametrize('operation', ['addition', 'replacement']) +def test_scored_cohort_composition_keeps_authority_without_starting_review(source, tmp_path, operation): + from harness.web_fetch.reference_lifecycle import valid_scoring_revision + from harness.web_fetch.scoring_revision import revise + + root, _ = source + revise(root, root / 'dataset.json', root_cause_category='answer-surface', + root_cause='Synthetic retained-answer correction is checked on the complete cohort.', + regression_ids=['answer-title-supplies-required-fact']) + assert valid_scoring_revision(root) + original_bytes = (root / 'results.jsonl').read_bytes() + original_history = (root / 'scoring-revisions.json').read_bytes() + dataset = json.loads((root / 'dataset.json').read_text()) + case = EvalCase(**dataset['cases'][0]) + class Fixture(Provider): + def fetch(self, case, timeout): + return ProviderResult(True, 'Expected') + provider = 'added' if operation == 'addition' else 'retrying' + incoming = tmp_path / 'incoming' + run_evaluation(output_dir=incoming, dataset_meta={**dataset, + 'coverage_plan': {'web': [provider]}}, dataset_path=tmp_path / 'input.json', + cases=[case], providers=[Fixture(provider, {'type': 'fixture'})], + provider_config_path=tmp_path / 'providers.json', runs=1, timeout=1, delay=0) + output = tmp_path / 'assembled' + assemble(root, tmp_path / 'certificates', [], output, + provider_additions=[incoming] if operation == 'addition' else [], + provider_replacements=[incoming] if operation == 'replacement' else []) + assert valid_scoring_revision(output) + assert (output / 'diagnostics/inherited-scoring-history.json').read_bytes() == original_history + receipt = json.loads((output / 'scoring-revisions.json').read_text())[-1] + assert receipt['inherited_scoring_authority']['receipts_sha256'] == hashlib.sha256(original_history).hexdigest() + assert receipt['task_decision_changes'] == [] + assert not (output / 'result-review.json').exists() + assert (root / 'results.jsonl').read_bytes() == original_bytes + assert (root / 'scoring-revisions.json').read_bytes() == original_history + result = [json.loads(line) for line in (output / 'results.jsonl').read_text().splitlines()] + inputs = ([json.loads(line) for line in original_bytes.splitlines()] + if operation == 'addition' else []) + inputs += [json.loads(line) for line in (incoming / 'results.jsonl').read_text().splitlines()] + indexed = {(row['provider'], row['case_id']): row for row in inputs} + assert len(result) == len(indexed) + for row in result: + old = indexed[row['provider'], row['case_id']] + assert row == old + for field in ('attempt_id', 'run_id', 'elapsed_ms', 'started_at', 'completed_at', 'metrics'): + assert row.get(field) == old.get(field) + assert (output / row['artifact_path']).read_bytes() == ( + (root if row['provider'] == 'retrying' and operation == 'addition' else incoming) + / old['artifact_path']).read_bytes() + + +@pytest.mark.parametrize('damage', ['source-authority', 'incoming-verdict', 'declared-review']) +def test_scored_cohort_composition_rejects_invalid_authority_or_changed_verdict(source, tmp_path, damage): + from harness.web_fetch.scoring_revision import revise + + root, _ = source + if damage == 'declared-review': + metadata = json.loads((root / 'dataset.json').read_text()) + metadata.setdefault('comparison', {})['require_independent_result_review'] = True + (root / 'dataset.json').write_text(json.dumps(metadata)) + revise(root, root / 'dataset.json', root_cause_category='answer-surface', + root_cause='Synthetic retained-answer correction is checked on the complete cohort.', + regression_ids=['answer-title-supplies-required-fact']) + dataset = json.loads((root / 'dataset.json').read_text()) + case = EvalCase(**dataset['cases'][0]) + class Fixture(Provider): + def fetch(self, case, timeout): + return ProviderResult(True, 'Expected') + incoming = tmp_path / 'incoming' + run_evaluation(output_dir=incoming, dataset_meta={**dataset, + 'coverage_plan': {'web': ['added']}}, dataset_path=tmp_path / 'input.json', + cases=[case], providers=[Fixture('added', {'type': 'fixture'})], + provider_config_path=tmp_path / 'providers.json', runs=1, timeout=1, delay=0) + if damage == 'source-authority': + path = root / 'scoring-revisions.json' + history = json.loads(path.read_text()) + history[-1]['output_results_sha256'] = 'f' * 64 + path.write_text(json.dumps(history)) + elif damage == 'incoming-verdict': + path = incoming / 'results.jsonl' + row = json.loads(path.read_text()) + row['metrics']['content_pass'] = False + path.write_text(json.dumps(row) + '\n') + protected = {folder: {p.relative_to(folder): p.read_bytes() for p in folder.rglob('*') + if p.is_file()} for folder in (root, incoming)} + output = tmp_path / 'assembled' + with pytest.raises(ValueError): + assemble(root, tmp_path / 'certificates', [], output, provider_additions=[incoming]) + assert not output.exists() + for folder, files in protected.items(): + assert {p.relative_to(folder): p.read_bytes() for p in folder.rglob('*') + if p.is_file()} == files diff --git a/tests/test_headless_report.py b/tests/test_headless_report.py index e067f0c32..9ab9ddb72 100644 --- a/tests/test_headless_report.py +++ b/tests/test_headless_report.py @@ -127,7 +127,10 @@ def test_resource_table_excludes_unobservable_remote_provider(self): 'planned_cases': ['a'] * 20} page, interpretation = _render_article_resources(summary, selected) self.assertNotIn('Kitesurf', page) - self.assertIn('8/20 个普通网页', page) + self.assertIn('data-resource-population="all_declared_calls"', page) + self.assertIn('全部 20 个普通网页，包含失败调用', page) + self.assertIn('共同通过 8/20', page) + self.assertIn('不用于选择资源表的样本', page) self.assertIn('本轮资源效率领先', page) self.assertIn('data-resource-winner="moli"', page) self.assertEqual(page.count('data-resource-metric-winner="moli"'), 12) diff --git a/tests/test_native_task_outputs.py b/tests/test_native_task_outputs.py index 3b6a841e9..53387c74e 100644 --- a/tests/test_native_task_outputs.py +++ b/tests/test_native_task_outputs.py @@ -91,6 +91,22 @@ def test_native_document_projection_rejects_bytes_and_preserves_native_text(self class NativeRoutePublicationTests(unittest.TestCase): + def test_dynamic_native_markdown_accepts_registered_builds_but_not_conversion(self): + from harness.web_fetch.task_outputs import require_native_task_routes + case={'id':'dynamic-page','url':'https://example.com/app','track':'dynamic','task_kind':'body'} + native={'route':'dynamic_native_markdown','output':'markdown','dump':'markdown'} + for provider in ('moli','moli-pr919'): + row={'provider':provider,'case_id':case['id'],'provider_metadata':native} + with self.subTest(provider=provider): + require_native_task_routes([case],[row]) + for bad in ({**native,'output':'final_dom_html+local_markdown'}, + {**native,'route':'dynamic_selected_text'}, + {**native,'dump':'json'}): + with self.subTest(provider=provider,bad=bad),self.assertRaises(ValueError): + require_native_task_routes([case],[{**row,'provider_metadata':bad}]) + with self.subTest(provider=provider,wrong_family=True),self.assertRaises(ValueError): + require_native_task_routes([{**case,'track':'web'}],[row]) + def test_page_routes_prioritize_native_output_and_keep_required_identity(self): from harness.web_fetch.task_outputs import require_native_task_routes pages=[{'id':'page','url':'https://example.com/page','track':'challenge','task_kind':'structure'}] diff --git a/tests/test_publication_privacy.py b/tests/test_publication_privacy.py index 7ac12df10..3a826776a 100644 --- a/tests/test_publication_privacy.py +++ b/tests/test_publication_privacy.py @@ -18,6 +18,28 @@ class PublicationPrivacyTests(unittest.TestCase): + def test_public_cors_boolean_and_abnf_are_not_credentials(self): + for policy in (b'Access-Control-Allow-Credentials: true', + b'Access-Control-Allow-Credentials = %s"true"', + b'Access-Control-Allow-Credentials = %i"false"', + b'Access-Control-Allow-Credentials = %s"true"'): + with self.subTest(policy=policy): + self.assertIsNone(publication_risk(policy)) + self.assertEqual(project_file('response.txt',policy),(policy,None)) + mixed=policy+b'\nAuthorization: Bearer PRIVATE_CANARY\n' + projected,receipt=project_file('response.txt',mixed) + self.assertIn(policy,projected) + self.assertNotIn(b'PRIVATE_CANARY',projected) + self.assertIsNone(publication_risk(projected)) + self.assertIsNotNone(receipt) + for secret in (b'Access-Control-Allow-Credentials: PRIVATE_CANARY', + b'Credentials: PRIVATE_CANARY',b'Authorization: Bearer PRIVATE_CANARY'): + with self.subTest(secret=secret): + self.assertEqual(publication_risk(secret),'sensitive field value') + projected,_=project_file('response.txt',secret) + self.assertNotIn(b'PRIVATE_CANARY',projected) + self.assertIsNone(publication_risk(projected)) + def test_header_at_end_of_json_text_cannot_consume_following_record_fields(self): record={'body':'RFC example: Authorization: Basic PRIVATE_CANARY', 'metrics':{'passed':False,'count':7},'case_id':'rfc-example'} diff --git a/tests/test_report_check_owner.py b/tests/test_report_check_owner.py index 5a27fae3e..b6845811a 100644 --- a/tests/test_report_check_owner.py +++ b/tests/test_report_check_owner.py @@ -296,3 +296,27 @@ def test_presentation_inventories_require_unique_registered_entries(tmp_path, de ) with pytest.raises(ValueError, match="Duplicate adoption|Interface inventory"): check_revision(root, authority_root=authority) + + +@pytest.mark.parametrize('defect', [None, 'missing', 'duplicate', 'unregistered']) +def test_registered_browser_build_has_its_own_interface_panel(tmp_path, defect): + from harness.web_fetch.report_check import check_revision + from tests.report_check_fixture import create_revision_report + root, authority = create_revision_report(tmp_path) + row=json.loads((root/'results.jsonl').read_text()) + row['provider']='moli-pr919' + (root/'results.jsonl').write_text(json.dumps(row)+'\n') + counts=json.loads((root/'adoption/counts.json').read_text()) + for item in counts['methods']: + if item['provider']=='alpha':item['provider']='moli-pr919' + write_json(root/'adoption/counts.json',counts) + panel='
Request
Response
' + path=root/'adoption/index.html' + additions='' if defect=='missing' else panel + if defect=='duplicate':additions+=panel + if defect=='unregistered':additions+=panel.replace('moli-pr919','unknown-build') + path.write_text(path.read_text().replace('',additions+'')) + if defect: + with pytest.raises(ValueError):check_revision(root,authority_root=authority) + else: + assert check_revision(root,authority_root=authority).startswith('PASS:') diff --git a/tests/test_resource_cohort_boundary.py b/tests/test_resource_cohort_boundary.py index 1eae5273f..035cc3f6b 100644 --- a/tests/test_resource_cohort_boundary.py +++ b/tests/test_resource_cohort_boundary.py @@ -151,3 +151,84 @@ def test_resource_summary_rejects_incomplete_scope_and_invalid_measurements(runs }[damage] with pytest.raises(ValueError): build_summary(dataset, records, batches) + + +def resource_population_fixture(tmp_path): + from copy import deepcopy + root = tmp_path / 'resource-populations' + original = make_run(root, 6, resource=True) + dataset = json.loads((root / 'dataset.json').read_text()) + dataset['coverage_plan']['web'] = ['fixture', 'other'] + records = deepcopy(original) + for index, row in enumerate(records): + row['provider_metadata']['client_resource']['cpu_ms'] = 100 if index == 5 else 1 + if index == 5: + row['metrics']['content_pass'] = False + for row in original: + added = deepcopy(row) + added['provider'] = 'other' + added['attempt_id'] = row['attempt_id'].replace('fixture', 'other') + records.append(added) + batches = [json.loads((root / 'manifest.json').read_text())] + return dataset, records, batches + + +def test_resource_primary_population_keeps_failed_calls_and_conditional_counts(tmp_path): + dataset, records, batches = resource_population_fixture(tmp_path) + summary = build_summary(dataset, records, batches) + indexed = {row['provider']:row for row in summary['rows']} + assert indexed['fixture']['metrics']['cpu_ms'] == {'n':6,'mean':17.5,'p50':1,'p95':100} + assert summary['primary_population'] == 'all_declared_calls' + assert indexed['other']['metrics']['cpu_ms']['mean'] == 2 + assert indexed['fixture']['successful_attempts'] == 5 + assert indexed['fixture']['successful_metrics']['cpu_ms']['mean'] == 1 + assert indexed['other']['successful_attempts'] == 6 + assert len(summary['common_pass_cases']) == 5 + + +def test_resource_cohort_with_no_common_success_still_reports_complete_measurements(tmp_path): + dataset, records, batches = resource_population_fixture(tmp_path) + for row in records: + if row['provider'] == 'fixture': + row['metrics']['content_pass'] = False + summary = build_summary(dataset, records, batches) + assert summary['common_pass_cases'] == [] + assert all(row['attempts'] == 6 for row in summary['rows']) + failed = next(row for row in summary['rows'] if row['provider'] == 'fixture') + assert failed['successful_metrics']['cpu_ms'] == {'n':0,'mean':None,'p50':None,'p95':None} + assert failed['metrics']['cpu_ms']['mean'] == 17.5 + + +@pytest.mark.parametrize('damage', [None,'wrong-population','missing-value','wrong-value','missing-winner','tied-winner','wrong-leader','wrong-direction']) +def test_resource_release_checks_populations_measurements_and_unique_highlights(tmp_path, damage): + from bs4 import BeautifulSoup + from harness.web_fetch.report_resources import check_resource_presentation, render_resource_comparison + dataset, records, batches = resource_population_fixture(tmp_path) + summary = build_summary(dataset, records, batches) + html, _ = render_resource_comparison(summary, summary['providers']) + soup = BeautifulSoup(html,'html.parser') + cpu = soup.select_one('td[data-resource-provider="other"][data-resource-key="cpu_ms"][data-resource-statistic="mean"]') + assert cpu.select_one('.comparison-winner') is not None + tied = soup.select_one('td[data-resource-key="memory_mean_bytes"]') + assert tied.select_one('.comparison-winner') is None + if damage == 'wrong-population': + soup.select_one('[data-resource-population]')['data-resource-cases'] = '5' + elif damage == 'missing-value': + cpu.decompose() + elif damage == 'wrong-value': + cpu['data-sort'] = '999' + elif damage == 'missing-winner': + cpu.select_one('.comparison-winner').unwrap() + elif damage == 'tied-winner': + tied.append(BeautifulSoup('wrong','html.parser')) + elif damage == 'wrong-leader': + soup.select_one('[data-resource-winner]')['data-resource-winner'] = 'fixture' + elif damage == 'wrong-direction': + for cell in soup.select('td[data-resource-key="cpu_ms"][data-resource-statistic="mean"]'): + cell['data-metric-direction'] = 'neutral' + cpu.select_one('.comparison-winner').unwrap() + if damage is None: + check_resource_presentation(str(soup),summary) + else: + with pytest.raises(ValueError): + check_resource_presentation(str(soup),summary) diff --git a/tests/test_scoring_revision.py b/tests/test_scoring_revision.py index ca9a5ee8c..f096157ef 100644 --- a/tests/test_scoring_revision.py +++ b/tests/test_scoring_revision.py @@ -93,3 +93,106 @@ def test_valid_history_can_extend_and_retains_prior_receipts(tmp_path: Path): for key in ("attempt_id", "run_id", "elapsed_ms", "artifact_path"): assert current[key] == original_record[key] assert current["metrics"]["content_sha256"] == original_record["metrics"]["content_sha256"] + + +@pytest.mark.parametrize('failed', [False, True]) +def test_scoring_validation_preserves_complete_saved_attempt_bytes(tmp_path, failed): + evidence, _, case = revision_fixture(tmp_path, b'[]') + row = json.loads((evidence / 'results.jsonl').read_text()) + if failed: + raw = b'other content\n' * 20 + (evidence / 'answer.txt').write_bytes(raw) + row['metrics'] = score_attempt(case, raw, ok=True) + row['scoring'] = {'scored_at':'frozen-time', 'mode':'original-scoring'} + row['diagnosis'] = {'retained':'original annotation'} + row['failure_category'] = 'original category' + (evidence / 'results.jsonl').write_text(json.dumps(row) + '\n') + before = {name:(evidence / name).read_bytes() + for name in ('dataset.json','results.jsonl','answer.txt','summary.json')} + receipt = revise(evidence, evidence / 'dataset.json', validate_only=True, + root_cause_category='answer-surface', + root_cause='Validate the saved scoring without rewriting a physical-call record.', + regression_ids=['answer-title-supplies-required-fact']) + assert receipt['validation_only'] is True + assert receipt['task_decision_changes'] == [] + assert receipt['input_results_sha256'] == receipt['output_results_sha256'] + assert valid_scoring_revision(evidence) + assert {name:(evidence / name).read_bytes() for name in before} == before + + +@pytest.mark.parametrize('damage', ['score','requirements']) +def test_scoring_validation_rejects_changed_verdict_or_requirements_atomically(tmp_path, damage): + evidence, canonical, _ = revision_fixture(tmp_path, b'[]') + if damage == 'score': + canonical = evidence / 'dataset.json' + row = json.loads((evidence / 'results.jsonl').read_text()) + row['metrics']['content_pass'] = False + (evidence / 'results.jsonl').write_text(json.dumps(row) + '\n') + before = {p.name:p.read_bytes() for p in evidence.iterdir()} + with pytest.raises(ValueError): + revise(evidence, canonical, validate_only=True, + root_cause_category='answer-surface', + root_cause='Validation rejects changed scores and requirements without writing evidence.', + regression_ids=['answer-title-supplies-required-fact']) + assert {p.name:p.read_bytes() for p in evidence.iterdir()} == before + + +def test_report_assets_can_update_together_without_changing_scored_calls(tmp_path): + from harness.web_fetch.scoring_revision import rebind_report_interpretation + evidence, canonical, _ = revision_fixture(tmp_path, b'[]') + revise(evidence, canonical, root_cause_category='answer-surface', + root_cause='Synthetic required-answer correction has an executable acceptance control.', + regression_ids=['answer-title-supplies-required-fact']) + calls = (evidence / 'results.jsonl').read_bytes() + content = evidence / 'report-assets' / 'browser-interpretation.json' + content.parent.mkdir() + content.write_text('{"title":"New report"}') + implementation = evidence / 'implementation-provenance.json' + implementation.write_text('{"implementations":[{"version":"source build"}]}') + binding = lambda p: {'path':p.relative_to(evidence).as_posix(), + 'sha256':hashlib.sha256(p.read_bytes()).hexdigest()} + receipt = rebind_report_interpretation(evidence, binding(content), + implementation_binding=binding(implementation)) + assert receipt['bound_asset_fields'] == ['report_interpretation', 'implementation_provenance'] + assert valid_scoring_revision(evidence) + assert (evidence / 'results.jsonl').read_bytes() == calls + dataset = json.loads((evidence / 'dataset.json').read_text()) + dataset['cases'][0]['required_patterns'] = [] + raw = (json.dumps(dataset, ensure_ascii=False, indent=2) + '\n').encode() + (evidence / 'dataset.json').write_bytes(raw) + (evidence / receipt['output_dataset_path']).write_bytes(raw) + receipt['output_dataset_sha256'] = hashlib.sha256(raw).hexdigest() + (evidence / 'report-binding-revisions.json').write_text(json.dumps([receipt])) + assert not valid_scoring_revision(evidence) + + +@pytest.mark.parametrize('damage', ['wrong-hash', 'wrong-path', 'missing', 'symlink']) +def test_report_asset_update_rejects_unbound_provenance_before_writing(tmp_path, damage): + from harness.web_fetch.scoring_revision import rebind_report_interpretation + evidence, canonical, _ = revision_fixture(tmp_path, b'[]') + revise(evidence, canonical, root_cause_category='answer-surface', + root_cause='Synthetic required-answer correction has an executable acceptance control.', + regression_ids=['answer-title-supplies-required-fact']) + content = evidence / 'article.json' + content.write_text('{}') + implementation = evidence / 'implementation-provenance.json' + implementation.write_text('{}') + candidate = {'path':implementation.name, + 'sha256':hashlib.sha256(implementation.read_bytes()).hexdigest()} + if damage == 'wrong-hash': + candidate['sha256'] = '0' * 64 + elif damage == 'wrong-path': + candidate['path'] = '../implementation-provenance.json' + elif damage == 'symlink': + target = evidence / 'other-provenance.json' + target.write_bytes(implementation.read_bytes()) + implementation.unlink() + implementation.symlink_to(target.name) + else: + implementation.unlink() + before = {p.relative_to(evidence):p.read_bytes() for p in evidence.rglob('*') if p.is_file()} + with pytest.raises((ValueError, OSError)): + rebind_report_interpretation(evidence, + {'path':content.name,'sha256':hashlib.sha256(content.read_bytes()).hexdigest()}, + implementation_binding=candidate) + assert {p.relative_to(evidence):p.read_bytes() for p in evidence.rglob('*') if p.is_file()} == before From 7641bbe808169dc14c2d71a96798f7c0b5462835 Mon Sep 17 00:00:00 2001 From: lanyue-llk <270302213+lanyue-llk@users.noreply.github.com> Date: Sat, 10 Oct 2026 19:08:02 +0800 Subject: [PATCH 004/172] chore(reports): preserve unchanged release record formatting --- research/report-changes.json | 2888 ++++++++-------------------------- 1 file changed, 641 insertions(+), 2247 deletions(-) diff --git a/research/report-changes.json b/research/report-changes.json index d87433d25..1c3e5bc57 100644 --- a/research/report-changes.json +++ b/research/report-changes.json @@ -174,18 +174,16 @@ "status": "已处理", "scope": "Browser Use 首页、核心指标表、同题矩阵、性能统计和结果分析页。", "findings": "上一版误把排除 14 个已知外部干扰任务后的 76 题事后诊断子集写成正式主结果，造成报告看起来只有 76 题；首页还重复突出首次执行与重试恢复，掩盖了每题一个最终逻辑结果的正式口径。", - "corrections": "首页和结果分析恢复完整 90 题官方 evaluator 结果；四种配置的核心指标、同题矩阵和性能统计均使用 90 题。76 题仅保留为明确标注的事后诊断子集，不改分、不替代官方结果。首页删除首次/重试拆分模块；共同成功性能统计按最终逻辑结果及其全部物理执行计算，物理执行明细只在单题证据页下钻。", + "corrections": "首页和结果分析恢复完整 90 题官方 evaluator 结果；四种配置的核心指标、同题矩阵和性能统计均使用 90 题。76 题仅保留为明确标注的事后诊断子集，不改分、不替代官方结果。首页删除首次/重试拆分模块；共同成功性能统计按最终逻辑结果及其全部物理执行计算，物理执行明细只在单题证据页下钻。", "remaining": "14 个外部干扰任务的历史分数保持不变；只有修复后按冻结协议产生新的完整 90 题证据批次，才能更新官方结果。", - "i18n": { - "en": { - "title": "Restore the complete 90-task official result as the primary view", - "status": "Resolved", - "scope": "The Browser Use homepage, core metrics, same-task matrix, performance statistics, and result analysis.", - "findings": "The prior version incorrectly presented the 76-task post-hoc diagnostic subset, which excludes 14 known interference tasks, as the formal primary result. It therefore appeared to contain only 76 tasks. The homepage also overemphasized first attempts and retry recovery instead of one final logical result per task.", - "corrections": "Restored the complete official 90-task evaluator result on the homepage and analysis page. Core metrics, the same-task matrix, and performance statistics now use all 90 tasks across all four configurations. The 76-task view remains only as an explicitly labeled post-hoc diagnostic subset and does not rescore or replace the official result. Removed the first-attempt/retry split from the homepage; common-success performance now uses each final logical result and all of its physical executions, while physical-execution details remain on task evidence pages.", - "remaining": "Historical scores for the 14 interference tasks remain unchanged. Only a new complete 90-task evidence cohort produced after corrections under the frozen protocol can update the official result." - } - } + "i18n": {"en": { + "title": "Restore the complete 90-task official result as the primary view", + "status": "Resolved", + "scope": "The Browser Use homepage, core metrics, same-task matrix, performance statistics, and result analysis.", + "findings": "The prior version incorrectly presented the 76-task post-hoc diagnostic subset, which excludes 14 known interference tasks, as the formal primary result. It therefore appeared to contain only 76 tasks. The homepage also overemphasized first attempts and retry recovery instead of one final logical result per task.", + "corrections": "Restored the complete official 90-task evaluator result on the homepage and analysis page. Core metrics, the same-task matrix, and performance statistics now use all 90 tasks across all four configurations. The 76-task view remains only as an explicitly labeled post-hoc diagnostic subset and does not rescore or replace the official result. Removed the first-attempt/retry split from the homepage; common-success performance now uses each final logical result and all of its physical executions, while physical-execution details remain on task evidence pages.", + "remaining": "Historical scores for the 14 interference tasks remain unchanged. Only a new complete 90-task evidence cohort produced after corrections under the frozen protocol can update the official result." + }} }, { "id": "review-2026-09-11-interpretation-reader-audit", @@ -196,16 +194,14 @@ "findings": "原稿把不同模型配置放在一起标记优胜，混淆了程序判分位置与人工复核根因；重试累计也会扭曲性能对比。部分失败缺少直接下钻入口，英文页仍有中文残留，平板宽度过早堆成单列。", "corrections": "优胜只在同一模型内比较浏览器；分别披露 43 个仅网络规则失败、20 个仅回答规则失败、12 个双规则失败，以及 29/24/10/8/2/2 的复核根因。性能资源改用四配置首次共同成功的 64 题且只取首次执行，并补齐逐结果链接、英文门禁和响应式布局。", "remaining": "当前仍是每种配置一个正式样本。14 个已确认外部干扰任务需在修复后完整重跑；增加重复次数前不发布模型或浏览器总体排名。", - "i18n": { - "en": { - "title": "Reader audit of interpretation, comparison semantics, and bilingual layout", - "status": "Resolved", - "scope": "The report home, guide, analysis, failure drill-down, English localization, and desktop, tablet, and phone breakpoints.", - "findings": "The draft marked winners across different model configurations, conflated programmatic failure location with human-reviewed root cause, and allowed retry accumulation to distort performance comparisons. Some failures lacked direct drill-down, English pages retained Chinese prose, and tablet layouts collapsed into one column too early.", - "corrections": "Winner marks now compare browsers only within one model. The report separately discloses 43 network-only, 20 response-only, and 12 dual-rule failures, plus the 29/24/10/8/2/2 reviewed root-cause split. Performance and resource statistics use the 64 tasks all four configurations passed on the first attempt and only the first execution; direct result links, English gates, and responsive layouts were added.", - "remaining": "Each configuration still has one formal sample. The 14 confirmed interference tasks require a complete rerun after fixes, and no overall model or browser ranking should be published before repetitions are added." - } - } + "i18n": {"en": { + "title": "Reader audit of interpretation, comparison semantics, and bilingual layout", + "status": "Resolved", + "scope": "The report home, guide, analysis, failure drill-down, English localization, and desktop, tablet, and phone breakpoints.", + "findings": "The draft marked winners across different model configurations, conflated programmatic failure location with human-reviewed root cause, and allowed retry accumulation to distort performance comparisons. Some failures lacked direct drill-down, English pages retained Chinese prose, and tablet layouts collapsed into one column too early.", + "corrections": "Winner marks now compare browsers only within one model. The report separately discloses 43 network-only, 20 response-only, and 12 dual-rule failures, plus the 29/24/10/8/2/2 reviewed root-cause split. Performance and resource statistics use the 64 tasks all four configurations passed on the first attempt and only the first execution; direct result links, English gates, and responsive layouts were added.", + "remaining": "Each configuration still has one formal sample. The 14 confirmed interference tasks require a complete rerun after fixes, and no overall model or browser ranking should be published before repetitions are added." + }} }, { "id": "review-2026-09-11-astra-layer-and-environment-isolation", @@ -216,16 +212,14 @@ "findings": "Sol 层为 Chromium 73/90、Moli 70/90；Astra 层两者均为 71/90。Astra Task 77 的两个失败来自共享 Shopping Admin 状态在准备后被并发执行改写，不是 Agent 误读或镜像基线错误。两个模型层使用的 browser-eval 提交不同，因此跨模型变化只能作观察性比较。", "corrections": "所有模型、浏览器和任务仍只保留一个逻辑结果；公开包补齐每次物理执行的脱敏证据与哈希；75/75 个失败均绑定当前证据并核验根因。运行器现在用按 Docker daemon 区分的跨进程文件锁覆盖 reset、Agent、证据收尾和 evaluator；同一环境 lane 串行。", "remaining": "历史官方分数不以人工核验覆盖；Task 77 只有在修复后的隔离环境中形成新的完整证据批次后才能更新成绩。真正并行执行仍要求每个 worker 拥有独立的完整站点栈、端口和浏览器容器。", - "i18n": { - "en": { - "title": "Astra model layer, per-execution evidence, and environment-isolation review", - "status": "Resolved", - "scope": "The GPT-5.6 Sol and GPT-6 Astra model layers, 360 logical results, 438 physical executions, 75 final failures, and the execution boundary of the shared WebArena site stack.", - "findings": "In the Sol layer Chromium scored 73/90 and Moli 70/90; in the Astra layer both scored 71/90. The two Astra Task 77 failures came from concurrent modification of shared Shopping Admin state after preparation, not an Agent reading error or a bad image baseline. The model layers used different browser-eval commits, so cross-model changes are observational only.", - "corrections": "Kept one logical result per model, browser, and task; published sanitized evidence and hashes for every physical execution; and bound all 75 failures to verified root causes. The runner now holds a Docker-daemon-scoped cross-process lock across reset, Agent execution, evidence finalization, and evaluation, serializing every run within one environment lane.", - "remaining": "Human review does not override historical official scores. Task 77 can change only after a new complete evidence cohort is produced under the corrected isolation protocol. Safe parallel execution still requires a separate full site stack, ports, and browser containers for every worker." - } - } + "i18n": {"en": { + "title": "Astra model layer, per-execution evidence, and environment-isolation review", + "status": "Resolved", + "scope": "The GPT-5.6 Sol and GPT-6 Astra model layers, 360 logical results, 438 physical executions, 75 final failures, and the execution boundary of the shared WebArena site stack.", + "findings": "In the Sol layer Chromium scored 73/90 and Moli 70/90; in the Astra layer both scored 71/90. The two Astra Task 77 failures came from concurrent modification of shared Shopping Admin state after preparation, not an Agent reading error or a bad image baseline. The model layers used different browser-eval commits, so cross-model changes are observational only.", + "corrections": "Kept one logical result per model, browser, and task; published sanitized evidence and hashes for every physical execution; and bound all 75 failures to verified root causes. The runner now holds a Docker-daemon-scoped cross-process lock across reset, Agent execution, evidence finalization, and evaluation, serializing every run within one environment lane.", + "remaining": "Human review does not override historical official scores. Task 77 can change only after a new complete evidence cohort is produced under the corrected isolation protocol. Safe parallel execution still requires a separate full site stack, ports, and browser containers for every worker." + }} }, { "id": "review-2026-09-11-90-task-expansion", @@ -236,16 +230,14 @@ "findings": "Chromium 通过 73/90，Moli 通过 70/90；69 题共同通过、16 题共同失败、5 题结果不一致。37 个最终失败覆盖 Agent 检索与执行、站点依赖、官方验收合同及 Moli 网络证据采集四类责任层，不能用“模型能力不行”统一解释。", "corrections": "逐结果绑定官方合同、最终回答、HAR 和 Agent 轨迹，完成 37/37 根因核验；报告改用 90 题动态口径并披露 220/220 环境准备回执。运行器对 Agent 启动前的瞬态 BrowserContextNotLoaded 增加一次仅限会话准备的恢复，不改变逻辑结果协议。", "remaining": "Astra 未由当前 OpenAI 代理暴露，本轮保持 gpt-5.6-sol、reasoning=low。Wikipedia 与 Map 尚未进入实跑；官方 evaluator 的已核验缺陷需上游修复后以新证据复测。", - "i18n": { - "en": { - "title": "90-task expansion and failure-cause review", - "status": "Resolved", - "scope": "The 90 preregistered tasks, 180 logical browser results, 220 physical executions, environment-preparation receipts, official scoring, and every final failure.", - "findings": "Chromium passed 73/90 and Moli passed 70/90. The paired results contain 69 joint passes, 16 joint failures, and five disagreements. The 37 final failures span agent retrieval or execution, site dependencies, official acceptance contracts, and Moli network-evidence collection; they do not support one blanket model-capability explanation.", - "corrections": "Bound each failed result to its official contract, final answer, HAR, and agent trajectory and completed 37/37 root-cause reviews. Updated the report to the 90-task cohort and disclosed 220/220 environment-preparation receipts. Session preparation now recovers once from a transient BrowserContextNotLoaded before the agent starts, without changing logical-result authority.", - "remaining": "This run remains on gpt-5.6-sol with reasoning=low because Astra was unavailable through the current model endpoint. Wikipedia and Map are not measured. Verified defects in the official evaluator require upstream correction and new execution evidence." - } - } + "i18n": {"en": { + "title": "90-task expansion and failure-cause review", + "status": "Resolved", + "scope": "The 90 preregistered tasks, 180 logical browser results, 220 physical executions, environment-preparation receipts, official scoring, and every final failure.", + "findings": "Chromium passed 73/90 and Moli passed 70/90. The paired results contain 69 joint passes, 16 joint failures, and five disagreements. The 37 final failures span agent retrieval or execution, site dependencies, official acceptance contracts, and Moli network-evidence collection; they do not support one blanket model-capability explanation.", + "corrections": "Bound each failed result to its official contract, final answer, HAR, and agent trajectory and completed 37/37 root-cause reviews. Updated the report to the 90-task cohort and disclosed 220/220 environment-preparation receipts. Session preparation now recovers once from a transient BrowserContextNotLoaded before the agent starts, without changing logical-result authority.", + "remaining": "This run remains on gpt-5.6-sol with reasoning=low because Astra was unavailable through the current model endpoint. Wikipedia and Map are not measured. Verified defects in the official evaluator require upstream correction and new execution evidence." + }} }, { "id": "review-2026-09-10-shared-failure-and-expansion", @@ -256,16 +248,14 @@ "findings": "相同官方分数掩盖了不同根因：Task 68 的官方 expected 与 schema 冲突；124 缺少候选语义核验；142 的类别验收边界未定义；168 的本项目响应协议与官方 null 合同冲突；506 中 Chromium 为 HAR 采集误判，Moli 为步数耗尽且未确认完成。", "corrections": "保留 evaluator 的程序自动定位；新增按浏览器与任务记录、绑定 attempts 哈希的根因子类和原因核验，未核验字段保持为空，正式发布要求全部失败结果完成核验。另新增运行前合同一致性校验、候选语义与提交前检查、空结果状态协议、JSON 响应体即时 HAR 采集、末步工具调用 fail-closed 和可配置步数上限。历史分数不改写。", "remaining": "修复后的 #124、#168、#506 尚需形成新执行证据；#68 与 #142 退出下一轮干净 A/B，直至合同被修复或验收口径可观察。Wikipedia 与 Map 尚未在本机实跑。", - "i18n": { - "en": { - "title": "Failure causes, corrections, and local-environment expansion", - "status": "Resolved", - "scope": "Automatic failure location, evidence-bound root-cause review, runtime corrections, result-page hierarchy, Colima network identity, and the locally runnable environment set.", - "findings": "The ten failed logical results span task-contract defects, agent protocol or policy defects, and evidence-collection defects. Equal official scores can therefore have different causes, including different causes for the two browsers on the same task.", - "corrections": "Kept automatic evaluator location separate from reviewed causes; added per-browser, per-task cause records bound to the attempts hash and required complete failure coverage for release. Added contract admission checks, candidate and pre-submit validation, an explicit empty-result protocol, immediate JSON HAR-body capture, fail-closed final-step handling, and a configurable step ceiling. Historical scores remain unchanged.", - "remaining": "Tasks affected by runtime corrections require new execution evidence. Tasks with defective or unobservable contracts remain outside the next clean A/B until those contracts are corrected. Wikipedia and Map remain unmeasured locally." - } - } + "i18n": {"en": { + "title": "Failure causes, corrections, and local-environment expansion", + "status": "Resolved", + "scope": "Automatic failure location, evidence-bound root-cause review, runtime corrections, result-page hierarchy, Colima network identity, and the locally runnable environment set.", + "findings": "The ten failed logical results span task-contract defects, agent protocol or policy defects, and evidence-collection defects. Equal official scores can therefore have different causes, including different causes for the two browsers on the same task.", + "corrections": "Kept automatic evaluator location separate from reviewed causes; added per-browser, per-task cause records bound to the attempts hash and required complete failure coverage for release. Added contract admission checks, candidate and pre-submit validation, an explicit empty-result protocol, immediate JSON HAR-body capture, fail-closed final-step handling, and a configurable step ceiling. Historical scores remain unchanged.", + "remaining": "Tasks affected by runtime corrections require new execution evidence. Tasks with defective or unobservable contracts remain outside the next clean A/B until those contracts are corrected. Wikipedia and Map remain unmeasured locally." + }} }, { "id": "review-2026-09-10-track-naming-and-release-gates", @@ -276,16 +266,14 @@ "findings": "轨道名称按实现层命名，读者不易判断测量对象；并列最优和显示值相同的指标仍可能被标优；英文可访问属性与失效筛选链接缺少回归保护；SVG 和跨断点布局只受人工规范约束。", "corrections": "统一命名为 Browser Use 与 Browser Trajectory Replay（轨迹重放），完成代码与命令迁移且不保留旧入口；优胜只允许显示值唯一领先；补充英文属性翻译、失效链接回退、SVG 拒绝和核心路由双语双主题宽窄屏渲染检查。", "remaining": "视觉门禁验证结构性溢出、裁切和表头竖排，仍不能替代人工对信息层级与审美质量的判断。", - "i18n": { - "en": { - "title": "Track naming and release-gate review", - "status": "Resolved", - "scope": "Names for autonomous and fixed-trajectory evaluation, winner marks, English accessibility text, filter state, and visual release gates.", - "findings": "Implementation-layer names obscured what each track measures; ties or display-equal metrics could be marked as winners; English accessibility attributes and invalid filter links lacked regression coverage; and responsive layout depended on manual checks.", - "corrections": "Standardized the names Browser Use and Browser Trajectory Replay, marked only a unique displayed winner, localized English accessibility attributes, restored invalid filters to valid defaults, rejected SVG assets, and added bilingual, two-theme desktop and narrow-screen checks for core routes.", - "remaining": "Automated visual gates detect structural overflow, clipping, and vertical headers, but do not replace human review of information hierarchy and visual quality." - } - } + "i18n": {"en": { + "title": "Track naming and release-gate review", + "status": "Resolved", + "scope": "Names for autonomous and fixed-trajectory evaluation, winner marks, English accessibility text, filter state, and visual release gates.", + "findings": "Implementation-layer names obscured what each track measures; ties or display-equal metrics could be marked as winners; English accessibility attributes and invalid filter links lacked regression coverage; and responsive layout depended on manual checks.", + "corrections": "Standardized the names Browser Use and Browser Trajectory Replay, marked only a unique displayed winner, localized English accessibility attributes, restored invalid filters to valid defaults, rejected SVG assets, and added bilingual, two-theme desktop and narrow-screen checks for core routes.", + "remaining": "Automated visual gates detect structural overflow, clipping, and vertical headers, but do not replace human review of information hierarchy and visual quality." + }} }, { "id": "review-2026-09-10-scalable-comparison-ui", @@ -296,16 +284,13 @@ "findings": "同题视图虽动态生成列，但交互仍以两浏览器关系为中心；动作表层级拥挤且对不同调用量的总耗时判优；完整 evaluator 错误被放入窄指标卡。", "corrections": "同题比较改为基准浏览器加多选对比浏览器和明确关系条件；动作延迟改为按 action 分卡并只比较单次均值；详情首屏只保留简明规则定位，完整 expected、actual 和错误进入规则证据区。", "remaining": "当前仅有两个参评浏览器；扩展行为已由三浏览器测试模型覆盖，新增真实浏览器仍需形成完整同题结果。", - "i18n": { - "en": { - "title": "Multi-browser comparison and result-detail review", - "status": "Resolved", - "scope": "Same-task comparison controls, BrowserTool action latency, and the opening section of result-detail pages.", - "findings": "The same-task view remained centered on a two-browser relationship; action tables ranked totals across unequal call counts; and full evaluator errors occupied narrow KPI cards.", - "corrections": "Introduced one baseline with multi-select comparison browsers and explicit outcome relations, compared action latency only by mean per call, and moved complete expected, actual, and error evidence out of the result summary.", - "remaining": "Only two browsers are measured. Three-browser fixtures cover interface scaling, but every new real browser still requires a complete same-task result set." - } - } + "i18n": {"en": { + "title": "Multi-browser comparison and result-detail review", "status": "Resolved", + "scope": "Same-task comparison controls, BrowserTool action latency, and the opening section of result-detail pages.", + "findings": "The same-task view remained centered on a two-browser relationship; action tables ranked totals across unequal call counts; and full evaluator errors occupied narrow KPI cards.", + "corrections": "Introduced one baseline with multi-select comparison browsers and explicit outcome relations, compared action latency only by mean per call, and moved complete expected, actual, and error evidence out of the result summary.", + "remaining": "Only two browsers are measured. Three-browser fixtures cover interface scaling, but every new real browser still requires a complete same-task result set." + }} }, { "id": "review-2026-09-10-report-model", @@ -316,16 +301,13 @@ "findings": "统计标签重复表达 P50；比较表缺少方向一致的优胜标记；结果页存在重复失败清单；核验页承担了逐结果清单职责。", "corrections": "统一为均值、P50、P95；按指标方向突出唯一优胜者；删除重复失败清单；核验页改为按轮次记录范围、发现、处理和遗留事项。", "remaining": "当前固定样本没有浏览器单边通过任务；优胜标记主要反映耗时与 Tokens，不改变官方任务成绩。", - "i18n": { - "en": { - "title": "Report model and comparison review", - "status": "Resolved", - "scope": "Summary statistics, winner presentation, result drill-down, review history, and browser-participant descriptions.", - "findings": "P50 was duplicated under another label, comparison tables lacked directionally consistent winner marks, the results page duplicated failure inventory, and the review page carried a per-result list.", - "corrections": "Standardized summaries on mean, P50, and P95; marked only unique directionally valid winners; removed the duplicate failure list; and made review history round-based.", - "remaining": "This fixed cohort has no one-browser-only pass. Winner marks mainly describe latency and token usage and do not alter official task scores." - } - } + "i18n": {"en": { + "title": "Report model and comparison review", "status": "Resolved", + "scope": "Summary statistics, winner presentation, result drill-down, review history, and browser-participant descriptions.", + "findings": "P50 was duplicated under another label, comparison tables lacked directionally consistent winner marks, the results page duplicated failure inventory, and the review page carried a per-result list.", + "corrections": "Standardized summaries on mean, P50, and P95; marked only unique directionally valid winners; removed the duplicate failure list; and made review history round-based.", + "remaining": "This fixed cohort has no one-browser-only pass. Winner marks mainly describe latency and token usage and do not alter official task scores." + }} }, { "id": "review-2026-09-10-evidence-boundaries", @@ -336,16 +318,13 @@ "findings": "环境拓扑与网关含义缺少集中说明；历史总耗时边界描述不准确；逐执行环境准备没有独立回执。", "corrections": "补充部署流程图、端口和镜像身份；区分 Agent 链路与 BrowserTool 墙钟时间；新版 runner 保存环境准备和分阶段计时。", "remaining": "本批冻结结果不具备逐执行准备回执，报告仅按现有证据声明可复现边界。", - "i18n": { - "en": { - "title": "Deployment evidence and timing-boundary review", - "status": "Resolved", - "scope": "Local-site deployment, the Colima network path, environment-preparation evidence, and Agent and BrowserTool timing.", - "findings": "The topology and gateway lacked a consolidated explanation, historical total-latency wording overstated its boundary, and per-execution preparation lacked an independent receipt.", - "corrections": "Added the deployment flow, ports, and image identities; separated agent-stack and BrowserTool wall time; and made the current runner retain environment-preparation and phase timings.", - "remaining": "This frozen run lacks per-execution preparation receipts, so reproducibility claims remain limited to the evidence actually retained." - } - } + "i18n": {"en": { + "title": "Deployment evidence and timing-boundary review", "status": "Resolved", + "scope": "Local-site deployment, the Colima network path, environment-preparation evidence, and Agent and BrowserTool timing.", + "findings": "The topology and gateway lacked a consolidated explanation, historical total-latency wording overstated its boundary, and per-execution preparation lacked an independent receipt.", + "corrections": "Added the deployment flow, ports, and image identities; separated agent-stack and BrowserTool wall time; and made the current runner retain environment-preparation and phase timings.", + "remaining": "This frozen run lacks per-execution preparation receipts, so reproducibility claims remain limited to the evidence actually retained." + }} }, { "id": "review-2026-09-09-runner-integrity", @@ -356,16 +335,13 @@ "findings": "异常清理可能遮蔽原始错误；HAR 停止操作缺少幂等保护；异常响应和原始证据目录需要更严格保护。", "corrections": "隔离清理异常、使 HAR 停止幂等、加强模型响应结构校验，并阻止原始浏览器证据进入 Git。", "remaining": "无阻塞项。", - "i18n": { - "en": { - "title": "Runner and evidence-integrity review", - "status": "Resolved", - "scope": "Model-response validation, BrowserTool cleanup, HAR retention, evidence directories, and resume behavior.", - "findings": "Cleanup exceptions could mask primary failures, HAR shutdown lacked idempotency, and malformed responses and raw-evidence directories required stronger protection.", - "corrections": "Isolated cleanup failures, made HAR shutdown idempotent, strengthened model-response validation, and prevented raw browser evidence from entering Git.", - "remaining": "No blocking items." - } - } + "i18n": {"en": { + "title": "Runner and evidence-integrity review", "status": "Resolved", + "scope": "Model-response validation, BrowserTool cleanup, HAR retention, evidence directories, and resume behavior.", + "findings": "Cleanup exceptions could mask primary failures, HAR shutdown lacked idempotency, and malformed responses and raw-evidence directories required stronger protection.", + "corrections": "Isolated cleanup failures, made HAR shutdown idempotent, strengthened model-response validation, and prevented raw browser evidence from entering Git.", + "remaining": "No blocking items." + }} } ], "browser_use_releases": [ @@ -380,39 +356,17 @@ }, "selection_sha256": "2cb716552e36df7535e83ed6d2cf3f84283e3aca86072817cd6e039a38547c7b", "changes": [ - [ - "跨报告比较", - "相同任务编号并不保证相同验收条件；已验证轨迹也不等于自主决策。", - "列出轨迹来源与合同修订范围，并绑定冻结定义的校验和。", - "重放 90/90 表示给定合格轨迹与对应合同下的执行结果，不能作为仅移除 Agent 的同条件对照。" - ], - [ - "失败机制", - "聚合成功率不能区分决策错误、预算限制与错误验收。", - "增加 Task 124、510、676 的轨迹和验收证据入口，解释具体失败机制及归因限制。", - "官方分数、90 题分母及原始耗时均保持不变；没有用诊断结果替换历史测量。" - ] + ["跨报告比较", "相同任务编号并不保证相同验收条件；已验证轨迹也不等于自主决策。", "列出轨迹来源与合同修订范围，并绑定冻结定义的校验和。", "重放 90/90 表示给定合格轨迹与对应合同下的执行结果，不能作为仅移除 Agent 的同条件对照。"], + ["失败机制", "聚合成功率不能区分决策错误、预算限制与错误验收。", "增加 Task 124、510、676 的轨迹和验收证据入口，解释具体失败机制及归因限制。", "官方分数、90 题分母及原始耗时均保持不变；没有用诊断结果替换历史测量。"] ], - "i18n": { - "en": { - "title": "Clarify autonomous-execution and trajectory-replay comparison limits", - "summary": "Added trajectory provenance and acceptance-contract differences: 60 of 90 trajectories come from successful historical executions and 30 were expert-repaired; 81 retain the original contract and nine use revised contracts. Per-task evidence distinguishes Agent decisions, budget exhaustion, evaluator event selection, and browser execution.", - "changes": [ - [ - "Cross-report comparison", - "Matching task IDs do not guarantee matching acceptance conditions, and executing a qualified trajectory is not autonomous planning.", - "List trajectory provenance and contract revisions, bound to checksums of the frozen definitions.", - "Replay's 90/90 describes execution under qualified trajectories and their contracts, not an otherwise identical experiment with only the Agent removed." - ], - [ - "Failure mechanisms", - "Aggregate pass rates do not separate decision errors, budget limits, and incorrect acceptance checks.", - "Add links to trajectory and evaluation evidence for tasks 124, 510, and 676, with concrete mechanisms and attribution limits.", - "Official scores, the 90-task denominator, and original timings remain unchanged. Diagnostic findings do not replace historical measurements." - ] - ] - } - } + "i18n": {"en": { + "title": "Clarify autonomous-execution and trajectory-replay comparison limits", + "summary": "Added trajectory provenance and acceptance-contract differences: 60 of 90 trajectories come from successful historical executions and 30 were expert-repaired; 81 retain the original contract and nine use revised contracts. Per-task evidence distinguishes Agent decisions, budget exhaustion, evaluator event selection, and browser execution.", + "changes": [ + ["Cross-report comparison", "Matching task IDs do not guarantee matching acceptance conditions, and executing a qualified trajectory is not autonomous planning.", "List trajectory provenance and contract revisions, bound to checksums of the frozen definitions.", "Replay's 90/90 describes execution under qualified trajectories and their contracts, not an otherwise identical experiment with only the Agent removed."], + ["Failure mechanisms", "Aggregate pass rates do not separate decision errors, budget limits, and incorrect acceptance checks.", "Add links to trajectory and evaluation evidence for tasks 124, 510, and 676, with concrete mechanisms and attribution limits.", "Official scores, the 90-task denominator, and original timings remain unchanged. Diagnostic findings do not replace historical measurements."] + ] + }} }, { "id": "2026-09-14-restore-complete-90-task-results", @@ -714,38 +668,16 @@ "发布质量门槛不降低，常规迭代时间缩短。" ] ], - "i18n": { - "en": { - "title": "Reorder the results report and migrate to production Sites", - "summary": "The home page now starts with the four core score and latency rows, followed by same-task differences, verified causes for 75 failures, and results for the three task types. This release changes presentation, checks, and the publication target only; no evaluation score changes.", - "changes": [ - [ - "Home-page structure", - "Core results were split across cards, making comparison difficult.", - "Replaced them with one model-by-browser core-metrics table at the start of the home page.", - "Pass count, pass rate, agent total time, BrowserTool time, and tokens can now be compared directly." - ], - [ - "Result interpretation", - "Several short labels required implementation context.", - "Replaced them with explicit sentences and column names for paired outcomes, failure causes, scope, and limitations.", - "Calculation rules are unchanged; the report is easier to understand." - ], - [ - "Failure causes", - "The 75 failure causes were spread across a large card grid.", - "Grouped them into a table covering agent, task or evaluator, test site, browser evidence, shared environment, and mixed causes, while retaining links to every reviewed result.", - "Each failure is still counted once, and its reviewed cause and evidence remain unchanged." - ], - [ - "Checks and publication", - "Routine edits also launched the complete browser-based visual suite.", - "Routine checks now cover data, structure, links, and copy; packaging still runs every language, theme, and viewport visual check.", - "Release quality gates remain intact while ordinary iteration is faster." - ] - ] - } - } + "i18n": {"en": { + "title": "Reorder the results report and migrate to production Sites", + "summary": "The home page now starts with the four core score and latency rows, followed by same-task differences, verified causes for 75 failures, and results for the three task types. This release changes presentation, checks, and the publication target only; no evaluation score changes.", + "changes": [ + ["Home-page structure", "Core results were split across cards, making comparison difficult.", "Replaced them with one model-by-browser core-metrics table at the start of the home page.", "Pass count, pass rate, agent total time, BrowserTool time, and tokens can now be compared directly."], + ["Result interpretation", "Several short labels required implementation context.", "Replaced them with explicit sentences and column names for paired outcomes, failure causes, scope, and limitations.", "Calculation rules are unchanged; the report is easier to understand."], + ["Failure causes", "The 75 failure causes were spread across a large card grid.", "Grouped them into a table covering agent, task or evaluator, test site, browser evidence, shared environment, and mixed causes, while retaining links to every reviewed result.", "Each failure is still counted once, and its reviewed cause and evidence remain unchanged."], + ["Checks and publication", "Routine edits also launched the complete browser-based visual suite.", "Routine checks now cover data, structure, links, and copy; packaging still runs every language, theme, and viewport visual check.", "Release quality gates remain intact while ordinary iteration is faster."] + ] + }} }, { "id": "2026-09-11-astra-two-model-browser-comparison", @@ -789,44 +721,42 @@ "读者可重算累计值；HAR、Playwright trace、Cookie 和凭据继续不公开。" ] ], - "i18n": { - "en": { - "title": "Add GPT-6 Astra and complete a two-model browser comparison", - "summary": "Chromium and Moli are compared within two model layers on the same 90 tasks: 73/90 versus 70/90 with GPT-5.6 Sol, and 71/90 versus 71/90 with GPT-6 Astra. All 75 final failures have evidence-bound root-cause reviews; verified environment contamination does not manually change official scores.", - "changes": [ - [ - "Two model layers", - "The previous release contained only GPT-5.6 Sol, so it could not show whether browser differences persisted after changing the model.", - "Added an independent GPT-6 Astra low evidence layer. Models are not pooled; browsers are paired task by task only within each model layer.", - "In the Astra layer Chromium and Moli both scored 71/90, with two browser-only passes on each side and exact McNemar p=1.0; no significant pass-rate difference was observed." - ], - [ - "Cross-model boundary", - "The model layers were separate runs and used different browser-eval commits.", - "The report retains each layer's run ID, commit, manifest, and attempts hashes and labels cross-model differences as observational.", - "Sol–Astra score or latency changes cannot be attributed to the model version alone." - ], - [ - "Failure causes", - "Official evaluator messages identify unmet rules but not the responsible layer.", - "Reviewed the contract, final answer, network evidence, and autonomous trajectory for all 75 failed logical results. Automatic classification remains available, while detailed causes stay blank until verified.", - "All 75 failures are reviewed with none unresolved, spanning Agent, site, environment isolation, evaluator contracts, and browser evidence capture." - ], - [ - "Environment isolation and parallelism", - "The shared fixed-port site stack allowed another process to mutate state after reset and before the Agent observed it.", - "Added a cross-process lock across reset, browser session, CowAgent, evidence finalization, and evaluation. Parallel execution is allowed only with separate full site stacks.", - "The fix protects future runs. Historical Task 77 scores remain unchanged until replaced by a new evidence cohort." - ], - [ - "Public evidence", - "Logical results accumulated every execution's latency and cost, but the previous page exposed evidence only for the score-authoritative execution.", - "The report still shows one logical result per model, browser, and task, while publishing sanitized responses, evaluator output, events, trajectories, metrics, and hashes for all 438 physical executions.", - "Readers can recompute aggregates; HAR, Playwright trace, cookies, and credentials remain private." - ] + "i18n": {"en": { + "title": "Add GPT-6 Astra and complete a two-model browser comparison", + "summary": "Chromium and Moli are compared within two model layers on the same 90 tasks: 73/90 versus 70/90 with GPT-5.6 Sol, and 71/90 versus 71/90 with GPT-6 Astra. All 75 final failures have evidence-bound root-cause reviews; verified environment contamination does not manually change official scores.", + "changes": [ + [ + "Two model layers", + "The previous release contained only GPT-5.6 Sol, so it could not show whether browser differences persisted after changing the model.", + "Added an independent GPT-6 Astra low evidence layer. Models are not pooled; browsers are paired task by task only within each model layer.", + "In the Astra layer Chromium and Moli both scored 71/90, with two browser-only passes on each side and exact McNemar p=1.0; no significant pass-rate difference was observed." + ], + [ + "Cross-model boundary", + "The model layers were separate runs and used different browser-eval commits.", + "The report retains each layer's run ID, commit, manifest, and attempts hashes and labels cross-model differences as observational.", + "Sol–Astra score or latency changes cannot be attributed to the model version alone." + ], + [ + "Failure causes", + "Official evaluator messages identify unmet rules but not the responsible layer.", + "Reviewed the contract, final answer, network evidence, and autonomous trajectory for all 75 failed logical results. Automatic classification remains available, while detailed causes stay blank until verified.", + "All 75 failures are reviewed with none unresolved, spanning Agent, site, environment isolation, evaluator contracts, and browser evidence capture." + ], + [ + "Environment isolation and parallelism", + "The shared fixed-port site stack allowed another process to mutate state after reset and before the Agent observed it.", + "Added a cross-process lock across reset, browser session, CowAgent, evidence finalization, and evaluation. Parallel execution is allowed only with separate full site stacks.", + "The fix protects future runs. Historical Task 77 scores remain unchanged until replaced by a new evidence cohort." + ], + [ + "Public evidence", + "Logical results accumulated every execution's latency and cost, but the previous page exposed evidence only for the score-authoritative execution.", + "The report still shows one logical result per model, browser, and task, while publishing sanitized responses, evaluator output, events, trajectories, metrics, and hashes for all 438 physical executions.", + "Readers can recompute aggregates; HAR, Playwright trace, cookies, and credentials remain private." ] - } - } + ] + }} }, { "id": "2026-09-11-90-task-browser-use-comparison", @@ -861,38 +791,16 @@ "支持确认本批确在自托管站点执行；不代表 Wikipedia 与 Map 已部署。" ] ], - "i18n": { - "en": { - "title": "Expand Browser Use to 90 tasks", - "summary": "The preregistered cohort now contains 90 tasks: 15 easy, 30 medium, and 45 hard. Chromium passed 73 and Moli passed 70. All 37 final failures have evidence-bound root-cause reviews; human review does not override official scores.", - "changes": [ - [ - "Evaluation scope", - "The previous 30-task cohort did not provide enough observations across short, medium, and long workflows.", - "Expanded to 90 outcome-blind tasks, preregistered by minimum visible UI actions as 15 easy, 30 medium, and 45 hard across Shopping, Shopping Admin, Reddit, and GitLab.", - "The denominator increased from 30 to 90. Tiers describe project-defined operation length, not official difficulty." - ], - [ - "Official results", - "The previous cohort gave Chromium and Moli 25/30 each.", - "This cohort gives Chromium 73/90 and Moli 70/90: 69 joint passes, 16 joint failures, four Chromium-only passes, and one Moli-only pass.", - "Results apply only to the current CowAgent, gpt-5.6-sol low, four local sites, and frozen run protocol; they do not generalize to the full benchmark." - ], - [ - "Failure causes", - "The programmatic evaluator locates failed rules but cannot by itself identify the responsible layer.", - "Reviewed the contract, answer, network evidence, and autonomous trajectory for all 37 failed logical results and recorded a specific cause subtype and rationale.", - "All 37 failures are reviewed with none unresolved; human review does not change official scores." - ], - [ - "Environment evidence", - "The prior cohort lacked preparation receipts bound to each execution.", - "All 220 physical executions retain reset, readiness, application-health, and sign-in preparation status, alongside the Colima gateway and local image topology.", - "This establishes that the cohort ran against self-hosted sites; it does not claim that Wikipedia or Map was deployed." - ] - ] - } - } + "i18n": {"en": { + "title": "Expand Browser Use to 90 tasks", + "summary": "The preregistered cohort now contains 90 tasks: 15 easy, 30 medium, and 45 hard. Chromium passed 73 and Moli passed 70. All 37 final failures have evidence-bound root-cause reviews; human review does not override official scores.", + "changes": [ + ["Evaluation scope", "The previous 30-task cohort did not provide enough observations across short, medium, and long workflows.", "Expanded to 90 outcome-blind tasks, preregistered by minimum visible UI actions as 15 easy, 30 medium, and 45 hard across Shopping, Shopping Admin, Reddit, and GitLab.", "The denominator increased from 30 to 90. Tiers describe project-defined operation length, not official difficulty."], + ["Official results", "The previous cohort gave Chromium and Moli 25/30 each.", "This cohort gives Chromium 73/90 and Moli 70/90: 69 joint passes, 16 joint failures, four Chromium-only passes, and one Moli-only pass.", "Results apply only to the current CowAgent, gpt-5.6-sol low, four local sites, and frozen run protocol; they do not generalize to the full benchmark."], + ["Failure causes", "The programmatic evaluator locates failed rules but cannot by itself identify the responsible layer.", "Reviewed the contract, answer, network evidence, and autonomous trajectory for all 37 failed logical results and recorded a specific cause subtype and rationale.", "All 37 failures are reviewed with none unresolved; human review does not change official scores."], + ["Environment evidence", "The prior cohort lacked preparation receipts bound to each execution.", "All 220 physical executions retain reset, readiness, application-health, and sign-in preparation status, alongside the Colima gateway and local image topology.", "This establishes that the cohort ran against self-hosted sites; it does not claim that Wikipedia or Map was deployed."] + ] + }} }, { "id": "2026-09-10-shared-failure-diagnosis-and-mac-expansion", @@ -939,50 +847,18 @@ "Wikipedia、Map 仍标为尚未实跑，不把理论可部署写成已验证。" ] ], - "i18n": { - "en": { - "title": "Trace failures to causes and apply runtime corrections", - "summary": "Ten failed logical results were reviewed across task contracts, agent protocol or policy, and evidence collection. Automatic failure location and reviewed causes are presented separately and bound to frozen evidence. Runtime corrections await validation by new executions; historical official scores remain 25/30 for each browser.", - "changes": [ - [ - "Report interpretation", - "Equal totals did not explain outcomes, and equal zero scores did not imply equal causes.", - "Made cause, owning layer, evidence chain, corrective action, and retest gate central to the report interpretation.", - "The explanatory hierarchy changed; scores, denominators, and frozen evidence did not." - ], - [ - "Shared failures", - "Earlier interpretation stopped at symptoms rather than locating the contract, policy, or collector responsible.", - "Used official contracts, final answers, evaluator output, HAR, and events to identify four cause classes, including browser-specific causes on Task 506.", - "No hidden model reasoning is inferred; historical official scores remain unchanged." - ], - [ - "Cause review", - "Automatic evaluators locate an unmet acceptance type but do not prove its cause.", - "Stored per-browser, per-task causes bound to the attempts hash; unreviewed fields remain blank and complete failure coverage is required for release.", - "All ten failed logical results in this version are reviewed; this does not change official scores." - ], - [ - "Runtime corrections", - "Invalid contracts and weaknesses in empty-result handling, candidate aggregation, HAR capture, and final-step handling could create avoidable failures or false negatives.", - "Added contract admission, candidate and pre-submit checks, immediate JSON HAR capture, fail-closed final-step handling, and a recorded step ceiling.", - "Code gates passed, but any task-outcome improvement still requires new execution evidence." - ], - [ - "Result details", - "Contracts, answers, and rule results were duplicated while raw JSON dominated the reading order.", - "Combined them into a task-and-acceptance section with readable conclusions first and technical values collapsed.", - "Public evidence and download links remain unchanged." - ], - [ - "Local environment and expansion", - "The Colima gateway and the scope of locally runnable official environments were unclear.", - "Distinguished VM and host gateway roles, documented the four measured sites, and corrected the available candidate counts.", - "Wikipedia and Map remain explicitly unmeasured; deployability is not presented as verification." - ] - ] - } - } + "i18n": {"en": { + "title": "Trace failures to causes and apply runtime corrections", + "summary": "Ten failed logical results were reviewed across task contracts, agent protocol or policy, and evidence collection. Automatic failure location and reviewed causes are presented separately and bound to frozen evidence. Runtime corrections await validation by new executions; historical official scores remain 25/30 for each browser.", + "changes": [ + ["Report interpretation", "Equal totals did not explain outcomes, and equal zero scores did not imply equal causes.", "Made cause, owning layer, evidence chain, corrective action, and retest gate central to the report interpretation.", "The explanatory hierarchy changed; scores, denominators, and frozen evidence did not."], + ["Shared failures", "Earlier interpretation stopped at symptoms rather than locating the contract, policy, or collector responsible.", "Used official contracts, final answers, evaluator output, HAR, and events to identify four cause classes, including browser-specific causes on Task 506.", "No hidden model reasoning is inferred; historical official scores remain unchanged."], + ["Cause review", "Automatic evaluators locate an unmet acceptance type but do not prove its cause.", "Stored per-browser, per-task causes bound to the attempts hash; unreviewed fields remain blank and complete failure coverage is required for release.", "All ten failed logical results in this version are reviewed; this does not change official scores."], + ["Runtime corrections", "Invalid contracts and weaknesses in empty-result handling, candidate aggregation, HAR capture, and final-step handling could create avoidable failures or false negatives.", "Added contract admission, candidate and pre-submit checks, immediate JSON HAR capture, fail-closed final-step handling, and a recorded step ceiling.", "Code gates passed, but any task-outcome improvement still requires new execution evidence."], + ["Result details", "Contracts, answers, and rule results were duplicated while raw JSON dominated the reading order.", "Combined them into a task-and-acceptance section with readable conclusions first and technical values collapsed.", "Public evidence and download links remain unchanged."], + ["Local environment and expansion", "The Colima gateway and the scope of locally runnable official environments were unclear.", "Distinguished VM and host gateway roles, documented the four measured sites, and corrected the available candidate counts.", "Wikipedia and Map remain explicitly unmeasured; deployability is not presented as verification."] + ] + }} }, { "id": "2026-09-10-track-naming-and-release-hardening", @@ -1011,32 +887,15 @@ "新增 104 个渲染状态的结构性视觉门禁；不改变报告数据。" ] ], - "i18n": { - "en": { - "title": "Standardize track names and strengthen release checks", - "summary": "The autonomous track is Browser Use and the fixed-action track is Browser Trajectory Replay. Winner marks, English accessibility text, filter recovery, asset format, and responsive layout now have automated gates. Scores, denominators, and evidence are unchanged.", - "changes": [ - [ - "Track names", - "Earlier names mixed implementation layers with evaluation subjects.", - "Renamed the tracks Browser Use and Browser Trajectory Replay across code, commands, configuration, and publication paths without compatibility aliases.", - "Only names and entry points changed; tasks, runs, and scoring did not." - ], - [ - "Comparison and localization", - "Ties or display-equal values could be marked as winners, and some English attributes retained Chinese.", - "Mark only uniquely better displayed values; localize titles and accessibility attributes; recover invalid browser filters to a valid default.", - "Reader interface defects were corrected without changing measurements or scores." - ], - [ - "Regression gates", - "Naming, graphics, and responsive layout relied mainly on manual memory.", - "Added retired-name checks, SVG rejection, and bilingual light/dark desktop and narrow-screen rendering checks.", - "The 104 structural render states do not change report data." - ] - ] - } - } + "i18n": {"en": { + "title": "Standardize track names and strengthen release checks", + "summary": "The autonomous track is Browser Use and the fixed-action track is Browser Trajectory Replay. Winner marks, English accessibility text, filter recovery, asset format, and responsive layout now have automated gates. Scores, denominators, and evidence are unchanged.", + "changes": [ + ["Track names", "Earlier names mixed implementation layers with evaluation subjects.", "Renamed the tracks Browser Use and Browser Trajectory Replay across code, commands, configuration, and publication paths without compatibility aliases.", "Only names and entry points changed; tasks, runs, and scoring did not."], + ["Comparison and localization", "Ties or display-equal values could be marked as winners, and some English attributes retained Chinese.", "Mark only uniquely better displayed values; localize titles and accessibility attributes; recover invalid browser filters to a valid default.", "Reader interface defects were corrected without changing measurements or scores."], + ["Regression gates", "Naming, graphics, and responsive layout relied mainly on manual memory.", "Added retired-name checks, SVG rejection, and bilingual light/dark desktop and narrow-screen rendering checks.", "The 104 structural render states do not change report data."] + ] + }} }, { "id": "2026-09-10-comparison-and-review-presentation", @@ -1077,44 +936,17 @@ "修正比较解释与页面层级，不改变测量值、判分或证据。" ] ], - "i18n": { - "en": { - "title": "Standardize comparison statistics and scalable browser comparison", - "summary": "Comparison summaries now use mean, P50, and P95. Same-task comparison uses a baseline plus multiple comparison browsers. Action latency ranks mean per call only, and detailed evaluator evidence no longer appears in KPI cards. Both official scores remain 25/30.", - "changes": [ - [ - "Summary statistics", - "Latency and token summaries lacked a consistent central and tail structure.", - "Show mean, P50, and P95 for each metric without duplicating P50 under another name.", - "Only presentation changed; scores, denominators, and evidence did not." - ], - [ - "Comparison presentation", - "Readers had to infer metric direction cell by cell.", - "Mark higher passes and pass rates and lower latency and token usage as better; keep ties and call counts neutral.", - "Winner marks are descriptive and do not change task verdicts." - ], - [ - "Page responsibilities", - "A failure list duplicated result filters, and review history contained a per-result inventory.", - "Keep pass/fail filters and details in Results; keep only review rounds, corrections, and remaining items in Review.", - "Duplicate presentation was removed without deleting evidence." - ], - [ - "Multi-browser comparison", - "The matrix widened per browser and its filter could not compare one baseline with several browsers.", - "Use one baseline, multi-select comparison browsers, explicit outcome relations, and linked pass/fail browser groups.", - "The interface can add browsers without changing current results." - ], - [ - "Action latency and result details", - "Action cells mixed three metrics and ranked totals despite unequal call counts; long errors disrupted result summaries.", - "Use one card per action, rank only mean per call, keep totals and counts neutral, and retain full diagnostics in rule evidence.", - "Interpretation and hierarchy changed; measurements, verdicts, and evidence did not." - ] - ] - } - } + "i18n": {"en": { + "title": "Standardize comparison statistics and scalable browser comparison", + "summary": "Comparison summaries now use mean, P50, and P95. Same-task comparison uses a baseline plus multiple comparison browsers. Action latency ranks mean per call only, and detailed evaluator evidence no longer appears in KPI cards. Both official scores remain 25/30.", + "changes": [ + ["Summary statistics", "Latency and token summaries lacked a consistent central and tail structure.", "Show mean, P50, and P95 for each metric without duplicating P50 under another name.", "Only presentation changed; scores, denominators, and evidence did not."], + ["Comparison presentation", "Readers had to infer metric direction cell by cell.", "Mark higher passes and pass rates and lower latency and token usage as better; keep ties and call counts neutral.", "Winner marks are descriptive and do not change task verdicts."], + ["Page responsibilities", "A failure list duplicated result filters, and review history contained a per-result inventory.", "Keep pass/fail filters and details in Results; keep only review rounds, corrections, and remaining items in Review.", "Duplicate presentation was removed without deleting evidence."], + ["Multi-browser comparison", "The matrix widened per browser and its filter could not compare one baseline with several browsers.", "Use one baseline, multi-select comparison browsers, explicit outcome relations, and linked pass/fail browser groups.", "The interface can add browsers without changing current results."], + ["Action latency and result details", "Action cells mixed three metrics and ranked totals despite unequal call counts; long errors disrupted result summaries.", "Use one card per action, rank only mean per call, keep totals and counts neutral, and retain full diagnostics in rule evidence.", "Interpretation and hierarchy changed; measurements, verdicts, and evidence did not."] + ] + }} }, { "id": "2026-09-10-environment-timing-and-report-model-correction", @@ -1155,44 +987,17 @@ "降低结论强度；不改变 evaluator 分数。" ] ], - "i18n": { - "en": { - "title": "Correct deployment evidence, timing boundaries, and report responsibilities", - "summary": "The report documents the four WebArena-Verified sites deployed in Colima/Docker and the private host gateway used by browser containers. Dataset pages contain definitions only; result pages add agent end-to-end and BrowserTool latency. Both scores remain 25/30.", - "changes": [ - [ - "Frozen protocol", - "The renderer chose the last execution even though the frozen manifest made the primary run authoritative.", - "Read result authority from the frozen manifest and restore the matching result, answer, and trajectory.", - "Neither browser recovered on later executions, so scores remain 25/30; only result identity and evidence semantics changed." - ], - [ - "Deployment environment", - "The diagram did not locate the four sites, explain the private gateway, or disclose missing preparation receipts.", - "Document the macOS, Colima/Docker, site-container, and browser-container topology, ports, image identities, and evidence boundary.", - "The 30 tasks cover four sites only; Wikipedia and Map are neither deployed nor scored. No results were rerun." - ], - [ - "Timing boundary", - "Earlier wording incorrectly included site reset in end-to-end latency and did not separate browser-tool time.", - "Exclude the preceding site reset, aggregate BrowserTool wall time from events, and retain phase timings for future runs.", - "Only diagnostics and wording changed; BrowserTool wall time still includes more than browser-core CPU time." - ], - [ - "Report model", - "Dataset pages mixed outcomes with definitions, result tables assumed two browsers, and participants were unclear.", - "Keep contracts in Dataset, render browsers dynamically in Results, and distinguish browser participants from shared controls.", - "Information ownership and extensibility changed; current data and scores did not." - ], - [ - "Review boundary", - "Evidence-integrity checks could be mistaken for independent semantic review.", - "Show evidence integrity and semantic-review coverage separately and remove population-style confidence intervals from the fixed cohort.", - "Claims were narrowed without changing evaluator scores." - ] - ] - } - } + "i18n": {"en": { + "title": "Correct deployment evidence, timing boundaries, and report responsibilities", + "summary": "The report documents the four WebArena-Verified sites deployed in Colima/Docker and the private host gateway used by browser containers. Dataset pages contain definitions only; result pages add agent end-to-end and BrowserTool latency. Both scores remain 25/30.", + "changes": [ + ["Frozen protocol", "The renderer chose the last execution even though the frozen manifest made the primary run authoritative.", "Read result authority from the frozen manifest and restore the matching result, answer, and trajectory.", "Neither browser recovered on later executions, so scores remain 25/30; only result identity and evidence semantics changed."], + ["Deployment environment", "The diagram did not locate the four sites, explain the private gateway, or disclose missing preparation receipts.", "Document the macOS, Colima/Docker, site-container, and browser-container topology, ports, image identities, and evidence boundary.", "The 30 tasks cover four sites only; Wikipedia and Map are neither deployed nor scored. No results were rerun."], + ["Timing boundary", "Earlier wording incorrectly included site reset in end-to-end latency and did not separate browser-tool time.", "Exclude the preceding site reset, aggregate BrowserTool wall time from events, and retain phase timings for future runs.", "Only diagnostics and wording changed; BrowserTool wall time still includes more than browser-core CPU time."], + ["Report model", "Dataset pages mixed outcomes with definitions, result tables assumed two browsers, and participants were unclear.", "Keep contracts in Dataset, render browsers dynamically in Results, and distinguish browser participants from shared controls.", "Information ownership and extensibility changed; current data and scores did not."], + ["Review boundary", "Evidence-integrity checks could be mistaken for independent semantic review.", "Show evidence integrity and semantic-review coverage separately and remove population-style confidence intervals from the fixed cohort.", "Claims were narrowed without changing evaluator scores."] + ] + }} }, { "id": "2026-09-10-logical-results-and-publication-qa", @@ -1227,38 +1032,16 @@ "后续报告若重新出现这些已知问题，将在发布前检查中失败。" ] ], - "i18n": { - "en": { - "title": "Unify logical-result semantics and complete report readability corrections", - "summary": "The public report now exposes one logical result per task and browser, with all physical-execution latency and model usage accumulated. Scores remain 25/30 for both browsers. Dense tables, code contrast, diagrams, narrow layouts, and release history now share one publication gate.", - "changes": [ - [ - "Logical results", - "Physical executions were presented as separate headline results.", - "Publish one logical result per task and browser; let the final allowed execution decide the outcome while accumulating every execution's cost.", - "Scores and denominators remain unchanged and physical evidence is retained." - ], - [ - "Readability", - "Short table headers wrapped vertically, code contrast was weak, and the text-heavy diagram was unreadable at normal size.", - "Add semantic column widths and scrolling, correct theme contrast, replace the diagram with a readable raster asset, and inspect desktop and narrow layouts.", - "Only presentation changed." - ], - [ - "Release history", - "The page lacked a current summary, compact archive, effect boundaries, and a structured download.", - "Use a report-style current release, expandable details, compact history, and structured data.", - "Release history became part of the report without changing evidence or results." - ], - [ - "Repeatable publication gate", - "Manual memory could not reliably prevent known regressions.", - "Encode logical-result, privacy, table, contrast, responsive, and changelog requirements in automated release checks.", - "Known regressions now fail before publication." - ] - ] - } - } + "i18n": {"en": { + "title": "Unify logical-result semantics and complete report readability corrections", + "summary": "The public report now exposes one logical result per task and browser, with all physical-execution latency and model usage accumulated. Scores remain 25/30 for both browsers. Dense tables, code contrast, diagrams, narrow layouts, and release history now share one publication gate.", + "changes": [ + ["Logical results", "Physical executions were presented as separate headline results.", "Publish one logical result per task and browser; let the final allowed execution decide the outcome while accumulating every execution's cost.", "Scores and denominators remain unchanged and physical evidence is retained."], + ["Readability", "Short table headers wrapped vertically, code contrast was weak, and the text-heavy diagram was unreadable at normal size.", "Add semantic column widths and scrolling, correct theme contrast, replace the diagram with a readable raster asset, and inspect desktop and narrow layouts.", "Only presentation changed."], + ["Release history", "The page lacked a current summary, compact archive, effect boundaries, and a structured download.", "Use a report-style current release, expandable details, compact history, and structured data.", "Release history became part of the report without changing evidence or results."], + ["Repeatable publication gate", "Manual memory could not reliably prevent known regressions.", "Encode logical-result, privacy, table, contrast, responsive, and changelog requirements in automated release checks.", "Known regressions now fail before publication."] + ] + }} }, { "id": "2026-09-10-deployment-and-autonomy-evidence", @@ -1281,26 +1064,14 @@ "既有测量不重跑；新增说明直接绑定现有 manifest、事件、HAR 与 trace 哈希。" ] ], - "i18n": { - "en": { - "title": "Complete deployment and autonomous-execution evidence", - "summary": "The title now identifies Chromium and Moli as the Browser Use participants. Methodology adds the full path from local WebArena-Verified deployment and per-execution reset through the CowAgent loop, official acceptance, and retained evidence. Measurements are unchanged.", - "changes": [ - [ - "Report title", - "The earlier title foregrounded a control variable rather than the evaluated browsers.", - "Make Chromium and Moli the title subjects and present CowAgent, model, reasoning, and tasks as conditions.", - "Narrative hierarchy changed; design and results did not." - ], - [ - "Deployment and autonomy evidence", - "Reset and autonomous planning were asserted without showing deployment ownership or the agent loop.", - "Add a bilingual flow and evidence table linking site startup, health checks, sign-in, and dynamic BrowserTool calls.", - "No measurements were rerun; the explanation binds to existing manifests, events, HAR, and trace hashes." - ] - ] - } - } + "i18n": {"en": { + "title": "Complete deployment and autonomous-execution evidence", + "summary": "The title now identifies Chromium and Moli as the Browser Use participants. Methodology adds the full path from local WebArena-Verified deployment and per-execution reset through the CowAgent loop, official acceptance, and retained evidence. Measurements are unchanged.", + "changes": [ + ["Report title", "The earlier title foregrounded a control variable rather than the evaluated browsers.", "Make Chromium and Moli the title subjects and present CowAgent, model, reasoning, and tasks as conditions.", "Narrative hierarchy changed; design and results did not."], + ["Deployment and autonomy evidence", "Reset and autonomous planning were asserted without showing deployment ownership or the agent loop.", "Add a bilingual flow and evidence table linking site startup, health checks, sign-in, and dynamic BrowserTool calls.", "No measurements were rerun; the explanation binds to existing manifests, events, HAR, and trace hashes."] + ] + }} }, { "id": "2026-09-10-report-evidence-restructure", @@ -1335,38 +1106,16 @@ "缺失或错配 selection、模型、reasoning、官方数据或证据审计时禁止发布。" ] ], - "i18n": { - "en": { - "title": "Restructure the Browser Use report around the evidence chain", - "summary": "The report adds task-family coverage, Wilson intervals, 30 paired-task comparisons, and route-for-route English pages. It correctly labels deterministic evidence-integrity review and avoids attributing five shared failures without a successful path. Measurements are unchanged.", - "changes": [ - [ - "Home and results", - "The home page did not jointly expose cohort coverage, uncertainty, call completeness, and paired outcomes.", - "Add task families, Wilson intervals, solvability coverage, evidence-audit status, and a filterable 30-task Chromium/Moli comparison.", - "Only explanation and drill-down changed; 25/30 scores and 30 pairs did not." - ], - [ - "Review terminology and attribution", - "Deterministic consistency checks were called independent result review, and shared failures were attributed too early.", - "Call the process evidence-integrity review and require separate non-scoring solvability review for tasks without a successful trajectory.", - "The official evaluator remains the sole scoring authority; shared failures are no longer presented as confirmed causes." - ], - [ - "Bilingual and mobile", - "English was summary-only and fixed card spans clipped narrow layouts.", - "Generate an English counterpart for every Chinese route, verify links, and force cards to one column on narrow screens.", - "Both languages use the same measured data; all 206 logical pages had valid links." - ], - [ - "Build and publication gates", - "A formal build could silently degrade without official data, while routine checks did not require complete evidence.", - "Bind builds to the official checksum and current publication profile, require complete evidence, and replace output atomically with a marker.", - "Publication fails on missing or mismatched selection, model, reasoning, official data, or evidence review." - ] - ] - } - } + "i18n": {"en": { + "title": "Restructure the Browser Use report around the evidence chain", + "summary": "The report adds task-family coverage, Wilson intervals, 30 paired-task comparisons, and route-for-route English pages. It correctly labels deterministic evidence-integrity review and avoids attributing five shared failures without a successful path. Measurements are unchanged.", + "changes": [ + ["Home and results", "The home page did not jointly expose cohort coverage, uncertainty, call completeness, and paired outcomes.", "Add task families, Wilson intervals, solvability coverage, evidence-audit status, and a filterable 30-task Chromium/Moli comparison.", "Only explanation and drill-down changed; 25/30 scores and 30 pairs did not."], + ["Review terminology and attribution", "Deterministic consistency checks were called independent result review, and shared failures were attributed too early.", "Call the process evidence-integrity review and require separate non-scoring solvability review for tasks without a successful trajectory.", "The official evaluator remains the sole scoring authority; shared failures are no longer presented as confirmed causes."], + ["Bilingual and mobile", "English was summary-only and fixed card spans clipped narrow layouts.", "Generate an English counterpart for every Chinese route, verify links, and force cards to one column on narrow screens.", "Both languages use the same measured data; all 206 logical pages had valid links."], + ["Build and publication gates", "A formal build could silently degrade without official data, while routine checks did not require complete evidence.", "Bind builds to the official checksum and current publication profile, require complete evidence, and replace output atomically with a marker.", "Publication fails on missing or mismatched selection, model, reasoning, official data, or evidence review."] + ] + }} }, { "id": "2026-09-09-webarena-verified-pilot", @@ -1413,50 +1162,18 @@ "本轮是 GPT-5.6 条件下的正式对比，不能写成 Astra 成绩；Astra 获权后必须新建独立运行。" ] ], - "i18n": { - "en": { - "title": "Complete the WebArena-Verified autonomous-agent browser comparison", - "summary": "CowAgent with gpt-5.6-sol at reasoning=low produced 60 logical results across 30 preregistered tasks in Chromium and Moli. Each browser passed 25 tasks. Deterministic evidence-integrity checks covered every result with no unresolved inconsistency.", - "changes": [ - [ - "Evidence and interpretation", - "Navigation alone did not make task selection, executions, review coverage, solvability, or conclusions auditable.", - "Use the home page as a decision entry point and publish contracts, measured results, assertions, trajectories, sanitized events, an English edition, and an evidence index.", - "Publication structure changed; tasks, denominator, outputs, scores, and evidence bindings did not." - ], - [ - "Autonomous-agent track", - "The repository also contained a trajectory-replay harness, making the phrase 'the original 30 tasks' ambiguous.", - "State that CowAgent autonomously observes, plans, and acts for this report, while trajectory replay is a separate microbenchmark.", - "Only the autonomous-agent track contributes to these scores." - ], - [ - "Benchmark selection", - "Browser-use, computer-use, offline action prediction, and agentic-search claims used incompatible evaluation units.", - "Compare primary-source adoption and evaluation contracts across relevant benchmark families.", - "Choose self-hosted WebArena-Verified for deterministic outcome evaluation; live-site and model-judge tracks remain future work." - ], - [ - "Complete formal run", - "The cohort needed short, medium, and long workflows without post-run task selection.", - "Preregister 5/10/15 operation-length tiers and one logical result per task-browser pair, with at most one declared execution after failure.", - "Both browsers passed 25/30; five tasks failed jointly and none produced a one-browser-only pass." - ], - [ - "Evidence integrity", - "Scores needed traceability to final answers, network side effects, and official evaluators without manual overrides.", - "Retain responses, events, HAR, traces, and evaluator details and verify files, hashes, scores, statuses, and transport attempts.", - "All logical results passed evidence-integrity checks with no scoring disagreement or unresolved item; sensitive network bodies remain private." - ], - [ - "Model-coverage boundary", - "Availability in a model catalog did not establish access for this run.", - "Record gpt-5.6-sol as the only measured model after Astra access was unavailable.", - "This is a GPT-5.6 comparison, not an Astra result; Astra requires a separate run when available." - ] - ] - } - } + "i18n": {"en": { + "title": "Complete the WebArena-Verified autonomous-agent browser comparison", + "summary": "CowAgent with gpt-5.6-sol at reasoning=low produced 60 logical results across 30 preregistered tasks in Chromium and Moli. Each browser passed 25 tasks. Deterministic evidence-integrity checks covered every result with no unresolved inconsistency.", + "changes": [ + ["Evidence and interpretation", "Navigation alone did not make task selection, executions, review coverage, solvability, or conclusions auditable.", "Use the home page as a decision entry point and publish contracts, measured results, assertions, trajectories, sanitized events, an English edition, and an evidence index.", "Publication structure changed; tasks, denominator, outputs, scores, and evidence bindings did not."], + ["Autonomous-agent track", "The repository also contained a trajectory-replay harness, making the phrase 'the original 30 tasks' ambiguous.", "State that CowAgent autonomously observes, plans, and acts for this report, while trajectory replay is a separate microbenchmark.", "Only the autonomous-agent track contributes to these scores."], + ["Benchmark selection", "Browser-use, computer-use, offline action prediction, and agentic-search claims used incompatible evaluation units.", "Compare primary-source adoption and evaluation contracts across relevant benchmark families.", "Choose self-hosted WebArena-Verified for deterministic outcome evaluation; live-site and model-judge tracks remain future work."], + ["Complete formal run", "The cohort needed short, medium, and long workflows without post-run task selection.", "Preregister 5/10/15 operation-length tiers and one logical result per task-browser pair, with at most one declared execution after failure.", "Both browsers passed 25/30; five tasks failed jointly and none produced a one-browser-only pass."], + ["Evidence integrity", "Scores needed traceability to final answers, network side effects, and official evaluators without manual overrides.", "Retain responses, events, HAR, traces, and evaluator details and verify files, hashes, scores, statuses, and transport attempts.", "All logical results passed evidence-integrity checks with no scoring disagreement or unresolved item; sensitive network bodies remain private."], + ["Model-coverage boundary", "Availability in a model catalog did not establish access for this run.", "Record gpt-5.6-sol as the only measured model after Astra access was unavailable.", "This is a GPT-5.6 comparison, not an Astra result; Astra requires a separate run when available."] + ] + }} } ], "review_rounds": [ @@ -1724,97 +1441,98 @@ } } }, - { - "id": "2026-10-10-five-browsers-seven-families-430", - "publication_state": "published", - "scoring_revision_input_sha256": [ - "7a79357747f8a0a9bdf9d216667edfeaf3bedf0ff25a3685ad29f17abfa1d712", - "884edf8ca3f9111aea956f9635688f9c7af6492ac32a1966f4275c4d9f19dd21" - ], - "date": "2026-10-10", - "comparison_kinds": [ - "headless_markdown", - "headless_html" - ], - "title": "发布五种浏览器、七类任务的 430 题报告", - "summary": "每种浏览器完成 430 题，共 2,150 个结果。Lightpanda 通过 243 题，Moli 233 题，Chromium 199 题，Obscura 192 题，Kitesurf 171 题。新增风控网页正文专项；动态内容与热门站点各覆盖 100 题。", + +{ + "id": "2026-10-10-five-browsers-seven-families-430", + "publication_state": "published", + "scoring_revision_input_sha256": [ + "7a79357747f8a0a9bdf9d216667edfeaf3bedf0ff25a3685ad29f17abfa1d712", + "884edf8ca3f9111aea956f9635688f9c7af6492ac32a1966f4275c4d9f19dd21" + ], + "date": "2026-10-10", + "comparison_kinds": [ + "headless_markdown", + "headless_html" + ], + "title": "发布五种浏览器、七类任务的 430 题报告", + "summary": "每种浏览器完成 430 题，共 2,150 个结果。Lightpanda 通过 243 题，Moli 233 题，Chromium 199 题，Obscura 192 题，Kitesurf 171 题。新增风控网页正文专项；动态内容与热门站点各覆盖 100 题。", + "changes": [ + [ + "评测范围", + "此前未覆盖当前七类完整题集", + "普通网页、动态内容、热门站点、风控网页正文各 100 题，文档、图片、视频各 10 题", + "总分母为每种浏览器 430 题；旧报告保留。" + ], + [ + "文档和资源接口", + "部分调用将文档当网页、借用外部转换或未明确选择参评引擎", + "文档通过参评浏览器读取原始字节并共用正文解析；资源使用已加载页面与明确指定的引擎", + "替换全部受影响的调用，保留此前证据；题目、固定答案及评分规则不变。" + ], + [ + "耗时与资源", + "任务通过率无法说明获取耗时和资源开销", + "分别列出全部调用、通过任务的均值、P50、P95；保留 20 题四种本地浏览器的 80 次资源测量", + "提取与解析计入时限；托管 Kitesurf 的后端资源未公开。" + ], + [ + "原生接口与共同处理", + "正文格式和资源接口未完整区分，部分 CDP 调用混用了托管 API 的等待参数", + "展示七类任务与五种浏览器的 35 个接口组合；Kitesurf 资源使用官方 CDP 与共同 DOM 范围", + "Moli 全部 100 个热门站点和 Kitesurf 全部 19 个受影响任务重新执行，旧调用保留。后一次纠正使 Kitesurf 视频通过数从 5/10 改为 6/10，整体从 168/430 改为 169/430；分母不变，耗时采用修正后的完整调用。" + ], + [ + "共享评分器", + "Markdown 加号列表、相邻卡片和表头可能漏计已返回的内容", + "采用最新共享评分器重算全部保留输出，并用成对测试防止漏判和误判", + "四个动态任务结果由失败变为通过：Kitesurf 增加两例，Lightpanda 与 Obscura 各增加一例。未新增浏览器调用，题目、答案、输出和耗时均未改变。" + ] + ], + "i18n": { + "en": { + "title": "Five browsers across 430 tasks in seven families", + "summary": "All five browsers completed 430 tasks, yielding 2,150 results. Lightpanda passed 243, Moli 233, Chromium 199, Obscura 192, and Kitesurf 171. Risk-controlled webpage text is a new family; dynamic and popular-site tasks each cover 100 cases.", "changes": [ [ - "评测范围", - "此前未覆盖当前七类完整题集", - "普通网页、动态内容、热门站点、风控网页正文各 100 题，文档、图片、视频各 10 题", - "总分母为每种浏览器 430 题；旧报告保留。" + "Scope", + "The previous cohort did not cover the current seven-family matrix", + "100 cases each for ordinary, dynamic, popular and risk-controlled webpage text; 10 each for documents, images and video", + "430 cases per browser; the earlier report is preserved." ], [ - "文档和资源接口", - "部分调用将文档当网页、借用外部转换或未明确选择参评引擎", - "文档通过参评浏览器读取原始字节并共用正文解析；资源使用已加载页面与明确指定的引擎", - "替换全部受影响的调用，保留此前证据；题目、固定答案及评分规则不变。" + "Document and resource routes", + "Some calls treated documents as pages, used external conversion or omitted explicit engine selection", + "Browser-native original document bytes with shared text extraction; resources from loaded pages with explicit engine binding", + "All affected calls are replaced with prior evidence retained; tasks, fixed answers and scoring remain unchanged." ], [ - "耗时与资源", - "任务通过率无法说明获取耗时和资源开销", - "分别列出全部调用、通过任务的均值、P50、P95；保留 20 题四种本地浏览器的 80 次资源测量", - "提取与解析计入时限；托管 Kitesurf 的后端资源未公开。" + "Latency and resources", + "Task acceptance alone does not describe retrieval cost", + "All-call and successful-call mean, P50 and P95 latency; 80 retained resource calls across 20 cases and four local browsers", + "Extraction and parsing share the deadline; hosted Kitesurf backend resources are unavailable." ], [ - "原生接口与共同处理", - "正文格式和资源接口未完整区分，部分 CDP 调用混用了托管 API 的等待参数", - "展示七类任务与五种浏览器的 35 个接口组合；Kitesurf 资源使用官方 CDP 与共同 DOM 范围", - "Moli 全部 100 个热门站点和 Kitesurf 全部 19 个受影响任务重新执行，旧调用保留。后一次纠正使 Kitesurf 视频通过数从 5/10 改为 6/10，整体从 168/430 改为 169/430；分母不变，耗时采用修正后的完整调用。" + "Native interfaces and shared processing", + "Output modes and resource scope were conflated; some CDP calls received a hosted API completion value", + "Expose all 35 browser-family interface combinations; Kitesurf resources use official CDP with the same DOM scope", + "Rerun all 100 Moli popular-site tasks and all 19 affected Kitesurf tasks, preserving prior calls. The latter correction changes Kitesurf video passes from 5/10 to 6/10 and overall passes from 168/430 to 169/430. Denominators stay fixed; latency uses the corrected complete calls." ], [ - "共享评分器", - "Markdown 加号列表、相邻卡片和表头可能漏计已返回的内容", - "采用最新共享评分器重算全部保留输出，并用成对测试防止漏判和误判", - "四个动态任务结果由失败变为通过：Kitesurf 增加两例，Lightpanda 与 Obscura 各增加一例。未新增浏览器调用，题目、答案、输出和耗时均未改变。" + "Shared scoring", + "Markdown plus lists, adjacent cards and table headers could omit returned content", + "Rescore all retained outputs with the current shared scorer and paired controls against false positives and false negatives", + "Four dynamic results change from fail to pass: two for Kitesurf and one each for Lightpanda and Obscura. No browser calls are added; tasks, answers, outputs and latency remain unchanged." ] - ], - "i18n": { - "en": { - "title": "Five browsers across 430 tasks in seven families", - "summary": "All five browsers completed 430 tasks, yielding 2,150 results. Lightpanda passed 243, Moli 233, Chromium 199, Obscura 192, and Kitesurf 171. Risk-controlled webpage text is a new family; dynamic and popular-site tasks each cover 100 cases.", - "changes": [ - [ - "Scope", - "The previous cohort did not cover the current seven-family matrix", - "100 cases each for ordinary, dynamic, popular and risk-controlled webpage text; 10 each for documents, images and video", - "430 cases per browser; the earlier report is preserved." - ], - [ - "Document and resource routes", - "Some calls treated documents as pages, used external conversion or omitted explicit engine selection", - "Browser-native original document bytes with shared text extraction; resources from loaded pages with explicit engine binding", - "All affected calls are replaced with prior evidence retained; tasks, fixed answers and scoring remain unchanged." - ], - [ - "Latency and resources", - "Task acceptance alone does not describe retrieval cost", - "All-call and successful-call mean, P50 and P95 latency; 80 retained resource calls across 20 cases and four local browsers", - "Extraction and parsing share the deadline; hosted Kitesurf backend resources are unavailable." - ], - [ - "Native interfaces and shared processing", - "Output modes and resource scope were conflated; some CDP calls received a hosted API completion value", - "Expose all 35 browser-family interface combinations; Kitesurf resources use official CDP with the same DOM scope", - "Rerun all 100 Moli popular-site tasks and all 19 affected Kitesurf tasks, preserving prior calls. The latter correction changes Kitesurf video passes from 5/10 to 6/10 and overall passes from 168/430 to 169/430. Denominators stay fixed; latency uses the corrected complete calls." - ], - [ - "Shared scoring", - "Markdown plus lists, adjacent cards and table headers could omit returned content", - "Rescore all retained outputs with the current shared scorer and paired controls against false positives and false negatives", - "Four dynamic results change from fail to pass: two for Kitesurf and one each for Lightpanda and Obscura. No browser calls are added; tasks, answers, outputs and latency remain unchanged." - ] - ] - } - } - }, - { - "id": "2026-10-09-dynamic-1000-native-presentation", - "publication_state": "published", - "date": "2026-10-09", - "comparison_kinds": [ - "uniform_native_local_measurement" + ] + } + } +}, + { + "id": "2026-10-09-dynamic-1000-native-presentation", + "publication_state": "published", + "date": "2026-10-09", + "comparison_kinds": [ + "uniform_native_local_measurement" ], "title": "补清 1000 次原生调用的配置与复现说明", "summary": "十种方法的页面说明统一 45 秒外层期限与产品内部默认超时；接口示例展示同一次实测保存的请求和返回。报告说明、执行顺序及冻结评分版本得到补充，成绩与实测文件未变。", @@ -2038,125 +1756,47 @@ "id": "webmainbench-545-pr919-e944d066-lexmain-f7cb97d-20260929", "publication_state": "published", "date": "2026-09-29", - "comparison_kinds": [ - "webmainbench_frozen_html_to_markdown" - ], + "comparison_kinds": ["webmainbench_frozen_html_to_markdown"], "title": "补齐渐显正文并按最新 lexmain 重算七方法结果", "summary": "Moli PR919 提交 e944d066b 修复一次性前向渐显页面的正文缺失，以固定 release 二进制完成 545 页认证运行。七方法共 3,815 条结果按 WebMainBench lexmain f7cb97d 的参考与固定公共总体重算；PR919 目标正文 545/545，综合分 39.41%。", "changes": [ - [ - "Moli 正文交付", - "CNN 冻结页的正文以 opacity:0 开始，页面加载后由同名关键帧渐显；旧可见性快照将其误判为永久隐藏。", - "仅在最终关键帧明确恢复不透明度且动画为正时长、无正延迟、单次、正向、运行中并保持终态时保留正文；零时长、循环、反向和无关关键帧继续视为隐藏。", - "固定二进制 e146b853… 完成 545 页认证运行，CNN 正文恢复，PR919 目标正文为 545/545。" - ], - [ - "固定计分总体", - "先前报告仍使用公式 113 页、表格 81 页的旧抽取总体。", - "按 lexmain f7cb97d 的参考抽取结果固定为全文 545、代码 110、公式 257、表格内容与结构各 179 页；适用页面遗漏或调用失败计零。", - "七种方法全部使用同一总体；3,815 条既有输出离线重算，未新增 provider 调用。" - ], - [ - "当前结果", - "旧图和文案绑定较早的 PR919 二进制与评分器，不能说明当前提交。", - "报告绑定源码 e944d066b、release 二进制 e146b853…、评分器 f7cb97d，并保留原版 Moli、PR785 与 PR919 三行。", - "PR919 综合分 39.41%，全文 40.79%、代码 91.74%、公式 28.16%、表格内容 22.61%、表格结构 13.74%；综合分和代码为七方法唯一最高。" - ] + ["Moli 正文交付", "CNN 冻结页的正文以 opacity:0 开始，页面加载后由同名关键帧渐显；旧可见性快照将其误判为永久隐藏。", "仅在最终关键帧明确恢复不透明度且动画为正时长、无正延迟、单次、正向、运行中并保持终态时保留正文；零时长、循环、反向和无关关键帧继续视为隐藏。", "固定二进制 e146b853… 完成 545 页认证运行，CNN 正文恢复，PR919 目标正文为 545/545。"], + ["固定计分总体", "先前报告仍使用公式 113 页、表格 81 页的旧抽取总体。", "按 lexmain f7cb97d 的参考抽取结果固定为全文 545、代码 110、公式 257、表格内容与结构各 179 页；适用页面遗漏或调用失败计零。", "七种方法全部使用同一总体；3,815 条既有输出离线重算，未新增 provider 调用。"], + ["当前结果", "旧图和文案绑定较早的 PR919 二进制与评分器，不能说明当前提交。", "报告绑定源码 e944d066b、release 二进制 e146b853…、评分器 f7cb97d，并保留原版 Moli、PR785 与 PR919 三行。", "PR919 综合分 39.41%，全文 40.79%、代码 91.74%、公式 28.16%、表格内容 22.61%、表格结构 13.74%；综合分和代码为七方法唯一最高。"] ], - "i18n": { - "en": { - "title": "Restore fade-in content and rescore seven methods with current lexmain", - "summary": "Moli PR919 commit e944d066b fixes missing content on one-shot forward fade-in pages and completes a certified 545-page run with a pinned release binary. WebMainBench lexmain f7cb97d rescored all 3,815 results against reference-defined fixed populations; PR919 delivers 545/545 target bodies and scores 39.41% overall.", - "changes": [ - [ - "Moli body delivery", - "The frozen CNN page starts its body at opacity zero and reveals it with a named keyframe after load; the previous visibility snapshot treated it as permanently hidden.", - "Retain the body only when the final keyframe explicitly restores opacity and the animation has positive duration, no positive delay, one forward running iteration, and a retained final state. Zero-duration, looping, reverse, and unrelated animations remain hidden.", - "Pinned binary e146b853… completes the certified 545-page run, restores the CNN body, and delivers 545/545 target bodies." - ], - [ - "Fixed scoring populations", - "The prior report still used the older 113-page formula and 81-page table extraction populations.", - "Bind lexmain f7cb97d reference extraction to 545 text, 110 code, 257 formula, and 179 table-content and table-structure pages. Omissions and failed calls score zero within applicable populations.", - "All seven methods use the same populations; the 3,815 stored outputs are rescored offline with zero provider calls." - ], - [ - "Current result", - "Earlier figures and prose were bound to an older PR919 binary and scorer.", - "Bind the report to source e944d066b, release binary e146b853…, and scorer f7cb97d while keeping separate rows for original Moli, PR785, and PR919.", - "PR919 scores 39.41% overall, 40.79% text, 91.74% code, 28.16% formula, 22.61% table content, and 13.74% table structure; overall and code are the unique seven-method maxima." - ] - ] - } - } + "i18n": {"en": { + "title": "Restore fade-in content and rescore seven methods with current lexmain", + "summary": "Moli PR919 commit e944d066b fixes missing content on one-shot forward fade-in pages and completes a certified 545-page run with a pinned release binary. WebMainBench lexmain f7cb97d rescored all 3,815 results against reference-defined fixed populations; PR919 delivers 545/545 target bodies and scores 39.41% overall.", + "changes": [ + ["Moli body delivery", "The frozen CNN page starts its body at opacity zero and reveals it with a named keyframe after load; the previous visibility snapshot treated it as permanently hidden.", "Retain the body only when the final keyframe explicitly restores opacity and the animation has positive duration, no positive delay, one forward running iteration, and a retained final state. Zero-duration, looping, reverse, and unrelated animations remain hidden.", "Pinned binary e146b853… completes the certified 545-page run, restores the CNN body, and delivers 545/545 target bodies."], + ["Fixed scoring populations", "The prior report still used the older 113-page formula and 81-page table extraction populations.", "Bind lexmain f7cb97d reference extraction to 545 text, 110 code, 257 formula, and 179 table-content and table-structure pages. Omissions and failed calls score zero within applicable populations.", "All seven methods use the same populations; the 3,815 stored outputs are rescored offline with zero provider calls."], + ["Current result", "Earlier figures and prose were bound to an older PR919 binary and scorer.", "Bind the report to source e944d066b, release binary e146b853…, and scorer f7cb97d while keeping separate rows for original Moli, PR785, and PR919.", "PR919 scores 39.41% overall, 40.79% text, 91.74% code, 28.16% formula, 22.61% table content, and 13.74% table structure; overall and code are the unique seven-method maxima."] + ] + }} }, { "id": "webmainbench-545-lexmain-reference-and-final-pr919-20260928", "publication_state": "published", "date": "2026-09-28", - "comparison_kinds": [ - "webmainbench_frozen_html_to_markdown" - ], + "comparison_kinds": ["webmainbench_frozen_html_to_markdown"], "title": "修正 WebMainBench 参考、专项计分范围与 TEDS", "summary": "Lexmount WebMainBench lexmain 修正 TEDS 有序树计算、22 条参考或代码标注，并让七种方法在相同的参考定义专项页面上计分。PR919 提交 b7a860600 的 545 页认证输出保持不变；3,815 条结果离线重算后，修复版成功生成正文 545/545，综合分为 55.25%。", "changes": [ - [ - "评测基准", - "官方 TEDS 把序列化字符串交给 APTED；任务 f080fbec 的参考包含隐藏编辑器副本，另有 21 条代码参考或标注与冻结页面不一致。", - "在 Lexmount WebMainBench lexmain 对显式有序树计算 TEDS，按冻结页面修正参考与代码范围，并用哈希绑定每条数据修订。", - "TEDS、参考和代码范围均对全部 3,815 条冻结结果离线复算，没有新增 provider 调用。" - ], - [ - "统一计分范围", - "代码、公式和表格分母曾由每种方法的输出决定；漏输出、抽取失败或额外输出会让不同方法落入不同计分页集合。", - "专项指标改用参考答案定义的公共页面范围；参考适用但方法遗漏或调用失败时计零，参考无法抽取为可评分表示的页面记为不可用。", - "七种方法统一使用全文 545、代码 110、公式 113、表格内容和结构各 81 页。成对回归控制同时覆盖漏计和误计。" - ], - [ - "实现复审", - "上一候选把所有 pre 输出为围栏，并拆平任意含 pre 的表格；这提高代码提取分，却删除真实链接、行列和跨行关系。", - "保留含链接的 pre 与语义表格；布局表继续按已有规则展开，同时修正祖先渐变、只读 DOM 查询和原始 fragment 优先级。", - "相关行为测试通过；固定 release 二进制完成 545 页全量认证，545 次最终调用均成功。" - ], - [ - "分数解释", - "旧报告的 33.11% 来自方法相关分母和不完整代码提取，不能作为同范围七方法比较。", - "保留每份 provider 输出与官方原分，另存修正前指标，再用统一参考范围和修正评分器生成当前指标。", - "PR919 综合分 55.25%，代码 91.06%、公式 64.08%、表格内容 50.09%、表格结构 30.25%，成功生成正文 545/545。" - ] + ["评测基准", "官方 TEDS 把序列化字符串交给 APTED；任务 f080fbec 的参考包含隐藏编辑器副本，另有 21 条代码参考或标注与冻结页面不一致。", "在 Lexmount WebMainBench lexmain 对显式有序树计算 TEDS，按冻结页面修正参考与代码范围，并用哈希绑定每条数据修订。", "TEDS、参考和代码范围均对全部 3,815 条冻结结果离线复算，没有新增 provider 调用。"], + ["统一计分范围", "代码、公式和表格分母曾由每种方法的输出决定；漏输出、抽取失败或额外输出会让不同方法落入不同计分页集合。", "专项指标改用参考答案定义的公共页面范围；参考适用但方法遗漏或调用失败时计零，参考无法抽取为可评分表示的页面记为不可用。", "七种方法统一使用全文 545、代码 110、公式 113、表格内容和结构各 81 页。成对回归控制同时覆盖漏计和误计。"], + ["实现复审", "上一候选把所有 pre 输出为围栏，并拆平任意含 pre 的表格；这提高代码提取分，却删除真实链接、行列和跨行关系。", "保留含链接的 pre 与语义表格；布局表继续按已有规则展开，同时修正祖先渐变、只读 DOM 查询和原始 fragment 优先级。", "相关行为测试通过；固定 release 二进制完成 545 页全量认证，545 次最终调用均成功。"], + ["分数解释", "旧报告的 33.11% 来自方法相关分母和不完整代码提取，不能作为同范围七方法比较。", "保留每份 provider 输出与官方原分，另存修正前指标，再用统一参考范围和修正评分器生成当前指标。", "PR919 综合分 55.25%，代码 91.06%、公式 64.08%、表格内容 50.09%、表格结构 30.25%，成功生成正文 545/545。"] ], - "i18n": { - "en": { - "title": "Correct WebMainBench references, component populations, and TEDS", - "summary": "Lexmount WebMainBench lexmain corrects ordered-tree TEDS, 22 reference or code annotations, and uses the same reference-defined component pages for all seven methods. The certified 545-page PR919 outputs at commit b7a860600 are unchanged. Offline rescoring of all 3,815 results yields 545/545 nonempty bodies and a 55.25% overall score.", - "changes": [ - [ - "Evaluation basis", - "Official TEDS passed a serialized string to APTED; task f080fbec included a hidden editor buffer, and 21 other code references or annotations differed from the frozen page.", - "Compute TEDS over the explicit ordered tree, correct references and code scope against the frozen pages, and hash-bind each data correction in Lexmount WebMainBench lexmain.", - "TEDS, references, and code scope are recomputed over all 3,815 frozen results with zero provider calls." - ], - [ - "Shared scoring scope", - "Code, formula, and table denominators previously depended on each method output, so omissions, extraction failures, and extra output changed the scored population.", - "Use reference-defined component populations for every method. Missing output and failed calls score zero inside an applicable population; pages whose references cannot be extracted into a scoreable form are unavailable.", - "All seven methods now use 545 text, 110 code, 113 formula, and 81 table-content and table-structure pages. Paired controls cover both error directions." - ], - [ - "Implementation review", - "The prior candidate fenced every pre element and flattened every table containing pre content, improving code extraction while deleting real links, rows, and spans.", - "Preserve pre elements with links and semantic tables, retain existing layout-table handling, and correct ancestor gradients, read-only DOM lookups, and raw fragment priority.", - "Scenario tests pass and the fixed release binary completes all 545 certified calls successfully." - ], - [ - "Score interpretation", - "The former 33.11% value used method-dependent populations and incomplete code extraction, so it was not a same-scope seven-method comparison.", - "Preserve each provider output and original official score, retain pre-correction metrics, and derive current metrics with the corrected scorer and shared reference scope.", - "PR919 scores 55.25% overall, 91.06% code, 64.08% formula, 50.09% table content, and 30.25% table structure, with 545/545 nonempty bodies." - ] - ] - } - } + "i18n": {"en": { + "title": "Correct WebMainBench references, component populations, and TEDS", + "summary": "Lexmount WebMainBench lexmain corrects ordered-tree TEDS, 22 reference or code annotations, and uses the same reference-defined component pages for all seven methods. The certified 545-page PR919 outputs at commit b7a860600 are unchanged. Offline rescoring of all 3,815 results yields 545/545 nonempty bodies and a 55.25% overall score.", + "changes": [ + ["Evaluation basis", "Official TEDS passed a serialized string to APTED; task f080fbec included a hidden editor buffer, and 21 other code references or annotations differed from the frozen page.", "Compute TEDS over the explicit ordered tree, correct references and code scope against the frozen pages, and hash-bind each data correction in Lexmount WebMainBench lexmain.", "TEDS, references, and code scope are recomputed over all 3,815 frozen results with zero provider calls."], + ["Shared scoring scope", "Code, formula, and table denominators previously depended on each method output, so omissions, extraction failures, and extra output changed the scored population.", "Use reference-defined component populations for every method. Missing output and failed calls score zero inside an applicable population; pages whose references cannot be extracted into a scoreable form are unavailable.", "All seven methods now use 545 text, 110 code, 113 formula, and 81 table-content and table-structure pages. Paired controls cover both error directions."], + ["Implementation review", "The prior candidate fenced every pre element and flattened every table containing pre content, improving code extraction while deleting real links, rows, and spans.", "Preserve pre elements with links and semantic tables, retain existing layout-table handling, and correct ancestor gradients, read-only DOM lookups, and raw fragment priority.", "Scenario tests pass and the fixed release binary completes all 545 certified calls successfully."], + ["Score interpretation", "The former 33.11% value used method-dependent populations and incomplete code extraction, so it was not a same-scope seven-method comparison.", "Preserve each provider output and original official score, retain pre-correction metrics, and derive current metrics with the corrected scorer and shared reference scope.", "PR919 scores 55.25% overall, 91.06% code, 64.08% formula, 50.09% table content, and 30.25% table structure, with 545/545 nonempty bodies."] + ] + }} }, { "id": "webmainbench-545-moli-pr919-final-20260928", @@ -2217,58 +1857,24 @@ { "id": "2026-09-24-dynamic-100-moli-119-local-audit", "publication_state": "published", - "comparison_kinds": [ - "single_provider_dynamic_native_fetch" - ], + "comparison_kinds": ["single_provider_dynamic_native_fetch"], "date": "2026-09-24", "title": "完成100例动态网站 Moli 1.1.9 本地审核版", "summary": "合并两个已保存的原生 Fetch 批次，以同一评分实现离线核对100例；结果为44例通过、56例未通过。没有新增Moli调用，原始调用时间、输出和日志继续保留在持久证据目录。", "changes": [ - [ - "调用口径", - "旧的逐站等待与提取设置会替产品决定何时抓取。", - "全部100例使用Moli 1.1.9原生Fetch、默认done和统一参数；只有网址不同。", - "不同日期的两个批次合并展示，每例保留真实调用时间。" - ], - [ - "规则与评分", - "动态名单、表格和正文需要同一套可复核规则。", - "冻结100例当前规则，使用稳定文字、同条目字段、名称链接、数量门槛和正文区域的公共评分实现。", - "保存的原始返回离线重算仍为44/56，没有判定变化。" - ], - [ - "证据与报告", - "原始结果、debug日志和生成HTML不适合进入Git。", - "Git只保存案例定义、直接来源证据、公共评分与报告代码及哈希收据；运行产物留在外部持久证据目录。", - "报告显示Moli版本、失败分类、正文证据及完整物理调用链下钻。" - ] + ["调用口径", "旧的逐站等待与提取设置会替产品决定何时抓取。", "全部100例使用Moli 1.1.9原生Fetch、默认done和统一参数；只有网址不同。", "不同日期的两个批次合并展示，每例保留真实调用时间。"], + ["规则与评分", "动态名单、表格和正文需要同一套可复核规则。", "冻结100例当前规则，使用稳定文字、同条目字段、名称链接、数量门槛和正文区域的公共评分实现。", "保存的原始返回离线重算仍为44/56，没有判定变化。"], + ["证据与报告", "原始结果、debug日志和生成HTML不适合进入Git。", "Git只保存案例定义、直接来源证据、公共评分与报告代码及哈希收据；运行产物留在外部持久证据目录。", "报告显示Moli版本、失败分类、正文证据及完整物理调用链下钻。"] ], - "i18n": { - "en": { - "title": "Complete the local audit build for 100 dynamic sites with Moli 1.1.9", - "summary": "Two persisted native Fetch cohorts are combined and checked offline with one scoring implementation across 100 cases: 44 pass and 56 fail. No new Moli calls were made; original call times, outputs and logs remain in persistent evidence.", - "changes": [ - [ - "Call contract", - "Earlier per-site waits and extractors could decide retrieval timing for the product.", - "All 100 cases use Moli 1.1.9 native Fetch, default done completion and uniform parameters; only the URL changes.", - "The two dated cohorts are combined while each case retains its actual call time." - ], - [ - "Rules and scoring", - "Dynamic lists, tables and body text require one auditable rule set.", - "Freeze the current 100-case rules and use shared stable-text, same-record field, name-link, count-threshold and body-region scoring.", - "Offline rescoring of saved raw responses remains 44/56 with no verdict changes." - ], - [ - "Evidence and report", - "Raw results, debug logs and generated HTML do not belong in Git.", - "Git retains case definitions, direct source evidence, shared scoring and report code, and hash receipts; run artifacts stay in persistent evidence.", - "The report shows the Moli version, failure classes, body evidence and a drill-down through the complete physical-call chain." - ] - ] - } - } + "i18n": {"en": { + "title": "Complete the local audit build for 100 dynamic sites with Moli 1.1.9", + "summary": "Two persisted native Fetch cohorts are combined and checked offline with one scoring implementation across 100 cases: 44 pass and 56 fail. No new Moli calls were made; original call times, outputs and logs remain in persistent evidence.", + "changes": [ + ["Call contract", "Earlier per-site waits and extractors could decide retrieval timing for the product.", "All 100 cases use Moli 1.1.9 native Fetch, default done completion and uniform parameters; only the URL changes.", "The two dated cohorts are combined while each case retains its actual call time."], + ["Rules and scoring", "Dynamic lists, tables and body text require one auditable rule set.", "Freeze the current 100-case rules and use shared stable-text, same-record field, name-link, count-threshold and body-region scoring.", "Offline rescoring of saved raw responses remains 44/56 with no verdict changes."], + ["Evidence and report", "Raw results, debug logs and generated HTML do not belong in Git.", "Git retains case definitions, direct source evidence, shared scoring and report code, and hash receipts; run artifacts stay in persistent evidence.", "The report shows the Moli version, failure classes, body evidence and a drill-down through the complete physical-call chain."] + ] + }} }, { "id": "webmainbench-545-table-controls-and-moli-provenance-20260928", @@ -2461,366 +2067,130 @@ { "id": "2026-09-24-webmainbench-target-body-population", "publication_state": "published", - "comparison_kinds": [ - "webmainbench_frozen_html_to_markdown" - ], + "comparison_kinds": ["webmainbench_frozen_html_to_markdown"], "date": "2026-09-24", "title": "区分请求成功与目标正文", "summary": "核心表和条件耗时改按实际生成目标正文的页面统计；浏览器错误页、空输出和请求失败不再计入正文页数。评分、原始输出和 3,270 条逻辑结果不变。", - "changes": [ - [ - "正文与耗时口径", - "进程成功退出会把 ERR_ACCESS_DENIED 等浏览器错误页计入“返回成功”，并进入成功页条件耗时。", - "从已保存的逐页诊断生成目标正文标记；核心表、条件耗时和 Moli 修复前后对照共用该标记。", - "Chrome 的错误页不再算目标正文；全部调用耗时、质量评分、重试成本和冻结证据保持不变。" - ] - ], - "i18n": { - "en": { - "title": "Separate successful requests from target-body delivery", - "summary": "The core table and conditional latency now count pages that actually produced target body content. Browser error pages, empty output, and request failures are excluded. Scores, raw outputs, and all 3,270 logical results are unchanged.", - "changes": [ - [ - "Body and latency population", - "A zero-exit process could count an ERR_ACCESS_DENIED browser page as a successful return and include it in conditional success latency.", - "Derive target-body delivery from the retained per-page diagnosis and use it consistently in the core table, conditional latency, and Moli before/after comparison.", - "The Chrome error page no longer counts as target body. All-call latency, quality scores, retry cost, and frozen evidence remain unchanged." - ] - ] - } - } + "changes": [["正文与耗时口径", "进程成功退出会把 ERR_ACCESS_DENIED 等浏览器错误页计入“返回成功”，并进入成功页条件耗时。", "从已保存的逐页诊断生成目标正文标记；核心表、条件耗时和 Moli 修复前后对照共用该标记。", "Chrome 的错误页不再算目标正文；全部调用耗时、质量评分、重试成本和冻结证据保持不变。"]], + "i18n": {"en": {"title": "Separate successful requests from target-body delivery", "summary": "The core table and conditional latency now count pages that actually produced target body content. Browser error pages, empty output, and request failures are excluded. Scores, raw outputs, and all 3,270 logical results are unchanged.", "changes": [["Body and latency population", "A zero-exit process could count an ERR_ACCESS_DENIED browser page as a successful return and include it in conditional success latency.", "Derive target-body delivery from the retained per-page diagnosis and use it consistently in the core table, conditional latency, and Moli before/after comparison.", "The Chrome error page no longer counts as target body. All-call latency, quality scores, retry cost, and frozen evidence remain unchanged."]]}} }, { "id": "2026-09-24-webmainbench-six-versions-certified", "publication_state": "published", - "comparison_kinds": [ - "webmainbench_frozen_html_to_markdown" - ], + "comparison_kinds": ["webmainbench_frozen_html_to_markdown"], "date": "2026-09-24", "title": "将修复版 Moli 纳入完整六方法比较", "summary": "修复版 Moli 按同一批 545 页完成独立认证运行；主表、结果清单和同页对照均展示六种方法，共 3,270 条结果。", - "changes": [ - [ - "六方法结果", - "修复版先前只在附注页出现，主表与同页证据仍只有五种方法。", - "将修复版作为独立第六方法纳入完整矩阵，两轮结果使用同一表格结构修正口径。", - "逐页输出和失败可直接对照；修复版另日运行，其耗时不参加原五方法的速度标优。" - ] - ], - "i18n": { - "en": { - "title": "Include repaired Moli in the complete six-version comparison", - "summary": "A separate certified run measured repaired Moli on the same 545 pages. The main table, result inventory, and same-case comparisons now include all six versions and 3,270 results.", - "changes": [ - [ - "Six-version results", - "The repaired build previously appeared only on a follow-up page, outside the main result matrix.", - "Include it as a sixth version, and apply the same corrected table-structure scoring to both complete cohorts.", - "Page outputs and failures are directly inspectable; the later timing run is not ranked against the original five." - ] - ] - } - } + "changes": [["六方法结果", "修复版先前只在附注页出现，主表与同页证据仍只有五种方法。", "将修复版作为独立第六方法纳入完整矩阵，两轮结果使用同一表格结构修正口径。", "逐页输出和失败可直接对照；修复版另日运行，其耗时不参加原五方法的速度标优。"]], + "i18n": {"en": {"title": "Include repaired Moli in the complete six-version comparison", "summary": "A separate certified run measured repaired Moli on the same 545 pages. The main table, result inventory, and same-case comparisons now include all six versions and 3,270 results.", "changes": [["Six-version results", "The repaired build previously appeared only on a follow-up page, outside the main result matrix.", "Include it as a sixth version, and apply the same corrected table-structure scoring to both complete cohorts.", "Page outputs and failures are directly inspectable; the later timing run is not ranked against the original five."]]}} }, { "id": "2026-09-24-webmainbench-moli-repair-545-followup", "publication_state": "published", - "comparison_kinds": [ - "webmainbench_frozen_html_to_markdown" - ], + "comparison_kinds": ["webmainbench_frozen_html_to_markdown"], "date": "2026-09-24", "title": "补充 Moli 修复后的 545 页复测", "summary": "同一批冻结输入上的 Moli 单方法复测由 542/545 页返回成功增至 544/545 页；两条解析器崩溃恢复，一条隔离环境阻断的跳转仍失败。", - "changes": [ - [ - "修复复测", - "原五方法报告只记录修复前的 Moli 结果，读者无法看到后续解析器修复的实际效果。", - "单列 545 页复测、同口径四项原评分、恢复页面的完整输出与剩余失败。", - "五方法认证结果和排名未改；恢复的 Autochina360 页仍夹杂大量无关链接，全文分 22.8%。不同时间的耗时和未经同一 TEDS 修正的综合分不作前后比较。" - ] - ], - "i18n": { - "en": { - "title": "Add the 545-page Moli repair follow-up", - "summary": "In a Moli-only rerun of the same frozen pages, request success rose from 542/545 to 544/545. Two parser crashes recovered; one redirect blocked by the isolated fixture still failed.", - "changes": [ - [ - "Repair follow-up", - "The five-method report only showed Moli before the parser repair.", - "Present the full-page rerun, four original-scorer metrics, complete outputs for recovered pages, and the remaining failure.", - "The certified five-method ranking is unchanged. The recovered Autochina360 output still contains many unrelated links and scores 22.8% on full text; timing across dates and aggregate scores without the same TEDS correction are not compared." - ] - ] - } - } + "changes": [["修复复测", "原五方法报告只记录修复前的 Moli 结果，读者无法看到后续解析器修复的实际效果。", "单列 545 页复测、同口径四项原评分、恢复页面的完整输出与剩余失败。", "五方法认证结果和排名未改；恢复的 Autochina360 页仍夹杂大量无关链接，全文分 22.8%。不同时间的耗时和未经同一 TEDS 修正的综合分不作前后比较。"]], + "i18n": {"en": {"title": "Add the 545-page Moli repair follow-up", "summary": "In a Moli-only rerun of the same frozen pages, request success rose from 542/545 to 544/545. Two parser crashes recovered; one redirect blocked by the isolated fixture still failed.", "changes": [["Repair follow-up", "The five-method report only showed Moli before the parser repair.", "Present the full-page rerun, four original-scorer metrics, complete outputs for recovered pages, and the remaining failure.", "The certified five-method ranking is unchanged. The recovered Autochina360 output still contains many unrelated links and scores 22.8% on full text; timing across dates and aggregate scores without the same TEDS correction are not compared."]]}} }, { "id": "2026-09-23-webmainbench-error-attribution-and-case-comparison", "publication_state": "published", - "comparison_kinds": [ - "webmainbench_frozen_html_to_markdown" - ], + "comparison_kinds": ["webmainbench_frozen_html_to_markdown"], "date": "2026-09-23", "title": "按方法定位机器问题并展示同页完整输出", "summary": "机器问题图与解读页交叉列出错误类型、方法和可确认的归属。逐题对照页在同一处显示人工主内容答案与五种方法的完整输出，结果清单可按复核原因筛选。", - "changes": [ - [ - "错误归属与逐题阅读", - "汇总数不能直接显示每种错误发生在哪个方法，逐题页只列分数而需分别打开答案和输出。", - "增加错误类型与方法矩阵、归属说明及原因筛选；同页显示参考答案和五种完整结果。", - "545 题输入、2725 条结果、分数、耗时和已复核的 28 条机器问题均不变。" - ] - ], - "i18n": { - "en": { - "title": "Attribute machine issues by method and show complete same-page outputs", - "summary": "The issue graphic and interpretation cross-tabulate error type, method, and evidenced attribution. Each case comparison displays the human main-content reference beside all five complete outputs, and the result inventory can filter by reviewed cause.", - "changes": [ - [ - "Issue attribution and case reading", - "Aggregate counts did not show which method encountered each issue, and case pages listed scores without the reference and outputs in one view.", - "Add an error-type-by-method matrix, attribution notes, and cause filtering; display the reference and five full outputs on each case page.", - "The 545 inputs, 2,725 results, scores, timings, and 28 reviewed machine issues remain unchanged." - ] - ] - } - } + "changes": [["错误归属与逐题阅读", "汇总数不能直接显示每种错误发生在哪个方法，逐题页只列分数而需分别打开答案和输出。", "增加错误类型与方法矩阵、归属说明及原因筛选；同页显示参考答案和五种完整结果。", "545 题输入、2725 条结果、分数、耗时和已复核的 28 条机器问题均不变。"]], + "i18n": {"en": {"title": "Attribute machine issues by method and show complete same-page outputs", "summary": "The issue graphic and interpretation cross-tabulate error type, method, and evidenced attribution. Each case comparison displays the human main-content reference beside all five complete outputs, and the result inventory can filter by reviewed cause.", "changes": [["Issue attribution and case reading", "Aggregate counts did not show which method encountered each issue, and case pages listed scores without the reference and outputs in one view.", "Add an error-type-by-method matrix, attribution notes, and cause filtering; display the reference and five full outputs on each case page.", "The 545 inputs, 2,725 results, scores, timings, and 28 reviewed machine issues remain unchanged."]]}} }, { "id": "2026-09-23-webmainbench-complete-545-five-methods", "publication_state": "published", - "comparison_kinds": [ - "webmainbench_frozen_html_to_markdown" - ], + "comparison_kinds": ["webmainbench_frozen_html_to_markdown"], "date": "2026-09-23", "title": "发布 WebMainBench 完整 545 题五方法对照", "summary": "以同一批 545 个官方冻结 HTML 和人工参考，实测五种稳定版浏览器方法。新报告提供逐题输入、输出、评分、失败证据和已复核的机器问题分析。", - "changes": [ - [ - "完整评测与评分", - "先前公开报告只覆盖 200 题，无法代表完整数据集。官方 TEDS 实现还可能高估部分表格相似度。", - "认证五种方法各 545 题，并用保留原始输出和官方分数的结构化树比较校正 TEDS；报告同时展示请求重试和失败分类。", - "共 2725 个逻辑结果、2747 次实际调用；292 个 TEDS 分数校正。28 个机器问题已逐条复核，其余 2697 个结果未逐条人工复核。" - ] - ], - "i18n": { - "en": { - "title": "Publish the complete 545-case WebMainBench comparison", - "summary": "Five stable browser methods were measured against the same 545 official frozen HTML inputs and human references. The new report exposes case inputs, outputs, scores, failure evidence, and reviewed machine-issue analysis.", - "changes": [ - [ - "Full evaluation and scoring", - "The earlier public report covered only 200 cases, and the official TEDS implementation could overstate some table similarities.", - "Certify 545 cases for each of five methods and correct TEDS with structured tree comparison while preserving raw outputs and official scores; show retries and failure classes.", - "There are 2,725 logical results and 2,747 physical calls; 292 TEDS scores were corrected. All 28 machine issues were reviewed individually, while the other 2,697 results were not individually human reviewed." - ] - ] - } - } + "changes": [["完整评测与评分", "先前公开报告只覆盖 200 题，无法代表完整数据集。官方 TEDS 实现还可能高估部分表格相似度。", "认证五种方法各 545 题，并用保留原始输出和官方分数的结构化树比较校正 TEDS；报告同时展示请求重试和失败分类。", "共 2725 个逻辑结果、2747 次实际调用；292 个 TEDS 分数校正。28 个机器问题已逐条复核，其余 2697 个结果未逐条人工复核。"]], + "i18n": {"en": {"title": "Publish the complete 545-case WebMainBench comparison", "summary": "Five stable browser methods were measured against the same 545 official frozen HTML inputs and human references. The new report exposes case inputs, outputs, scores, failure evidence, and reviewed machine-issue analysis.", "changes": [["Full evaluation and scoring", "The earlier public report covered only 200 cases, and the official TEDS implementation could overstate some table similarities.", "Certify 545 cases for each of five methods and correct TEDS with structured tree comparison while preserving raw outputs and official scores; show retries and failure classes.", "There are 2,725 logical results and 2,747 physical calls; 292 TEDS scores were corrected. All 28 machine issues were reviewed individually, while the other 2,697 results were not individually human reviewed."]]}} }, { "id": "2026-09-20-wcxb-measured-mode-labels", "publication_state": "published", - "comparison_kinds": [ - "wcxb_frozen_html_to_markdown" - ], + "comparison_kinds": ["wcxb_frozen_html_to_markdown"], "date": "2026-09-20", "title": "明确可选清理模式与实测版本", - "title_i18n": { - "en": "Clarify optional stripping and measured revisions" - }, + "title_i18n": {"en": "Clarify optional stripping and measured revisions"}, "summary": "商品说明案例的两列图注明确标注可选 UI 清理。PR 对照继续指向实测提交 384d1d2a，不随后续代码提交自动变为新测量。", - "summary_i18n": { - "en": "Both output captions for the product-description case identify optional UI stripping. PR comparisons remain bound to measured commit 384d1d2a; subsequent commits do not become new measurements." - }, - "changes": [ - [ - "图注与版本说明", - "可选清理案例与常规调用共用图注，版本措辞未明确冻结测量范围。", - "两列分别标注可选 UI 清理，将当前 PR 改为实测 PR。", - "504 页分数、耗时、响应与二进制绑定均不变。" - ] - ], - "changes_i18n": { - "en": [ - [ - "Captions and revision scope", - "Optional stripping shared captions with ordinary calls, and revision wording did not make the frozen scope explicit.", - "Label both outputs with optional UI stripping and refer to the measured PR revision.", - "Scores, latency, responses and binary bindings for all 504 pages remain unchanged." - ] - ] - } + "summary_i18n": {"en": "Both output captions for the product-description case identify optional UI stripping. PR comparisons remain bound to measured commit 384d1d2a; subsequent commits do not become new measurements."}, + "changes": [["图注与版本说明", "可选清理案例与常规调用共用图注，版本措辞未明确冻结测量范围。", "两列分别标注可选 UI 清理，将当前 PR 改为实测 PR。", "504 页分数、耗时、响应与二进制绑定均不变。"]], + "changes_i18n": {"en": [["Captions and revision scope", "Optional stripping shared captions with ordinary calls, and revision wording did not make the frozen scope explicit.", "Label both outputs with optional UI stripping and refer to the measured PR revision.", "Scores, latency, responses and binary bindings for all 504 pages remain unchanged."]]} }, { "id": "2026-09-18-wcxb-adjacent-code-scope-correction", "publication_state": "published", - "comparison_kinds": [ - "wcxb_frozen_html_to_markdown" - ], + "comparison_kinds": ["wcxb_frozen_html_to_markdown"], "date": "2026-09-18", "title": "纠正相邻代码修复的含义与评分解释", - "title_i18n": { - "en": "Correct the adjacent-code repair scope and score explanation" - }, + "title_i18n": {"en": "Correct the adjacent-code repair scope and score explanation"}, "summary": "5458 页的当前 PR 输出保留两个 code 节点，但不会还原由 CSS gap-2 产生的视觉间距。人工参考本身是分开的 name string；该页 F1 下降是因为评分器把内联 HTML 的 code 标签名称也计入输出词数。评测结果未改。", - "summary_i18n": { - "en": "The current PR output on page 5458 preserves two code nodes without reconstructing the visual gap supplied by CSS gap-2. The human reference already contains separate name and string tokens; F1 falls because the scorer also counts inline-HTML code tag names as output words. Measured results are unchanged." - }, - "changes": [ - [ - "报告解读", - "此前把节点边界保真写成了完整的可读间距修复，并错误解释了该页降分。", - "展示实际 Markdown 与渲染语义，区分 DOM 节点边界、CSS 布局和原始源码分词。", - "5458 页参考召回率由 98.45% 升至 100%；额外的 code 标签词使精确率与 F1 下降。504 页成绩、分母、耗时和原始响应保持不变。" - ] - ], - "changes_i18n": { - "en": [ - [ - "Interpretation", - "The earlier copy overstated node-boundary preservation as a complete readable-spacing repair and misattributed the page-level score drop.", - "Show the actual Markdown and rendered semantics while separating DOM boundaries, CSS layout and raw-source tokenization.", - "Reference recall on page 5458 rises from 98.45% to 100%; extra code-tag words lower precision and F1. The 504-page scores, denominator, latency and raw responses are unchanged." - ] - ] - } + "summary_i18n": {"en": "The current PR output on page 5458 preserves two code nodes without reconstructing the visual gap supplied by CSS gap-2. The human reference already contains separate name and string tokens; F1 falls because the scorer also counts inline-HTML code tag names as output words. Measured results are unchanged."}, + "changes": [["报告解读", "此前把节点边界保真写成了完整的可读间距修复，并错误解释了该页降分。", "展示实际 Markdown 与渲染语义，区分 DOM 节点边界、CSS 布局和原始源码分词。", "5458 页参考召回率由 98.45% 升至 100%；额外的 code 标签词使精确率与 F1 下降。504 页成绩、分母、耗时和原始响应保持不变。"]], + "changes_i18n": {"en": [["Interpretation", "The earlier copy overstated node-boundary preservation as a complete readable-spacing repair and misattributed the page-level score drop.", "Show the actual Markdown and rendered semantics while separating DOM boundaries, CSS layout and raw-source tokenization.", "Reference recall on page 5458 rises from 98.45% to 100%; extra code-tag words lower precision and F1. The 504-page scores, denominator, latency and raw responses are unchanged."]]} }, { "id": "2026-09-18-wcxb-moli-118-current-pr-comparison", "publication_state": "published", - "comparison_kinds": [ - "wcxb_frozen_html_to_markdown" - ], + "comparison_kinds": ["wcxb_frozen_html_to_markdown"], "date": "2026-09-18", "title": "更新 Moli 稳定版与当前修复版对照", - "title_i18n": { - "en": "Compare the current Moli release and PR build" - }, + "title_i18n": {"en": "Compare the current Moli release and PR build"}, "summary": "使用 Moli 1.1.8 正式发布文件与 PR #643 当前源码分别实测同一批 504 页。三张修复图现在统一展示冻结输入、1.1.8 输出和当前 PR 输出；可选界面清理另行标注。", - "summary_i18n": { - "en": "The official Moli 1.1.8 binary and current PR #643 source were measured on the same 504 pages. All three repair figures compare frozen input, 1.1.8 output and current PR output; optional UI stripping is identified separately." - }, - "changes": [ - [ - "结果与报告解读", - "此前使用旧版与早期 PR 的混合证据，版本含义不清。", - "重新认证两款 Moli 二进制的完整 504 页结果，并用同窗口交叉测量更新耗时；说明两页因结构保真而降分。", - "两版均有 504/504 个非空响应，平均 F1 分别为 52.5510% 和 52.5492%；最终轮平均耗时分别为 150.5 ms 和 151.9 ms。" - ] - ], - "changes_i18n": { - "en": [ - [ - "Results and interpretation", - "Earlier figures mixed an older release with an earlier PR source, making version identity unclear.", - "Certified full 504-page results for both Moli binaries and refreshed latency using a same-window crossover; explained two pages whose score falls while code structure improves.", - "Both builds return nonempty content on 504/504 pages; mean F1 is 52.5510% and 52.5492%, and final-round mean latency is 150.5 ms and 151.9 ms." - ] - ] - } + "summary_i18n": {"en": "The official Moli 1.1.8 binary and current PR #643 source were measured on the same 504 pages. All three repair figures compare frozen input, 1.1.8 output and current PR output; optional UI stripping is identified separately."}, + "changes": [["结果与报告解读", "此前使用旧版与早期 PR 的混合证据，版本含义不清。", "重新认证两款 Moli 二进制的完整 504 页结果，并用同窗口交叉测量更新耗时；说明两页因结构保真而降分。", "两版均有 504/504 个非空响应，平均 F1 分别为 52.5510% 和 52.5492%；最终轮平均耗时分别为 150.5 ms 和 151.9 ms。"]], + "changes_i18n": {"en": [["Results and interpretation", "Earlier figures mixed an older release with an earlier PR source, making version identity unclear.", "Certified full 504-page results for both Moli binaries and refreshed latency using a same-window crossover; explained two pages whose score falls while code structure improves.", "Both builds return nonempty content on 504/504 pages; mean F1 is 52.5510% and 52.5492%, and final-round mean latency is 150.5 ms and 151.9 ms."]]} }, { "id": "2026-09-18-wcxb-moli-repair-figure-provenance", "publication_state": "published", - "comparison_kinds": [ - "wcxb_frozen_html_to_markdown" - ], + "comparison_kinds": ["wcxb_frozen_html_to_markdown"], "date": "2026-09-18", "title": "标清 Moli 修复图的版本与证据范围", - "title_i18n": { - "en": "Identify versions and evidence scope in Moli repair figures" - }, + "title_i18n": {"en": "Identify versions and evidence scope in Moli repair figures"}, "summary": "三张图分别标出正式版 1.1.7、PR 初版与当前源码，并区分整批实测、可选功能额外对照和场景测试。评测数据未改。", - "summary_i18n": { - "en": "The three figures identify released 1.1.7, the initial PR and current source, and distinguish cohort results, an extra optional-feature comparison and scenario tests. Measured results are unchanged." - }, + "summary_i18n": {"en": "The three figures identify released 1.1.7, the initial PR and current source, and distinguish cohort results, an extra optional-feature comparison and scenario tests. Measured results are unchanged."}, "changes": [ - [ - "报告解读", - "图中的版本简称容易使源码场景测试看起来像稳定版或 504 页实测结果。", - "逐图标明两列的源码版本和证据范围，并在 PR 中展示相同图片。", - "分数、分母、耗时和原始响应不变；当前 PR 源码不冒充已完成整批重测。" - ] + ["报告解读", "图中的版本简称容易使源码场景测试看起来像稳定版或 504 页实测结果。", "逐图标明两列的源码版本和证据范围，并在 PR 中展示相同图片。", "分数、分母、耗时和原始响应不变；当前 PR 源码不冒充已完成整批重测。"] ], - "changes_i18n": { - "en": [ - [ - "Interpretation", - "Short figure labels could make a source-level scenario look like a released or full-cohort result.", - "Identify both versions and evidence scope for each figure and embed the same images in the PR.", - "Scores, denominators, latency and raw responses are unchanged; current PR source is not presented as a completed cohort rerun." - ] - ] - } + "changes_i18n": {"en": [["Interpretation", "Short figure labels could make a source-level scenario look like a released or full-cohort result.", "Identify both versions and evidence scope for each figure and embed the same images in the PR.", "Scores, denominators, latency and raw responses are unchanged; current PR source is not presented as a completed cohort rerun."]]} }, { "id": "2026-09-18-wcxb-moli-repair-visual-guide", "publication_state": "published", - "comparison_kinds": [ - "wcxb_frozen_html_to_markdown" - ], + "comparison_kinds": ["wcxb_frozen_html_to_markdown"], "date": "2026-09-18", "title": "用图和源码样本说明 Moli 的内容修复", - "title_i18n": { - "en": "Explain the Moli fixes with diagrams and source examples" - }, + "title_i18n": {"en": "Explain the Moli fixes with diagrams and source examples"}, "summary": "报告解读逐项展示代码块粘连、商品说明误删和多加空格，并在图下对照真实 HTML、转换结果与源码处理规则。评测数据未改。", - "summary_i18n": { - "en": "The interpretation shows joined code blocks, deleted product descriptions and an added space, pairing each image with actual HTML, output and source-level handling. Results are unchanged." - }, + "summary_i18n": {"en": "The interpretation shows joined code blocks, deleted product descriptions and an added space, pairing each image with actual HTML, output and source-level handling. Results are unchanged."}, "changes": [ - [ - "报告解读", - "原有三栏代码片段和长段解释不易看出原文、错误和修复的关系。", - "每个问题用一张图及真实 HTML、输出和源码规则展示差异；标明整批实测与后续测试的边界。", - "分数、分母、耗时、评测响应和审查范围不变；读者可从图旁链接核对完整证据。" - ] + ["报告解读", "原有三栏代码片段和长段解释不易看出原文、错误和修复的关系。", "每个问题用一张图及真实 HTML、输出和源码规则展示差异；标明整批实测与后续测试的边界。", "分数、分母、耗时、评测响应和审查范围不变；读者可从图旁链接核对完整证据。"] ], - "changes_i18n": { - "en": [ - [ - "Interpretation", - "The former three-column excerpts and long explanations obscured the relation between source, error and fix.", - "One image per issue is paired with real HTML, output and source rules; the measured cohort remains distinct from later source tests.", - "Scores, denominators, latency, responses and review scope are unchanged; adjacent links open the full evidence." - ] - ] - } + "changes_i18n": {"en": [["Interpretation", "The former three-column excerpts and long explanations obscured the relation between source, error and fix.", "One image per issue is paired with real HTML, output and source rules; the measured cohort remains distinct from later source tests.", "Scores, denominators, latency, responses and review scope are unchanged; adjacent links open the full evidence."]]} }, { "id": "2026-09-18-wcxb-moli-repair-explanation", "publication_state": "published", - "comparison_kinds": [ - "wcxb_frozen_html_to_markdown" - ], + "comparison_kinds": ["wcxb_frozen_html_to_markdown"], "date": "2026-09-18", "title": "补充 Moli 修复的输入与输出对照", - "title_i18n": { - "en": "Explain Moli repairs with input and output evidence" - }, + "title_i18n": {"en": "Explain Moli repairs with input and output evidence"}, "summary": "报告解读新增代码块、商品说明和相邻代码的可视对照，区分 504 页实测构建与其后的源码回归测试。评测结果、分母和耗时未改变。", - "summary_i18n": { - "en": "The interpretation now illustrates code blocks, product descriptions and adjacent code with input/output evidence. It separates the measured 504-page build from later source regression tests. Scores, denominators and latency are unchanged." - }, + "summary_i18n": {"en": "The interpretation now illustrates code blocks, product descriptions and adjacent code with input/output evidence. It separates the measured 504-page build from later source regression tests. Scores, denominators and latency are unchanged."}, "changes": [ - [ - "报告解读", - "修复缘由和源码行为分散在逐页结果与 PR 中。", - "在解读页展示冻结输入、原输出、修复输出及可核查证据；明确后续源码修改不属于已测二进制。", - "读者可以核对代码边界与正文保留的必要性；504 页成绩和耗时保持原值。" - ] + ["报告解读", "修复缘由和源码行为分散在逐页结果与 PR 中。", "在解读页展示冻结输入、原输出、修复输出及可核查证据；明确后续源码修改不属于已测二进制。", "读者可以核对代码边界与正文保留的必要性；504 页成绩和耗时保持原值。"] ], - "changes_i18n": { - "en": [ - [ - "Interpretation", - "The repair rationale and converter behavior were scattered across result pages and the PR.", - "Show frozen input, released output, repaired output and inspection links together; identify later source changes outside the measured binary.", - "Readers can verify why code boundaries and body text must be preserved; the 504-page scores and timing remain unchanged." - ] - ] - } + "changes_i18n": {"en": [["Interpretation", "The repair rationale and converter behavior were scattered across result pages and the PR.", "Show frozen input, released output, repaired output and inspection links together; identify later source changes outside the measured binary.", "Readers can verify why code boundaries and body text must be preserved; the 504-page scores and timing remain unchanged."]]} }, { "id": "2026-09-21-moli-seven-linux-release-resources", @@ -3121,19 +2491,12 @@ { "id": "2026-09-18-wcxb-frozen-html-reader-clarification", "publication_state": "published", - "comparison_kinds": [ - "wcxb_frozen_html_to_markdown" - ], + "comparison_kinds": ["wcxb_frozen_html_to_markdown"], "date": "2026-09-18", "title": "澄清冻结 HTML 阅读视图", "summary": "逐页入口明确区分冻结 HTML 的静态阅读视图与原站画面，提供完整 HTML 输入。504 页评测结果、评分和耗时均未改变。", "changes": [ - [ - "逐页输入", - "原入口称作网页预览，容易误认为复现了原站排版。", - "标明只按冻结 HTML 的文档顺序阅读；说明未冻结外部样式和图片，添加完整输入链接。", - "读者可核对输入内容，不会把未排版视图当作原站截图。" - ] + ["逐页输入", "原入口称作网页预览，容易误认为复现了原站排版。", "标明只按冻结 HTML 的文档顺序阅读；说明未冻结外部样式和图片，添加完整输入链接。", "读者可核对输入内容，不会把未排版视图当作原站截图。"] ] }, { @@ -3355,43 +2718,21 @@ { "id": "2026-09-14-public-benchmark-research", "publication_state": "published", - "comparison_kinds": [ - "public_benchmark_research" - ], + "comparison_kinds": ["public_benchmark_research"], "date": "2026-09-14", "title": "完成公开评测集调研与选型", "summary": "核查公开数据、评分器与使用证据，选定 WebMainBench-545，并冻结覆盖官方分类的 200 页抽样规则。调研版没有供应商实测结果。", "changes": [ - [ - "评测集", - "候选集的真值、评分器和使用范围尚未并列核查。", - "核对 WebMainBench-545、WCXB 及相关评测集的一手材料。", - "选定 WebMainBench-545 作为后续实测主集；调研结论不构成方法排名。" - ], - [ - "抽样", - "尚无可复现的测量输入范围。", - "定义覆盖官方分类的固定 200 页抽样合同。", - "只固定未来测量范围，未增加任何供应商调用或分数。" - ] + ["评测集", "候选集的真值、评分器和使用范围尚未并列核查。", "核对 WebMainBench-545、WCXB 及相关评测集的一手材料。", "选定 WebMainBench-545 作为后续实测主集；调研结论不构成方法排名。"], + ["抽样", "尚无可复现的测量输入范围。", "定义覆盖官方分类的固定 200 页抽样合同。", "只固定未来测量范围，未增加任何供应商调用或分数。"] ], "i18n": { "en": { "title": "Complete public benchmark research and selection", "summary": "Audited public data, scorers, and adoption evidence; selected WebMainBench-545 and froze a 200-page sampling rule covering official categories. The research edition contains no measured provider results.", "changes": [ - [ - "Benchmark", - "Candidate ground truth, scorers, and use had not been audited side by side.", - "Checked primary sources for WebMainBench-545, WCXB, and adjacent benchmarks.", - "Selected WebMainBench-545 as the primary set for later measurement; research findings are not a method ranking." - ], - [ - "Sampling", - "There was no reproducible scope for calls.", - "Defined a fixed 200-page sampling contract covering official categories.", - "Only the future measurement scope was fixed; no provider calls or scores were added." - ] + ["Benchmark", "Candidate ground truth, scorers, and use had not been audited side by side.", "Checked primary sources for WebMainBench-545, WCXB, and adjacent benchmarks.", "Selected WebMainBench-545 as the primary set for later measurement; research findings are not a method ranking."], + ["Sampling", "There was no reproducible scope for calls.", "Defined a fixed 200-page sampling contract covering official categories.", "Only the future measurement scope was fixed; no provider calls or scores were added."] ] } } @@ -3399,55 +2740,23 @@ { "id": "2026-09-17-public-webmainbench-moli-balanced-latency-correction", "publication_state": "published", - "comparison_kinds": [ - "webmainbench_frozen_html_to_markdown" - ], + "comparison_kinds": ["webmainbench_frozen_html_to_markdown"], "date": "2026-09-17", "title": "纠正 Moli 两版的正式耗时", "summary": "同一预选 200 页在无并行基准进程的窗口串行交叉复测两轮，按测前固定规则采用第二轮。Moli 1.1.5 的成功均值由 153 ms 改为 105 ms，1.1.7 由 285 ms 改为 98 ms；两版各有 199/200 页成功。原始逐页结果、正文、质量分和判定保留。", "changes": [ - [ - "正式耗时", - "两版来自不同时间的独立批次，1.1.7 表面上更慢。", - "200 页两轮交叉顺序、共 800 次物理调用；预先指定第二轮的 400 次作为两版新的逐页耗时。", - "1.1.7 的均值、P50、P95 均未高于 1.1.5；旧值可从纠正收据核对。" - ], - [ - "失败页", - "两版在同一页未生成正文。", - "保留页面脚本和隔离网络条件，复测各轮及此前重试。", - "原站跳转仍被网络隔离阻断；两版继续各计 199/200 页成功。" - ], - [ - "评分证据", - "耗时纠正可能与已审核的原始输出混淆。", - "旧结果和原始物理调用不覆盖；逐条仅替换 Moli 的 elapsed_ms，并对其余字段及另外四种方法做相等校验。", - "全部质量分、计分分母、正文哈希和结果判定不变。" - ] + ["正式耗时", "两版来自不同时间的独立批次，1.1.7 表面上更慢。", "200 页两轮交叉顺序、共 800 次物理调用；预先指定第二轮的 400 次作为两版新的逐页耗时。", "1.1.7 的均值、P50、P95 均未高于 1.1.5；旧值可从纠正收据核对。"], + ["失败页", "两版在同一页未生成正文。", "保留页面脚本和隔离网络条件，复测各轮及此前重试。", "原站跳转仍被网络隔离阻断；两版继续各计 199/200 页成功。"], + ["评分证据", "耗时纠正可能与已审核的原始输出混淆。", "旧结果和原始物理调用不覆盖；逐条仅替换 Moli 的 elapsed_ms，并对其余字段及另外四种方法做相等校验。", "全部质量分、计分分母、正文哈希和结果判定不变。"] ], "i18n": { "en": { "title": "Correct formal timing for both Moli versions", "summary": "Two serial, order-balanced rounds used the same preselected 200 pages without concurrent benchmark processes. Round two was designated before calls as the formal timing source. Successful-fetch means changed from 153 to 105 ms for Moli 1.1.5 and from 285 to 98 ms for 1.1.7; each succeeded on 199/200 pages. Prior per-page results, output text, quality scores, and verdicts are retained.", "changes": [ - [ - "Formal timing", - "The two versions came from separate batches at different times, making 1.1.7 appear slower.", - "Two order-balanced rounds on 200 pages produced 800 physical calls; the 400 calls in the predesignated second round became the formal per-page timings.", - "Mean, P50, and P95 for 1.1.7 are no higher than for 1.1.5. Prior values remain in the correction receipt." - ], - [ - "Failed page", - "Neither version produced target content on the same page.", - "Preserved the page script and isolated network conditions while checking both new rounds and earlier retries.", - "The source-site redirect is still blocked by network isolation; each version remains at 199/200 successful pages." - ], - [ - "Scoring evidence", - "Timing corrections could be confused with the reviewed original output.", - "Preserved prior results and physical calls; replaced elapsed_ms only for Moli and checked every other field and the other four methods for equality.", - "All quality scores, scored-page counts, output hashes, and verdicts are unchanged." - ] + ["Formal timing", "The two versions came from separate batches at different times, making 1.1.7 appear slower.", "Two order-balanced rounds on 200 pages produced 800 physical calls; the 400 calls in the predesignated second round became the formal per-page timings.", "Mean, P50, and P95 for 1.1.7 are no higher than for 1.1.5. Prior values remain in the correction receipt."], + ["Failed page", "Neither version produced target content on the same page.", "Preserved the page script and isolated network conditions while checking both new rounds and earlier retries.", "The source-site redirect is still blocked by network isolation; each version remains at 199/200 successful pages."], + ["Scoring evidence", "Timing corrections could be confused with the reviewed original output.", "Preserved prior results and physical calls; replaced elapsed_ms only for Moli and checked every other field and the other four methods for equality.", "All quality scores, scored-page counts, output hashes, and verdicts are unchanged."] ] } } @@ -3571,25 +2880,13 @@ { "id": "2026-09-17-public-webmainbench-denominators-and-version-examples", "publication_state": "published", - "comparison_kinds": [ - "webmainbench_frozen_html_to_markdown" - ], + "comparison_kinds": ["webmainbench_frozen_html_to_markdown"], "date": "2026-09-17", "title": "用公式分实例解释计分页，并核对 Moli 版本差异", "summary": "首页用 Moli 1.1.7 的 200 页公式分流程和四格实例解释计分分母；报告解读增加两版同页对照及正文、代码、公式的五个原始输出案例。所有分数、分母和耗时保持不变。", "changes": [ - [ - "计分分母", - "表格仅显示 n=xx/200，难以判断哪些页被计入。", - "首页展示正文生成、公式识别与计分分支，并按人工答案和输出各有无公式列出真实页数。", - "两侧均无公式跳过；仅一侧有公式以 0 分计入；分母不是非空输出数或通过数。" - ], - [ - "版本差异", - "Moli 1.1.7 的其他分项下降缺少逐页解释。", - "并列展示两版同页均值及五个可核对案例，区分实际输出差异、计分页变化和评分器误识别。", - "固定评分器下的下降成立；新版输出变长、缩进和转义改变了正文与结构项的得分。" - ] + ["计分分母", "表格仅显示 n=xx/200，难以判断哪些页被计入。", "首页展示正文生成、公式识别与计分分支，并按人工答案和输出各有无公式列出真实页数。", "两侧均无公式跳过；仅一侧有公式以 0 分计入；分母不是非空输出数或通过数。"], + ["版本差异", "Moli 1.1.7 的其他分项下降缺少逐页解释。", "并列展示两版同页均值及五个可核对案例，区分实际输出差异、计分页变化和评分器误识别。", "固定评分器下的下降成立；新版输出变长、缩进和转义改变了正文与结构项的得分。"] ], "i18n": { "en": { @@ -3635,37 +2932,15 @@ { "id": "2026-09-16-public-webmainbench-reading-view", "publication_state": "published", - "comparison_kinds": [ - "webmainbench_frozen_html_to_markdown" - ], + "comparison_kinds": ["webmainbench_frozen_html_to_markdown"], "date": "2026-09-16", "title": "明确主内容评测范围并提供网页阅读视图", "summary": "首页用输入、目标和分数含义说明低分来源，不再把本轮结果泛化为通用抓取排名。冻结 HTML 可直接阅读，来源网站与原始源码分开查看。分数、分母、耗时和供应商调用不变。", "changes": [ - [ - "评测范围", - "整页转换输出与主内容参考混在通用抓取能力描述中。", - "首页明确主内容提取目标，并提供低分案例入口；解读结论限定于本轮转换。", - "不代表实时网站访问、反爬或浏览器综合能力。" - ], - [ - "网页查看", - "HTML 入口只提供原始文件，来源链接藏在折叠区。", - "增加隔离的静态阅读视图与完整源码展开入口，来源网站直接可见。", - "阅读视图禁用脚本、外部资源和跳转，不冒充实测浏览器截图。" - ], - [ - "输入路径", - "同一 HTML 在远端服务和本地浏览器中的输入方式不直观。", - "并列展示 Kitesurf 的 API html 输入与四种本地方法的 HTTPS 路径。", - "输入字节相同，不等于执行环境相同。" - ], - [ - "表格评分", - "容易把表格可识别、完整恢复和得满分混为一谈。", - "并列展示完整表格、缺行表格和摊平文字的 Markdown 与阅读效果。", - "部分恢复仍计分；table_edit 比较规范化表格 HTML，TEDS 比较层级与内容。" - ] + ["评测范围", "整页转换输出与主内容参考混在通用抓取能力描述中。", "首页明确主内容提取目标，并提供低分案例入口；解读结论限定于本轮转换。", "不代表实时网站访问、反爬或浏览器综合能力。"], + ["网页查看", "HTML 入口只提供原始文件，来源链接藏在折叠区。", "增加隔离的静态阅读视图与完整源码展开入口，来源网站直接可见。", "阅读视图禁用脚本、外部资源和跳转，不冒充实测浏览器截图。"], + ["输入路径", "同一 HTML 在远端服务和本地浏览器中的输入方式不直观。", "并列展示 Kitesurf 的 API html 输入与四种本地方法的 HTTPS 路径。", "输入字节相同，不等于执行环境相同。"], + ["表格评分", "容易把表格可识别、完整恢复和得满分混为一谈。", "并列展示完整表格、缺行表格和摊平文字的 Markdown 与阅读效果。", "部分恢复仍计分；table_edit 比较规范化表格 HTML，TEDS 比较层级与内容。"] ], "i18n": { "en": { @@ -3676,43 +2951,16 @@ { "id": "2026-09-16-public-webmainbench-decision-guide", "publication_state": "published", - "comparison_kinds": [ - "webmainbench_frozen_html_to_markdown" - ], + "comparison_kinds": ["webmainbench_frozen_html_to_markdown"], "date": "2026-09-16", "title": "用真实案例重写评测解读", "summary": "成绩、分母、耗时和正式调用均未改变。报告解读现在从选型结论出发，用一页共同低分案例、一页表格结构案例和六页异常根因解释分数含义与适用边界；旧的重复说明表已删除。", "changes": [ - [ - "选型结论", - "旧页面罗列测量口径和方法画像，读者仍需自行拼出选择。", - "明确区分通用候选、低延迟文字与代码、以及严格按 overall 排名三种目标，并给出对应候选。", - "结论只适用于冻结 HTML 转 Markdown；真实 URL 仍需同时间窗复测。" - ], - [ - "分数含义", - "overall、分项分数和变化分母分散说明，容易被误读为逐页通过率。", - "用 Kitesurf 的五项实际分数和分母展示 overall 的完整算式，并直达评分流程。", - "官方没有统一的逐页及格线；正文生成、内容质量和耗时继续分开报告。" - ], - [ - "共同低分", - "低文字分缺少一页可以直接核对的人工答案与五方法输出。", - "加入 Wikipedia 冻结页实例，展示人工正文、方法输出开头、五种长度和 text_edit 分数。", - "35 页在五种方法上都低于 10%，说明整页输出与主内容答案存在共同范围偏差。" - ], - [ - "表格零分", - "Moli 和 Obscura 的 0 分容易被误读为空响应。", - "加入同一张两列表格的可解析 Markdown 与摊平文字对照，并列出五种方法的 TEDS。", - "该例五种方法都有正文；零分来自无法还原表格、行和单元格结构。" - ], - [ - "异常处理", - "九次无正文结果被笼统归为失败，无法决定修复方向。", - "按已验证对照分成页面脚本冲突、冻结页主动回原站、服务复杂度边界三类，并链接逐页证据。", - "未公开的 Cloudflare 内部阈值仍不猜测，只报告可重复触发的内容类型。" - ] + ["选型结论", "旧页面罗列测量口径和方法画像，读者仍需自行拼出选择。", "明确区分通用候选、低延迟文字与代码、以及严格按 overall 排名三种目标，并给出对应候选。", "结论只适用于冻结 HTML 转 Markdown；真实 URL 仍需同时间窗复测。"], + ["分数含义", "overall、分项分数和变化分母分散说明，容易被误读为逐页通过率。", "用 Kitesurf 的五项实际分数和分母展示 overall 的完整算式，并直达评分流程。", "官方没有统一的逐页及格线；正文生成、内容质量和耗时继续分开报告。"], + ["共同低分", "低文字分缺少一页可以直接核对的人工答案与五方法输出。", "加入 Wikipedia 冻结页实例，展示人工正文、方法输出开头、五种长度和 text_edit 分数。", "35 页在五种方法上都低于 10%，说明整页输出与主内容答案存在共同范围偏差。"], + ["表格零分", "Moli 和 Obscura 的 0 分容易被误读为空响应。", "加入同一张两列表格的可解析 Markdown 与摊平文字对照，并列出五种方法的 TEDS。", "该例五种方法都有正文；零分来自无法还原表格、行和单元格结构。"], + ["异常处理", "九次无正文结果被笼统归为失败，无法决定修复方向。", "按已验证对照分成页面脚本冲突、冻结页主动回原站、服务复杂度边界三类，并链接逐页证据。", "未公开的 Cloudflare 内部阈值仍不猜测，只报告可重复触发的内容类型。"] ], "i18n": { "en": { @@ -3723,43 +2971,16 @@ { "id": "2026-09-16-public-webmainbench-score-reading-and-layout", "publication_state": "published", - "comparison_kinds": [ - "webmainbench_frozen_html_to_markdown" - ], + "comparison_kinds": ["webmainbench_frozen_html_to_markdown"], "date": "2026-09-16", "title": "澄清计分分支、复算身份与核心表排序", "summary": "成绩、分母、耗时、输出和正式调用均未改变。核心表默认按 overall 从高到低排列；报告明确修正后分数来自报告侧独立复算，不是对官方仓库或官方榜单的修改，并把空内容的跳过与零分条件写成确定规则。", "changes": [ - [ - "核心排序", - "首页虽然展示 overall，默认排序却落在 text_edit，序号也随错误排序生成。", - "默认按修正后 overall 降序排列，读者打开页面即可看到同一口径的主结果顺序。", - "各列仍可交互排序；绿色只标记方向明确且唯一的领先值。" - ], - [ - "复算身份", - "“本地修正版”容易被理解为修改了官方代码或官方榜单。", - "统一改为“修正后复算”，说明官方仓库和榜单未改；报告侧沿用官方解析和代价规则，对保存输出独立复算。", - "官方原始分和修正后复算分继续并列保留，可从核验页和逐页详情核对。" - ], - [ - "分母规则", - "“可能不计分”和“通常跳过”没有说明官方程序的确定分支。", - "明确两侧都没有代码、公式或表格时该项返回失败并退出均值；只有一侧有时以 0 分进入均值；两侧都有时计算相似度。", - "分母是成功算出逐页分数的页数，不是通过数；官方没有统一逐页及格线。" - ], - [ - "表格结构", - "“节点”和“树编辑距离”缺少用户可见的结构含义。", - "用“表格—行—单元格”的还原过程解释 TEDS，并说明仅保留单元格文字、没有 Markdown 表格语法时不能还原结构。", - "table_edit 仍比较表格文字；table_TEDS 比较可还原的表格结构。" - ], - [ - "宽表布局", - "通用宽表把前七列写死后只给最后两列各留 36 像素，解读文字逐字换行，把一行撑到约 900–1,050 像素。", - "方法画像表为耗时和解读列设置独立宽度，保留横向滚动并恢复正常行高。", - "中英文、桌面和窄屏沿用同一语义表格。" - ] + ["核心排序", "首页虽然展示 overall，默认排序却落在 text_edit，序号也随错误排序生成。", "默认按修正后 overall 降序排列，读者打开页面即可看到同一口径的主结果顺序。", "各列仍可交互排序；绿色只标记方向明确且唯一的领先值。"], + ["复算身份", "“本地修正版”容易被理解为修改了官方代码或官方榜单。", "统一改为“修正后复算”，说明官方仓库和榜单未改；报告侧沿用官方解析和代价规则，对保存输出独立复算。", "官方原始分和修正后复算分继续并列保留，可从核验页和逐页详情核对。"], + ["分母规则", "“可能不计分”和“通常跳过”没有说明官方程序的确定分支。", "明确两侧都没有代码、公式或表格时该项返回失败并退出均值；只有一侧有时以 0 分进入均值；两侧都有时计算相似度。", "分母是成功算出逐页分数的页数，不是通过数；官方没有统一逐页及格线。"], + ["表格结构", "“节点”和“树编辑距离”缺少用户可见的结构含义。", "用“表格—行—单元格”的还原过程解释 TEDS，并说明仅保留单元格文字、没有 Markdown 表格语法时不能还原结构。", "table_edit 仍比较表格文字；table_TEDS 比较可还原的表格结构。"], + ["宽表布局", "通用宽表把前七列写死后只给最后两列各留 36 像素，解读文字逐字换行，把一行撑到约 900–1,050 像素。", "方法画像表为耗时和解读列设置独立宽度，保留横向滚动并恢复正常行高。", "中英文、桌面和窄屏沿用同一语义表格。"] ], "i18n": { "en": { @@ -3770,43 +2991,16 @@ { "id": "2026-09-16-public-webmainbench-causal-interpretation", "publication_state": "published", - "comparison_kinds": [ - "webmainbench_frozen_html_to_markdown" - ], + "comparison_kinds": ["webmainbench_frozen_html_to_markdown"], "date": "2026-09-16", "title": "用全局证据重写公开基准解读", "summary": "成绩、分母、耗时和正式调用均未改变。报告解读改为围绕成功层级、整页范围偏差、结构损失、评分器额外识别、冻结页运行冲突和评分修正展开，并把 35 个共同低分页、分母构成及 6 个异常页面的根因放入同一条分析主线。", "changes": [ - [ - "解读主线", - "原解读按测量、实例和配置分章，主要承担导航，读者难以回答高交付率为何仍只有中低质量分。", - "以 991/1,000 次目标正文与 21.6%–32.4% overall 的反差开篇，依次解释成功层级、低分机制、分母、方法画像和适用边界。", - "overall 是五个不同分母分项的平均，不是固定 200 页上的统一通过率。" - ], - [ - "低文字分", - "零散案例能看到多余内容，但没有量化它是不是跨方法的共同问题。", - "完整矩阵显示 35 页在五种方法上都低于 10%；这些低分页的输出长度中位数约为人工答案的 17–20 倍。", - "该证据支持整页输出与主内容答案的范围错位，不等同于证明所有编辑距离差异都来自模板文字。" - ], - [ - "分母解释", - "只说明分母会不同，没有展示差异由哪一侧的内容触发。", - "逐方法列出代码、公式和表格的计分页，以及仅输出侧、仅答案侧数量；例如 Lightpanda 公式 169 页中有 127 页只在输出侧识别出公式。", - "评分器按 Markdown 语法识别内容；额外识别不等于页面语义上真的含有该类内容。" - ], - [ - "异常根因", - "正文缺失与低分散落在多个页面，容易被概括为请求或网络失败。", - "把 9 次目标正文缺失归并为 6 个页面，展示脚本依赖、防框架循环、回源导航和 DOM / 表格复杂度等已验证原因。", - "服务端未公开资源阈值的两例只定位到触发内容类型，不虚构内部阶段。" - ], - [ - "展示与校验", - "长文缺少稳定的分析流程图，首次发布的 Mermaid 在分享页中未绘制。", - "增加可见 HTML 流程作为默认内容，Mermaid 只作渐进增强；补充 0–10% 筛选并将结构、链接和回退内容纳入发布检查。", - "中英文路由、章节、数据和证据链接保持一一对应。" - ] + ["解读主线", "原解读按测量、实例和配置分章，主要承担导航，读者难以回答高交付率为何仍只有中低质量分。", "以 991/1,000 次目标正文与 21.6%–32.4% overall 的反差开篇，依次解释成功层级、低分机制、分母、方法画像和适用边界。", "overall 是五个不同分母分项的平均，不是固定 200 页上的统一通过率。"], + ["低文字分", "零散案例能看到多余内容，但没有量化它是不是跨方法的共同问题。", "完整矩阵显示 35 页在五种方法上都低于 10%；这些低分页的输出长度中位数约为人工答案的 17–20 倍。", "该证据支持整页输出与主内容答案的范围错位，不等同于证明所有编辑距离差异都来自模板文字。"], + ["分母解释", "只说明分母会不同，没有展示差异由哪一侧的内容触发。", "逐方法列出代码、公式和表格的计分页，以及仅输出侧、仅答案侧数量；例如 Lightpanda 公式 169 页中有 127 页只在输出侧识别出公式。", "评分器按 Markdown 语法识别内容；额外识别不等于页面语义上真的含有该类内容。"], + ["异常根因", "正文缺失与低分散落在多个页面，容易被概括为请求或网络失败。", "把 9 次目标正文缺失归并为 6 个页面，展示脚本依赖、防框架循环、回源导航和 DOM / 表格复杂度等已验证原因。", "服务端未公开资源阈值的两例只定位到触发内容类型，不虚构内部阶段。"], + ["展示与校验", "长文缺少稳定的分析流程图，首次发布的 Mermaid 在分享页中未绘制。", "增加可见 HTML 流程作为默认内容，Mermaid 只作渐进增强；补充 0–10% 筛选并将结构、链接和回退内容纳入发布检查。", "中英文路由、章节、数据和证据链接保持一一对应。"] ], "i18n": { "en": { @@ -3817,37 +3011,15 @@ { "id": "2026-09-16-public-webmainbench-anomaly-root-causes", "publication_state": "published", - "comparison_kinds": [ - "webmainbench_frozen_html_to_markdown" - ], + "comparison_kinds": ["webmainbench_frozen_html_to_markdown"], "date": "2026-09-16", "title": "补齐异常结果根因与对照证据", "summary": "对 1,000 条结果执行确定性异常扫描，30 条触发结果集中在 6 个页面。删减对照定位到页面脚本、原站导航和表格 DOM 复杂度；逐页详情与核验页展示直接观察和证据边界。正式调用、输出、分数、分母和耗时均未改变。", "changes": [ - [ - "异常范围", - "缺正文、错误页、同页分歧和极低分仅有现象说明，无法判断是页面、方法、网络还是评分问题。", - "以固定规则扫描完整 1,000 条矩阵，并把 30 条触发结果归并到 6 个页面逐一复核。", - "规则用于发现可疑点，不把未触发结果宣称为逐条人工复核通过。" - ], - [ - "根因定位", - "五条 Kitesurf 异常和一组本地浏览器异常缺少能重复验证的触发条件。", - "用不计分的脚本、表格和导航删减对照定位递归错误处理器、错误防框架条件、降级跳转、原站跳转及累计表格复杂度。", - "Cloudflare 未公开的内部资源阈值仍不推断为具体内存或转换阶段。" - ], - [ - "逐页证据", - "浏览器错误页虽然有非空文字，容易与目标正文混淆；结果详情对根因覆盖不完整。", - "错误页统一归入未生成目标正文；Track ID 与方法详情展示接口状态、最终内容、完整输出、对照观察和证据支持的原因。", - "正文存在不是质量通过，官方仍没有统一逐页及格线。" - ], - [ - "结果处置", - "诊断输入经过删减，不能作为正式任务替换原调用。", - "发布版只携带绑定哈希的紧凑复核记录，保留原始 1,000 条结果及全部计分数据。", - "诊断调用不进入成绩、分母、耗时或供应商成功率。" - ] + ["异常范围", "缺正文、错误页、同页分歧和极低分仅有现象说明，无法判断是页面、方法、网络还是评分问题。", "以固定规则扫描完整 1,000 条矩阵，并把 30 条触发结果归并到 6 个页面逐一复核。", "规则用于发现可疑点，不把未触发结果宣称为逐条人工复核通过。"], + ["根因定位", "五条 Kitesurf 异常和一组本地浏览器异常缺少能重复验证的触发条件。", "用不计分的脚本、表格和导航删减对照定位递归错误处理器、错误防框架条件、降级跳转、原站跳转及累计表格复杂度。", "Cloudflare 未公开的内部资源阈值仍不推断为具体内存或转换阶段。"], + ["逐页证据", "浏览器错误页虽然有非空文字，容易与目标正文混淆；结果详情对根因覆盖不完整。", "错误页统一归入未生成目标正文；Track ID 与方法详情展示接口状态、最终内容、完整输出、对照观察和证据支持的原因。", "正文存在不是质量通过，官方仍没有统一逐页及格线。"], + ["结果处置", "诊断输入经过删减，不能作为正式任务替换原调用。", "发布版只携带绑定哈希的紧凑复核记录，保留原始 1,000 条结果及全部计分数据。", "诊断调用不进入成绩、分母、耗时或供应商成功率。"] ], "i18n": { "en": { @@ -3858,43 +3030,16 @@ { "id": "2026-09-16-public-webmainbench-report-information-architecture", "publication_state": "published", - "comparison_kinds": [ - "webmainbench_frozen_html_to_markdown" - ], + "comparison_kinds": ["webmainbench_frozen_html_to_markdown"], "date": "2026-09-16", "title": "重整公开基准报告的信息架构与逐页诊断", "summary": "测量结果、评分值和样本均未改变。首页合并为一张核心比较表并补齐均值、P50、P95；评测集按独立分类维度筛选；方法页增加流程图；结果页把响应、正文状态和证据支持的原因放回逐页详情。", "changes": [ - [ - "首页比较", - "质量、正文状态、配对耗时和分项分母分散在多张重复表中，排序列又被误当成绿色领先项。", - "用一张横向核心表展示分项分数与全部 200 次逻辑结果的均值、P50、P95；分母置于分数下方，绿色只标唯一且方向明确的领先值。", - "table_TEDS 与受其影响的 overall 继续只留档，不标领先者。" - ], - [ - "评测集", - "五套独立分类混在同一张长表里，无法看出每套分类的样本数都合计为 200。", - "按语言、难度、表格、代码和公式维度筛选分布表，并实时显示母集和样本合计。", - "分类维度彼此独立，同一页面同时属于五套分类。" - ], - [ - "方法说明", - "评测流程主要依赖连续长段文字，输入、响应、评分和聚合关系不直观。", - "用 Mermaid 流程图先展示冻结输入、方法转换、正文判断、逐页评分和按实际页数聚合；运行细节折叠为按需证据。", - "评分公式和已知 TEDS 缺陷仍保留可核对文字与源码链接。" - ], - [ - "逐页诊断", - "“交付”和字符数无法区分正文、错误响应、错误页或无响应，失败原因又集中在重复汇总表。", - "结果矩阵改为“是否生成正文”和“响应与正文”；每条失败详情显示直接观察、证据支持的原因范围、完整错误与评分字段。", - "没有证据的远端内部根因不推断；原始 URL 仍不参与本轮抓取。" - ], - [ - "报告结构", - "针对单条反馈不断增加模块，造成首页、分析页和解读页职责重叠。", - "评测集只讲样本，方法页只讲流程，分析页保留同页比较、分类筛选和逐页矩阵，详情页承接响应与原因。", - "中英文路由、数据、筛选和证据入口继续一一对应。" - ] + ["首页比较", "质量、正文状态、配对耗时和分项分母分散在多张重复表中，排序列又被误当成绿色领先项。", "用一张横向核心表展示分项分数与全部 200 次逻辑结果的均值、P50、P95；分母置于分数下方，绿色只标唯一且方向明确的领先值。", "table_TEDS 与受其影响的 overall 继续只留档，不标领先者。"], + ["评测集", "五套独立分类混在同一张长表里，无法看出每套分类的样本数都合计为 200。", "按语言、难度、表格、代码和公式维度筛选分布表，并实时显示母集和样本合计。", "分类维度彼此独立，同一页面同时属于五套分类。"], + ["方法说明", "评测流程主要依赖连续长段文字，输入、响应、评分和聚合关系不直观。", "用 Mermaid 流程图先展示冻结输入、方法转换、正文判断、逐页评分和按实际页数聚合；运行细节折叠为按需证据。", "评分公式和已知 TEDS 缺陷仍保留可核对文字与源码链接。"], + ["逐页诊断", "“交付”和字符数无法区分正文、错误响应、错误页或无响应，失败原因又集中在重复汇总表。", "结果矩阵改为“是否生成正文”和“响应与正文”；每条失败详情显示直接观察、证据支持的原因范围、完整错误与评分字段。", "没有证据的远端内部根因不推断；原始 URL 仍不参与本轮抓取。"], + ["报告结构", "针对单条反馈不断增加模块，造成首页、分析页和解读页职责重叠。", "评测集只讲样本，方法页只讲流程，分析页保留同页比较、分类筛选和逐页矩阵，详情页承接响应与原因。", "中英文路由、数据、筛选和证据入口继续一一对应。"] ], "i18n": { "en": { @@ -3905,37 +3050,15 @@ { "id": "2026-09-15-public-webmainbench-dataset-and-result-clarity", "publication_state": "published", - "comparison_kinds": [ - "webmainbench_frozen_html_to_markdown" - ], + "comparison_kinds": ["webmainbench_frozen_html_to_markdown"], "date": "2026-09-15", "title": "公开基准页面与方法结果分开展示", "summary": "200 页评测集只展示冻结输入、人工参考与官方分类；五方法结果仍在分析和方法页。结果页明确区分输出交付与官方连续质量分，增加逐指标计分页数明细。方法成绩、评分公式、样本范围和耗时均未改变。", "changes": [ - [ - "评测集", - "案例列表混入方法交付结果，冻结页面与人工参考难以独立查看。", - "案例列表只保留页面分类；每个 Track ID 可查看完整参考和冻结 HTML。", - "方法成绩在结果分析中继续提供。" - ], - [ - "评分分母", - "分项页数随方法变化但缺少直接的数量拆解。", - "逐方法逐指标列出固定页数、成功交付、实际计分和已交付未计分的页数。", - "沿用官方 success=true 计入规则，不设置逐页及格线。" - ], - [ - "结果交互", - "请求失败易被误读为原始 URL 请求失败，案例和方法详情入口不够清晰。", - "结果筛选使用已交付/未交付；Track ID 指向页面和参考，方法名指向该次评分与完整输出。搜索预处理并延迟触发以减少千行表筛选卡顿。", - "原始 URL 在本轮仅为来源元数据。" - ], - [ - "方法详情", - "物理调用、长标识和双栏长文本占据主结果版面。", - "主视图只展示最终采用的结果、官方分项评分和完整输出；调用明细、来源哈希可展开核验。", - "物理调用证据和累计耗时继续保留。" - ] + ["评测集", "案例列表混入方法交付结果，冻结页面与人工参考难以独立查看。", "案例列表只保留页面分类；每个 Track ID 可查看完整参考和冻结 HTML。", "方法成绩在结果分析中继续提供。"], + ["评分分母", "分项页数随方法变化但缺少直接的数量拆解。", "逐方法逐指标列出固定页数、成功交付、实际计分和已交付未计分的页数。", "沿用官方 success=true 计入规则，不设置逐页及格线。"], + ["结果交互", "请求失败易被误读为原始 URL 请求失败，案例和方法详情入口不够清晰。", "结果筛选使用已交付/未交付；Track ID 指向页面和参考，方法名指向该次评分与完整输出。搜索预处理并延迟触发以减少千行表筛选卡顿。", "原始 URL 在本轮仅为来源元数据。"], + ["方法详情", "物理调用、长标识和双栏长文本占据主结果版面。", "主视图只展示最终采用的结果、官方分项评分和完整输出；调用明细、来源哈希可展开核验。", "物理调用证据和累计耗时继续保留。"] ], "i18n": { "en": { @@ -3946,31 +3069,14 @@ { "id": "2026-09-15-public-webmainbench-reading-and-inspection", "publication_state": "published", - "comparison_kinds": [ - "webmainbench_frozen_html_to_markdown" - ], + "comparison_kinds": ["webmainbench_frozen_html_to_markdown"], "date": "2026-09-15", "title": "公开网页基准的逐页对照与分类筛选", "summary": "WebMainBench-545 预选 200 页的五方法分数、调用和官方评分未变。评测集可打开每页人工真值、冻结 HTML 与五种实际输出；分类分数改为按分类轴、类别和方法筛选，长表横向展开。", "changes": [ - [ - "评测集", - "候选基准表和分类表在窄屏拥挤，案例入口埋在页面末尾。", - "证据型长表在容器内横向滚动；200 页案例按官方分类筛选，并可继续打开人工真值、逐项评分原因和完整输出。", - "只说明已测 200 页，不把样本分数称作 545 页官方总榜。" - ], - [ - "结果分析", - "五个分类轴顺次展开，页面过长。", - "按分类轴、类别和方法筛选同一张结果表，保留官方分项分数和各项有效分母。", - "分类覆盖不等于比例代表性；无分母显示为未计算。" - ], - [ - "解读", - "单看 overall 容易掩盖正文、代码、公式和表格差异。", - "以同页实例连接各分项分榜、方法结果与完整人工真值。", - "请求交付、质量和端到端耗时仍分别报告。" - ] + ["评测集", "候选基准表和分类表在窄屏拥挤，案例入口埋在页面末尾。", "证据型长表在容器内横向滚动；200 页案例按官方分类筛选，并可继续打开人工真值、逐项评分原因和完整输出。", "只说明已测 200 页，不把样本分数称作 545 页官方总榜。"], + ["结果分析", "五个分类轴顺次展开，页面过长。", "按分类轴、类别和方法筛选同一张结果表，保留官方分项分数和各项有效分母。", "分类覆盖不等于比例代表性；无分母显示为未计算。"], + ["解读", "单看 overall 容易掩盖正文、代码、公式和表格差异。", "以同页实例连接各分项分榜、方法结果与完整人工真值。", "请求交付、质量和端到端耗时仍分别报告。"] ], "i18n": { "en": { @@ -3981,56 +3087,24 @@ { "id": "2026-09-15-resource-winner-presentation", "publication_state": "published", - "comparison_kinds": [ - "headless_html", - "headless_markdown" - ], + "comparison_kinds": ["headless_html", "headless_markdown"], "date": "2026-09-15", "title": "资源对比按指标分组并明确领先者", "summary": "首页与报告解读将 CPU、内存和网络拆为三个指标组，直接标出可按低消耗判断的逐项最低值，并在摘要中说明 Lightpanda 是本轮资源效率领先者。评测证据、任务判定和分母不变。", "changes": [ - [ - "资源结论", - "资源数据平铺在一张宽表中，读者需要自行寻找最低值。", - "增加资源效率结论卡和 CPU、内存、网络三张关键结果卡，明确 Lightpanda 在本轮低消耗指标中的领先。", - "资源领先与综合任务通过率分别判断，不合成总分。" - ], - [ - "逐项比较", - "表格只突出排序列，没有稳定标记每项最低值。", - "按计算、内存、网络分组，逐项标记最低消耗；CPU 利用率和传输速率只描述运行形态，不按低值评选。", - "沿用四种本地方法共同通过的 15/20 个普通网页，不产生新的供应商调用。" - ], - [ - "发布门禁", - "资源展示要求依赖人工记忆。", - "仓库规则和报告检查同时要求首页、解读页及中英文版具备赢家摘要、三类分组和逐项赢家标记。", - "以后新增或重建报告时缺失任一部分都会阻断发布。" - ] - ], + ["资源结论", "资源数据平铺在一张宽表中，读者需要自行寻找最低值。", "增加资源效率结论卡和 CPU、内存、网络三张关键结果卡，明确 Lightpanda 在本轮低消耗指标中的领先。", "资源领先与综合任务通过率分别判断，不合成总分。"], + ["逐项比较", "表格只突出排序列，没有稳定标记每项最低值。", "按计算、内存、网络分组，逐项标记最低消耗；CPU 利用率和传输速率只描述运行形态，不按低值评选。", "沿用四种本地方法共同通过的 15/20 个普通网页，不产生新的供应商调用。"], + ["发布门禁", "资源展示要求依赖人工记忆。", "仓库规则和报告检查同时要求首页、解读页及中英文版具备赢家摘要、三类分组和逐项赢家标记。", "以后新增或重建报告时缺失任一部分都会阻断发布。"] + ] + , "i18n": { "en": { "title": "Group resource comparisons and name the leaders", "summary": "The home page and interpretation split CPU, memory, and network into three metric groups, mark each lower-is-better minimum directly, and identify Lightpanda as the resource-efficiency leader in this run. Evaluation evidence, task verdicts, and denominators are unchanged.", "changes": [ - [ - "Resource findings", - "Resource data was spread across one wide table, requiring readers to find minima themselves.", - "Add a resource-efficiency finding card and three key-result cards for CPU, memory, and network, making Lightpanda's lower-use lead explicit.", - "Resource leadership and overall task pass rate are judged separately and are not combined into a composite score." - ], - [ - "Per-metric comparison", - "The table highlighted only the sorted column and did not persistently mark each metric's minimum.", - "Group metrics into compute, memory, and network and mark the lowest use in each; CPU utilization and transfer rates describe runtime behavior and are not ranked by lower values.", - "Retain the 15/20 general pages passed by all four local methods, with no new provider calls." - ], - [ - "Release gate", - "Resource-presentation requirements depended on manual memory.", - "Repository rules and report checks now require a winner summary, three grouped sections, and per-metric winner markers on the home and interpretation pages in both languages.", - "Future report creation or rebuilding is blocked if any of these elements is missing." - ] + ["Resource findings", "Resource data was spread across one wide table, requiring readers to find minima themselves.", "Add a resource-efficiency finding card and three key-result cards for CPU, memory, and network, making Lightpanda's lower-use lead explicit.", "Resource leadership and overall task pass rate are judged separately and are not combined into a composite score."], + ["Per-metric comparison", "The table highlighted only the sorted column and did not persistently mark each metric's minimum.", "Group metrics into compute, memory, and network and mark the lowest use in each; CPU utilization and transfer rates describe runtime behavior and are not ranked by lower values.", "Retain the 15/20 general pages passed by all four local methods, with no new provider calls."], + ["Release gate", "Resource-presentation requirements depended on manual memory.", "Repository rules and report checks now require a winner summary, three grouped sections, and per-metric winner markers on the home and interpretation pages in both languages.", "Future report creation or rebuilding is blocked if any of these elements is missing."] ] } } @@ -4038,69 +3112,27 @@ { "id": "2026-09-14-five-method-resource-profile", "publication_state": "published", - "comparison_kinds": [ - "headless_html", - "headless_markdown" - ], + "comparison_kinds": ["headless_html", "headless_markdown"], "date": "2026-09-14", "scoring_revision_input_sha256": "8e8edd5c6e327010b480d98df95e949589491e5b5d91963641d58303bc735f98", "title": "增加 CPU、内存、网络实测并收敛 Moli 版本", "summary": "当前主报告只保留 Moli 1.1.1，并为四种可本地观测的方法增加统一隔离的 CPU、内存和网络比较。资源结果来自 20 题 × 4 方法的 80 次正式调用，汇总使用共同通过的 15 题。", "changes": [ - [ - "当前方法矩阵", - "三个 Moli 版本同时展示会扩大后续全量重跑和复核范围，也偏离当前产品比较目标。", - "主报告保留 Kitesurf、Lightpanda、Obscura、Moli 1.1.1、Chromium 五种方法；Moli 1.0.6 和 1.1.3 的历史证据继续保留，但不进入当前页面、分母和榜单。", - "主矩阵复用已经完成并核验的 750 条五方法结果，没有重新调用供应商；同窗测速的共同成功分母由七配置的 8 题变为当前五配置的 9 题。" - ], - [ - "资源比较", - "旧报告只有端到端耗时，无法区分浏览器客户端对 CPU、内存和网络的占用。", - "预声明 20 个普通网页，单并发运行四种本地方法；每次调用限制为 2 CPU、2 GiB 内存和 512 PID，并保存累计 CPU、内存采样及网络收发计数。", - "只在四种方法共同通过的 15 题上汇总；失败结果仍保留在逐次证据中但不用于资源排名。" - ], - [ - "隔离与复用", - "宿主机进程采样会混入其他任务，旧版成品镜像的 Moli 和 Chromium 版本又与当前口径不一致。", - "抽出共享的 Linux/cgroup 资源计量与容器限制组件，Web Fetch 每次使用一次性容器和专用网络；Browser Use 复用相同基础计量。固定资源镜像构建一次并按镜像 ID 复用。", - "现有 Docker 下载层可复用；旧 Moli 1.1.3 和 Chromium 151 成品镜像不冒充当前 Moli 1.1.1 与 Chromium 152。" - ], - [ - "适用边界", - "远程 Kitesurf 的客户端请求流量不能代表其浏览器服务端的 CPU、内存和上游网络。", - "Kitesurf 继续参加正确率和端到端耗时比较，但从资源消耗表中排除并说明原因。", - "50 ms 采样可能遗漏更短峰值；本轮只能解释 arm64 Linux 隔离容器中的客户端成本。" - ] - ], + ["当前方法矩阵", "三个 Moli 版本同时展示会扩大后续全量重跑和复核范围，也偏离当前产品比较目标。", "主报告保留 Kitesurf、Lightpanda、Obscura、Moli 1.1.1、Chromium 五种方法；Moli 1.0.6 和 1.1.3 的历史证据继续保留，但不进入当前页面、分母和榜单。", "主矩阵复用已经完成并核验的 750 条五方法结果，没有重新调用供应商；同窗测速的共同成功分母由七配置的 8 题变为当前五配置的 9 题。"], + ["资源比较", "旧报告只有端到端耗时，无法区分浏览器客户端对 CPU、内存和网络的占用。", "预声明 20 个普通网页，单并发运行四种本地方法；每次调用限制为 2 CPU、2 GiB 内存和 512 PID，并保存累计 CPU、内存采样及网络收发计数。", "只在四种方法共同通过的 15 题上汇总；失败结果仍保留在逐次证据中但不用于资源排名。"], + ["隔离与复用", "宿主机进程采样会混入其他任务，旧版成品镜像的 Moli 和 Chromium 版本又与当前口径不一致。", "抽出共享的 Linux/cgroup 资源计量与容器限制组件，Web Fetch 每次使用一次性容器和专用网络；Browser Use 复用相同基础计量。固定资源镜像构建一次并按镜像 ID 复用。", "现有 Docker 下载层可复用；旧 Moli 1.1.3 和 Chromium 151 成品镜像不冒充当前 Moli 1.1.1 与 Chromium 152。"], + ["适用边界", "远程 Kitesurf 的客户端请求流量不能代表其浏览器服务端的 CPU、内存和上游网络。", "Kitesurf 继续参加正确率和端到端耗时比较，但从资源消耗表中排除并说明原因。", "50 ms 采样可能遗漏更短峰值；本轮只能解释 arm64 Linux 隔离容器中的客户端成本。"] + ] + , "i18n": { "en": { "title": "Add measured CPU, memory, and network use and narrow the Moli version", "summary": "The current primary report retains only Moli 1.1.1 and adds a uniformly isolated CPU, memory, and network comparison for four locally observable methods. Resource results come from 80 formal calls across 20 tasks × 4 methods, aggregated over the 15 shared passes.", "changes": [ - [ - "Current method matrix", - "Showing three Moli versions would expand every future rerun and review and distract from the current product comparison.", - "The primary report retains Kitesurf, Lightpanda, Obscura, Moli 1.1.1, and Chromium. Historical evidence for Moli 1.0.6 and 1.1.3 remains archived but is excluded from current pages, denominators, and rankings.", - "The primary matrix reuses 750 completed and reviewed five-method results without new provider calls. Removing the two historical configurations changes the same-window shared-pass denominator from 8 tasks across seven configurations to 9 tasks across the current five." - ], - [ - "Resource comparison", - "The previous report had only end-to-end latency and could not distinguish browser-client CPU, memory, and network use.", - "20 general pages were predeclared and run one at a time across four local methods. Each call was limited to 2 CPUs, 2 GiB of memory, and 512 PIDs, with cumulative CPU, sampled memory, and network byte counters retained.", - "Aggregates use only the 15 tasks passed by all four methods. Failed results remain in per-attempt evidence but do not enter the resource ranking." - ], - [ - "Isolation and reuse", - "Host-process sampling can include unrelated work, while the existing finished images use Moli and Chromium versions that do not match the current contract.", - "Shared Linux/cgroup accounting and container-limit components now serve both evaluations. Web Fetch uses a disposable container and dedicated network per call, while Browser Use reuses the same accounting foundation. The fixed resource image is built once and reused by image ID.", - "Existing Docker download layers are reusable, but finished images containing Moli 1.1.3 and Chromium 151 are not presented as the current Moli 1.1.1 and Chromium 152 environment." - ], - [ - "Scope limits", - "Kitesurf's local request traffic cannot represent CPU, memory, or upstream network use in its remote browser service.", - "Kitesurf remains in correctness and end-to-end latency comparisons but is excluded from the resource-use table with the reason disclosed.", - "Sampling every 50 ms may miss shorter peaks. This run describes only client cost in isolated arm64 Linux containers." - ] + ["Current method matrix", "Showing three Moli versions would expand every future rerun and review and distract from the current product comparison.", "The primary report retains Kitesurf, Lightpanda, Obscura, Moli 1.1.1, and Chromium. Historical evidence for Moli 1.0.6 and 1.1.3 remains archived but is excluded from current pages, denominators, and rankings.", "The primary matrix reuses 750 completed and reviewed five-method results without new provider calls. Removing the two historical configurations changes the same-window shared-pass denominator from 8 tasks across seven configurations to 9 tasks across the current five."], + ["Resource comparison", "The previous report had only end-to-end latency and could not distinguish browser-client CPU, memory, and network use.", "20 general pages were predeclared and run one at a time across four local methods. Each call was limited to 2 CPUs, 2 GiB of memory, and 512 PIDs, with cumulative CPU, sampled memory, and network byte counters retained.", "Aggregates use only the 15 tasks passed by all four methods. Failed results remain in per-attempt evidence but do not enter the resource ranking."], + ["Isolation and reuse", "Host-process sampling can include unrelated work, while the existing finished images use Moli and Chromium versions that do not match the current contract.", "Shared Linux/cgroup accounting and container-limit components now serve both evaluations. Web Fetch uses a disposable container and dedicated network per call, while Browser Use reuses the same accounting foundation. The fixed resource image is built once and reused by image ID.", "Existing Docker download layers are reusable, but finished images containing Moli 1.1.3 and Chromium 151 are not presented as the current Moli 1.1.1 and Chromium 152 environment."], + ["Scope limits", "Kitesurf's local request traffic cannot represent CPU, memory, or upstream network use in its remote browser service.", "Kitesurf remains in correctness and end-to-end latency comparisons but is excluded from the resource-use table with the reason disclosed.", "Sampling every 50 ms may miss shorter peaks. This run describes only client cost in isolated arm64 Linux containers."] ] } } @@ -4109,9 +3141,7 @@ "id": "2026-09-15-official-route-and-call-drilldown", "publication_state": "published", "date": "2026-09-15", - "comparison_kinds": [ - "web_fetch_methods" - ], + "comparison_kinds": ["web_fetch_methods"], "scoring_revision_input_sha256": [ "596f8005399be74d73dd14b8a77f5f775b3e2dd36f037643c421259bb5d8ff53", "b8b7adec934761ed374191912c05ef2da449cb1a7cbda104f671db08d033a5ee" @@ -4119,84 +3149,25 @@ "title": "按任务修正官方接口并补齐逐次调用下钻", "summary": "十种方法仍执行同一组 150 题，保留 1,500 条权威结果。官方接口修正认证重跑 Cloudflare 23 条、Firecrawl 17 条，共 40 次调用；通过数由 1,024 增至 1,029。document-05 的公共格式归一化零调用增加 1 条通过；Cloudflare 两条视频链接任务又因适配器纠错增加 4 次认证调用。累计 44 次新物理调用，当前通过数 1,032/1,500；Moli 仍为 1.1.4。", "changes": [ - [ - "Cloudflare 任务接口", - "原 /markdown 适合网页正文；部分直接 PDF/DOCX 出现接口成功但正文为空，页面图片和视频属性也不能稳定从 Markdown 取得。", - "网页正文与动态题保留 Browser Run /markdown；直接 PDF/DOCX 上传原文件给 Workers AI /ai/tomarkdown；页面图片和视频地址使用 Browser Run /scrape 的 DOM 属性。下载原文件与转换共用一次端到端预算。", - "仅替换 Cloudflare 的 7 条直接文档、9 条页面图片和 7 条页面视频结果；/pdf 生成网页 PDF，/links 只列普通链接，/crawl 扩展到其他页面，均不替代这些题的交付物。" - ], - [ - "Cloudflare 视频地址适配", - "/scrape 已返回 Blender 的 .mp4.zip 地址和 Bilibili 的 BV 页面入口，旧适配器只接纳常见裸媒体扩展名，因而把目标地址丢掉；放宽后又把普通导航和工具 iframe 误列为视频。", - "下载包保留 .mp4.zip 形态；视频详情入口仅请求 a[href]，再按视频页面路由筛选，不用标准答案挑选输出。两条受影响任务均按纠正后的接口认证重跑。", - "video-blender-release 与 video-bilibili-current 最终均通过；Bilibili 输出只含站内 BV 详情地址。中间调用和旧物理证据保留，但不择优进入权威矩阵。" - ], - [ - "Firecrawl 任务接口", - "动态题的固定 waitFor 延迟不能执行已声明的顶层选择器条件；页面视频题漏取 Scrape 的 video 字段。", - "动态题预选 Scrape Actions 的 selector wait；视频资源题同时请求原生 video 格式，并保留正文和资源地址。匹配数量与 iframe 子文档完成条件仍按接口边界披露。", - "仅替换 10 条动态与 7 条页面视频结果。video 下载入口不等于媒体原字节或画面理解；Search、Crawl 与 Agent 不用于给定 URL 的单页任务。" - ], - [ - "成绩与同口径分母", - "只改结果表会掩盖接口和评分入口变化，也会让读者把重新调用与离线复算混为一谈。", - "40 个方法—题目组合换用官方接口；两条视频链接组合又因适配错误增加 4 次物理调用。44 次新调用完整保留，最终权威矩阵仍只含 40 条新调用结果与 1,460 条原调用结果。document-05 的 Markdown 表格分隔行在公共格式归一化中按排版处理，保存输出离线复评分。", - "通过数依次为 1,024/1,500、接口纠正后 1,029/1,500、零调用归一化后 1,030/1,500、两条视频适配纠错后 1,032/1,500。分母、150 题、十方法和 Moli 1.1.4 均不变。" - ], - [ - "核验与资源别名", - "缩略图可由同一资产的另一 Wikimedia 域名交付，单看主机名会错判 image-04；反过来，无缩略图标识的同名地址也不能任意放行。", - "40 条新结果和 1,460 条旧结果均有逐条独立核验；最后两条 Cloudflare 输出重新盲审，严格复用身份不变的 1,498 条核验。image-04 按同资产缩略图域名迁移裁决，公共资源规则同时拒绝缺少缩略图标识的近似地址。", - "当前 1,500/1,500 条权威结果有有效逐条判定核验，未决判定 0。image-04 修正未改变成绩；异常根因调查另有独立门禁。" - ], - [ - "逐次调用日志", - "旧详情能看到最终输出，但输出为空时，读者看不到接口状态、最后错误或每次重试的过程。", - "新物理调用保存脱敏且哈希绑定的请求摘要、接口/进程状态、最后输出、错误与重试记录；详情页按需打开弹窗。完整原始供应商响应仍留在私有证据，不随公开包发布。", - "本轮全部 40 次新调用有可下钻安全日志；历史调用只展示实际留存字段，并明确标示没有保存的日志，不补造历史 stdout、stderr 或原始信封。" - ] - ], + ["Cloudflare 任务接口", "原 /markdown 适合网页正文；部分直接 PDF/DOCX 出现接口成功但正文为空，页面图片和视频属性也不能稳定从 Markdown 取得。", "网页正文与动态题保留 Browser Run /markdown；直接 PDF/DOCX 上传原文件给 Workers AI /ai/tomarkdown；页面图片和视频地址使用 Browser Run /scrape 的 DOM 属性。下载原文件与转换共用一次端到端预算。", "仅替换 Cloudflare 的 7 条直接文档、9 条页面图片和 7 条页面视频结果；/pdf 生成网页 PDF，/links 只列普通链接，/crawl 扩展到其他页面，均不替代这些题的交付物。"], + ["Cloudflare 视频地址适配", "/scrape 已返回 Blender 的 .mp4.zip 地址和 Bilibili 的 BV 页面入口，旧适配器只接纳常见裸媒体扩展名，因而把目标地址丢掉；放宽后又把普通导航和工具 iframe 误列为视频。", "下载包保留 .mp4.zip 形态；视频详情入口仅请求 a[href]，再按视频页面路由筛选，不用标准答案挑选输出。两条受影响任务均按纠正后的接口认证重跑。", "video-blender-release 与 video-bilibili-current 最终均通过；Bilibili 输出只含站内 BV 详情地址。中间调用和旧物理证据保留，但不择优进入权威矩阵。"], + ["Firecrawl 任务接口", "动态题的固定 waitFor 延迟不能执行已声明的顶层选择器条件；页面视频题漏取 Scrape 的 video 字段。", "动态题预选 Scrape Actions 的 selector wait；视频资源题同时请求原生 video 格式，并保留正文和资源地址。匹配数量与 iframe 子文档完成条件仍按接口边界披露。", "仅替换 10 条动态与 7 条页面视频结果。video 下载入口不等于媒体原字节或画面理解；Search、Crawl 与 Agent 不用于给定 URL 的单页任务。"], + ["成绩与同口径分母", "只改结果表会掩盖接口和评分入口变化，也会让读者把重新调用与离线复算混为一谈。", "40 个方法—题目组合换用官方接口；两条视频链接组合又因适配错误增加 4 次物理调用。44 次新调用完整保留，最终权威矩阵仍只含 40 条新调用结果与 1,460 条原调用结果。document-05 的 Markdown 表格分隔行在公共格式归一化中按排版处理，保存输出离线复评分。", "通过数依次为 1,024/1,500、接口纠正后 1,029/1,500、零调用归一化后 1,030/1,500、两条视频适配纠错后 1,032/1,500。分母、150 题、十方法和 Moli 1.1.4 均不变。"], + ["核验与资源别名", "缩略图可由同一资产的另一 Wikimedia 域名交付，单看主机名会错判 image-04；反过来，无缩略图标识的同名地址也不能任意放行。", "40 条新结果和 1,460 条旧结果均有逐条独立核验；最后两条 Cloudflare 输出重新盲审，严格复用身份不变的 1,498 条核验。image-04 按同资产缩略图域名迁移裁决，公共资源规则同时拒绝缺少缩略图标识的近似地址。", "当前 1,500/1,500 条权威结果有有效逐条判定核验，未决判定 0。image-04 修正未改变成绩；异常根因调查另有独立门禁。"], + ["逐次调用日志", "旧详情能看到最终输出，但输出为空时，读者看不到接口状态、最后错误或每次重试的过程。", "新物理调用保存脱敏且哈希绑定的请求摘要、接口/进程状态、最后输出、错误与重试记录；详情页按需打开弹窗。完整原始供应商响应仍留在私有证据，不随公开包发布。", "本轮全部 40 次新调用有可下钻安全日志；历史调用只展示实际留存字段，并明确标示没有保存的日志，不补造历史 stdout、stderr 或原始信封。" ] + ] + , "i18n": { "en": { "title": "Correct official routes by task and add per-call drilldown", "summary": "The ten methods still run the same 150 tasks, with 1,500 authoritative results. Official-route corrections certified 23 Cloudflare and 17 Firecrawl calls, 40 in total, raising passes from 1,024 to 1,029. A shared format-normalization correction added 1 document-05 pass without a provider call. Two Cloudflare video-link tasks then required 4 additional certified adapter-correction calls. The release retains 44 new physical calls and now passes 1,032/1,500 tasks; Moli remains at 1.1.4.", "changes": [ - [ - "Cloudflare routes by task", - "The former /markdown route suits webpage text. Some direct PDF/DOCX requests returned a successful interface response with empty body text, and page image and video attributes could not be reliably recovered from Markdown.", - "Web text and dynamic tasks retain Browser Run /markdown. Direct PDF/DOCX files are uploaded to Workers AI /ai/tomarkdown; page image and video URLs come from Browser Run /scrape DOM attributes. Original-file download and conversion share one end-to-end deadline.", - "Only Cloudflare's 7 direct-document, 9 page-image, and 7 page-video results were replaced. /pdf generates a webpage PDF, /links lists ordinary links, and /crawl expands to other pages; none substitutes for the deliverables in these tasks." - ], - [ - "Cloudflare video-URL adaptation", - "/scrape had returned Blender's .mp4.zip download URL and Bilibili BV detail links, but the old adapter discarded them by requiring a bare media extension. A broad relaxation then mislabeled ordinary navigation and utility iframes as video entries.", - "The archive route keeps .mp4.zip downloads. Video-detail tasks request only a[href] and filter by video-page URL route, without selecting output against the reference answer. Both affected tasks were certified again under the corrected interface.", - "video-blender-release and video-bilibili-current both pass in the final matrix; the Bilibili output contains only on-site BV detail URLs. Intermediate and earlier physical calls remain retained but are not selected by score." - ], - [ - "Firecrawl routes by task", - "A fixed waitFor delay could not enforce the declared top-level selector for dynamic tasks; page-video tasks omitted Scrape's video field.", - "Dynamic tasks preselect Scrape Actions selector waits. Video-resource tasks also request the native video format while retaining text and resource URLs. Match-count and iframe-child completion conditions remain disclosed as interface limits.", - "Only 10 dynamic and 7 page-video results were replaced. A video download URL is neither original media bytes nor visual understanding. Search, Crawl, and Agent were not used for single-page tasks with a given URL." - ], - [ - "Results on the same denominator", - "Editing only a results table would conceal changes to provider routes and the shared scorer, and blur the distinction between new calls and offline rescoring.", - "40 method–task pairs use corrected official routes; two video-link pairs added 4 physical adapter-correction calls. All 44 new calls are retained, while the authoritative matrix contains 40 final replacement results and 1,460 inherited results. document-05's Markdown table-divider row is treated as layout by shared normalization and its saved output was rescored offline.", - "Passes moved from 1,024/1,500 to 1,029/1,500 after route corrections, 1,030/1,500 after zero-call normalization, and 1,032/1,500 after video-adapter corrections. The denominator, 150 tasks, ten methods, and Moli 1.1.4 are unchanged." - ], - [ - "Review and resource aliases", - "A thumbnail of the same asset may be served from another Wikimedia host; comparing hostnames alone would misjudge image-04. A similarly named URL without a thumbnail marker must not be accepted indiscriminately.", - "All 40 new and 1,460 inherited results have per-attempt independent review. The final two Cloudflare outputs were blindly reviewed again, strictly reusing 1,498 unchanged review identities. image-04 was adjudicated as a same-asset thumbnail-host migration, while the shared resource rule rejects a lookalike URL without the thumbnail path.", - "All 1,500/1,500 authoritative results have valid per-attempt verdict review, with 0 unresolved verdicts. The image-04 correction did not change scores; root-cause anomaly investigation has a separate gate." - ], - [ - "Per-call logs", - "Earlier detail pages showed final output, but an empty answer did not reveal interface status, the final error, or each retry.", - "New physical calls retain redacted, hash-bound request summaries, interface/process status, final output, errors, and retry records. The detail page opens this evidence on demand in a dialog. Full raw provider responses remain private and are not included in the public bundle.", - "All 40 new calls in this release have safe drilldown logs. Older calls show only fields actually retained and explicitly identify unavailable logs; historical stdout, stderr, and raw envelopes are not fabricated." - ] + ["Cloudflare routes by task", "The former /markdown route suits webpage text. Some direct PDF/DOCX requests returned a successful interface response with empty body text, and page image and video attributes could not be reliably recovered from Markdown.", "Web text and dynamic tasks retain Browser Run /markdown. Direct PDF/DOCX files are uploaded to Workers AI /ai/tomarkdown; page image and video URLs come from Browser Run /scrape DOM attributes. Original-file download and conversion share one end-to-end deadline.", "Only Cloudflare's 7 direct-document, 9 page-image, and 7 page-video results were replaced. /pdf generates a webpage PDF, /links lists ordinary links, and /crawl expands to other pages; none substitutes for the deliverables in these tasks."], + ["Cloudflare video-URL adaptation", "/scrape had returned Blender's .mp4.zip download URL and Bilibili BV detail links, but the old adapter discarded them by requiring a bare media extension. A broad relaxation then mislabeled ordinary navigation and utility iframes as video entries.", "The archive route keeps .mp4.zip downloads. Video-detail tasks request only a[href] and filter by video-page URL route, without selecting output against the reference answer. Both affected tasks were certified again under the corrected interface.", "video-blender-release and video-bilibili-current both pass in the final matrix; the Bilibili output contains only on-site BV detail URLs. Intermediate and earlier physical calls remain retained but are not selected by score."], + ["Firecrawl routes by task", "A fixed waitFor delay could not enforce the declared top-level selector for dynamic tasks; page-video tasks omitted Scrape's video field.", "Dynamic tasks preselect Scrape Actions selector waits. Video-resource tasks also request the native video format while retaining text and resource URLs. Match-count and iframe-child completion conditions remain disclosed as interface limits.", "Only 10 dynamic and 7 page-video results were replaced. A video download URL is neither original media bytes nor visual understanding. Search, Crawl, and Agent were not used for single-page tasks with a given URL."], + ["Results on the same denominator", "Editing only a results table would conceal changes to provider routes and the shared scorer, and blur the distinction between new calls and offline rescoring.", "40 method–task pairs use corrected official routes; two video-link pairs added 4 physical adapter-correction calls. All 44 new calls are retained, while the authoritative matrix contains 40 final replacement results and 1,460 inherited results. document-05's Markdown table-divider row is treated as layout by shared normalization and its saved output was rescored offline.", "Passes moved from 1,024/1,500 to 1,029/1,500 after route corrections, 1,030/1,500 after zero-call normalization, and 1,032/1,500 after video-adapter corrections. The denominator, 150 tasks, ten methods, and Moli 1.1.4 are unchanged."], + ["Review and resource aliases", "A thumbnail of the same asset may be served from another Wikimedia host; comparing hostnames alone would misjudge image-04. A similarly named URL without a thumbnail marker must not be accepted indiscriminately.", "All 40 new and 1,460 inherited results have per-attempt independent review. The final two Cloudflare outputs were blindly reviewed again, strictly reusing 1,498 unchanged review identities. image-04 was adjudicated as a same-asset thumbnail-host migration, while the shared resource rule rejects a lookalike URL without the thumbnail path.", "All 1,500/1,500 authoritative results have valid per-attempt verdict review, with 0 unresolved verdicts. The image-04 correction did not change scores; root-cause anomaly investigation has a separate gate."], + ["Per-call logs", "Earlier detail pages showed final output, but an empty answer did not reveal interface status, the final error, or each retry.", "New physical calls retain redacted, hash-bound request summaries, interface/process status, final output, errors, and retry records. The detail page opens this evidence on demand in a dialog. Full raw provider responses remain private and are not included in the public bundle.", "All 40 new calls in this release have safe drilldown logs. Older calls show only fields actually retained and explicitly identify unavailable logs; historical stdout, stderr, and raw envelopes are not fabricated."] ] } } @@ -4205,9 +3176,7 @@ "id": "2026-09-11-full-matrix-root-cause-audit", "publication_state": "published", "date": "2026-09-14", - "comparison_kinds": [ - "web_fetch_methods" - ], + "comparison_kinds": ["web_fetch_methods"], "scoring_revision_input_sha256": [ "9b85443f840063956126bc233584e4cbe1d884bde1eb0c59571ce02fdd419697", "3b49615c9eda1503e0952d64187fc39b13207478d40e03effa7100ba6ddc0d41", @@ -4232,156 +3201,37 @@ "title": "修复评测全链路根因并设置全矩阵核验门禁", "summary": "当前十种方法的 1,500 条主结果已全部独立核验，未决为 0；其中 309 条异常触发结果进一步完成输入、参考、请求、原始响应、适配器、评分器和最终判定七层审查。已修复正文分段、具名链接、脚本状态、文档地标、请求状态、动态卡片边界、公开路径泄漏和参考元数据等共因；仅认证重跑接口或产物身份受影响的声明范围，其余修正复用原始产物离线完成。Moli 保持 1.1.4。", "changes": [ - [ - "全量结果核验", - "上一版只独立审查机器失败，机器通过尚未逐条核验，无法发现被宽松规则放行的系统性假阳性。", - "机器通过与失败统一使用隐藏机器结论的审查包；任务和审查可见证据哈希均未变化的审查继续有效，只重审发生变化的结果。", - "当前十方法矩阵审查覆盖 1,500/1,500。修正前审查提出 56 条代理与机器分歧，其中 2 条确认评分错误并先修根因；其余均依据实际产物、标准答案和任务合同裁决，未决为 0。同窗测速只作时延测量，不在结果审查范围。" - ], - [ - "正文、动态与挑战页评分", - "旧评分实现可能在代码更新后继续被报告读取；分段正文按参考顺序贪心匹配会让较长实际片段先占用错误区域；序列化脚本状态含目标字符串时又可能被当成可读正文。", - "报告构建拒绝旧评分实现；正文片段改为全局容量分配且不受参考区域顺序影响；脚本或应用状态壳不能满足正文任务。", - "两类修正均通过公共 score_attempt 与任务判定入口执行正反向回归；保存输出统一离线重算，不按方法或单条结果增加特例。" - ], - [ - "Exa 具名链接接口", - "旧适配器只读取裸 links 数组，既无法证明名称与 URL 的归属，又漏用了官方返回明确 anchor—URL 配对的 richLinks。", - "仅具名链接任务请求 extras.richLinks，并只接纳 anchor 与 URL 均非空的明确配对；正文标题与裸 links 永不按位置猜配。", - "接口合同变化只影响 Exa 的 10 个挑战任务，因此通过认证流程重跑该范围；3 条通过、7 条仍因实际产物不满足合同而失败。" - ], - [ - "DOCX 完整内容", - "document-05 只要求标题和章节名，导致返回正文叙述却漏掉表格单元格、日历与最终尾注的结果仍可通过。", - "任务合同增加来自冻结源文档的表格、日历和尾注内容地标；Markdown 表格分隔符按排版处理，不能把完整表格误判失败。", - "保存产物统一离线复评分：CowAgent 的不完整结果改为失败，Firecrawl 与 Tavily 的完整 Markdown 表格保持通过；不需要重新调用供应商。" - ], - [ - "图片资源范围", - "图片地址题一度被拟议成 target-only 输出，可能把已经命中目标、但同时保留页面 Logo 或推荐图的有效抓取误判失败；部分响应式目标别名又需要独立源证据。", - "资源验收只要求命中任务声明的目标或有证据的合法别名；额外页面图片保留为纯净度诊断，不参与二元通过判定。目标清单、别名、来源观察和资源合同均进入哈希绑定。", - "避免把页面级抓取强行等同于只返回一个 URL，同时仍拒绝没有命中目标的长文本或大量无关图片；契约变化后旧分数不能发布。" - ], - [ - "防复发与证据修订", - "评分、参考、数据集和审查收据可分别更新，若生命周期只校验部分字段，旧核验可能被错误沿用；只改最终判定也无法阻止同类错误复发。", - "每类根因登记正反向可执行回归；参考修订和评分修订分别串联完整输入输出哈希、根因与回归编号，报告构建校验评分实现、任务内容、资源合同和来源证据。", - "评分与参考修订不重发 provider 请求；只有接口、版本、输入或请求合同改变的预声明范围才重跑，并保留原调用与新调用的可追溯关系。" - ], - [ - "Firecrawl 动态接口", - "旧适配器使用当前账户地区不允许的 actions，10 个动态任务都在服务入口失败，错误地形成 0/10 的能力结论。", - "改用当前公开接口支持的 waitFor，并通过调用前后参考冻结的认证流程只重跑这 10 个任务；其余 140 条 Firecrawl 结果不变。", - "新结果 6/10 通过；新增 10 条全部由隐藏机器结论的 Codex CLI 盲审，0 分歧、0 未决。剩余 4 条因实际输出不满足任务合同而失败。" - ], - [ - "报告阅读与耗时", - "全部方法共同成功任务的跨批次耗时表默认展开，占据总榜下方较大篇幅；综合总榜没有同时给出完整 150 次调用的时延分布。", - "共同成功任务耗时改为按需展开；综合总榜新增每种方法全部 150 次调用的均值、P50、P95，和通过率使用相同固定分母。", - "时延列不参与通过率排序，也不选择成功子集；失败调用耗时保留在全调用分布中。" - ], - [ - "请求状态与空输出", - "旧适配器把供应商调用成功但没有可验收正文，与供应商或目标请求失败混成同一种失败，导致请求成功率与任务通过率语义串线。", - "Cloudflare 依据保存的 success、result 与 origin status 信封归一化；Moli 依据 1.1.4 的退出状态和完整响应错误信封归一化。空的成功响应仍是请求成功、任务失败；明确的目标非 2xx 仍是请求失败。", - "14 条状态修正均由哈希绑定的原始证据离线重建，任务判定和分数不变，无 provider 调用。" - ], - [ - "动态提取与公开产物", - "Crawl4AI 动态购物车请求声明了限定提取，但 worker 未传入对应配置并丢失卡片边界；OpenClaw 错误堆栈和 CowAgent 保存提示又会把本机路径带入公开证据。", - "Crawl4AI worker 传入已声明的提取配置并保留卡片间边界；OpenClaw/DeepSeek 错误面只保留稳定消息，CowAgent 仅删除本机保存路径而保留抓取正文。", - "Crawl4AI 该题认证重跑后通过，方法总分增至 119/150；OpenClaw 15 条和 CowAgent 11 条受影响产物认证重跑，任务总分不变，公开产物不再包含真实主机路径。" - ], - [ - "参考证据一致性", - "四个完整正文样本的来源说明曾错误指向其他网站；NOAA 样本已有当日独立 200 响应、main 区域和完整正文哈希，却保留了“全文不可用”的旧备注。", - "完整正文来源必须与题目 URL 同站；NOAA 元数据改为明确记录完整参考可用，稳定锚点继续决定二元验收，全文快照只提供诊断指标。新增数据集门禁拒绝有效全文与不可用声明并存。", - "来源与备注修正不改变 provider 输出或任务分数；相关结果重新绑定并独立复核。" - ], - [ - "异常根因门禁", - "仅比较标准答案与最终判定无法主动发现空输出、跨站相同错误、集中零分、接口误用、参考冲突和公开产物泄漏；按批次序号缓存又会在小修后错配审查结果。", - "逐条盲审之后，对确定性异常信号执行七层调查；异常缓存按精确 attempt 范围而不是批次位置复用，范围变化时只重审变化组。", - "最终矩阵 1,500/1,500 已核验，309 条异常触发结果全部完成调查，未决 0；缓存范围、参考修订和最终审查均有哈希绑定。" - ] - ], + ["全量结果核验", "上一版只独立审查机器失败，机器通过尚未逐条核验，无法发现被宽松规则放行的系统性假阳性。", "机器通过与失败统一使用隐藏机器结论的审查包；任务和审查可见证据哈希均未变化的审查继续有效，只重审发生变化的结果。", "当前十方法矩阵审查覆盖 1,500/1,500。修正前审查提出 56 条代理与机器分歧，其中 2 条确认评分错误并先修根因；其余均依据实际产物、标准答案和任务合同裁决，未决为 0。同窗测速只作时延测量，不在结果审查范围。"], + ["正文、动态与挑战页评分", "旧评分实现可能在代码更新后继续被报告读取；分段正文按参考顺序贪心匹配会让较长实际片段先占用错误区域；序列化脚本状态含目标字符串时又可能被当成可读正文。", "报告构建拒绝旧评分实现；正文片段改为全局容量分配且不受参考区域顺序影响；脚本或应用状态壳不能满足正文任务。", "两类修正均通过公共 score_attempt 与任务判定入口执行正反向回归；保存输出统一离线重算，不按方法或单条结果增加特例。"], + ["Exa 具名链接接口", "旧适配器只读取裸 links 数组，既无法证明名称与 URL 的归属，又漏用了官方返回明确 anchor—URL 配对的 richLinks。", "仅具名链接任务请求 extras.richLinks，并只接纳 anchor 与 URL 均非空的明确配对；正文标题与裸 links 永不按位置猜配。", "接口合同变化只影响 Exa 的 10 个挑战任务，因此通过认证流程重跑该范围；3 条通过、7 条仍因实际产物不满足合同而失败。"], + ["DOCX 完整内容", "document-05 只要求标题和章节名，导致返回正文叙述却漏掉表格单元格、日历与最终尾注的结果仍可通过。", "任务合同增加来自冻结源文档的表格、日历和尾注内容地标；Markdown 表格分隔符按排版处理，不能把完整表格误判失败。", "保存产物统一离线复评分：CowAgent 的不完整结果改为失败，Firecrawl 与 Tavily 的完整 Markdown 表格保持通过；不需要重新调用供应商。"], + ["图片资源范围", "图片地址题一度被拟议成 target-only 输出，可能把已经命中目标、但同时保留页面 Logo 或推荐图的有效抓取误判失败；部分响应式目标别名又需要独立源证据。", "资源验收只要求命中任务声明的目标或有证据的合法别名；额外页面图片保留为纯净度诊断，不参与二元通过判定。目标清单、别名、来源观察和资源合同均进入哈希绑定。", "避免把页面级抓取强行等同于只返回一个 URL，同时仍拒绝没有命中目标的长文本或大量无关图片；契约变化后旧分数不能发布。"], + ["防复发与证据修订", "评分、参考、数据集和审查收据可分别更新，若生命周期只校验部分字段，旧核验可能被错误沿用；只改最终判定也无法阻止同类错误复发。", "每类根因登记正反向可执行回归；参考修订和评分修订分别串联完整输入输出哈希、根因与回归编号，报告构建校验评分实现、任务内容、资源合同和来源证据。", "评分与参考修订不重发 provider 请求；只有接口、版本、输入或请求合同改变的预声明范围才重跑，并保留原调用与新调用的可追溯关系。"], + ["Firecrawl 动态接口", "旧适配器使用当前账户地区不允许的 actions，10 个动态任务都在服务入口失败，错误地形成 0/10 的能力结论。", "改用当前公开接口支持的 waitFor，并通过调用前后参考冻结的认证流程只重跑这 10 个任务；其余 140 条 Firecrawl 结果不变。", "新结果 6/10 通过；新增 10 条全部由隐藏机器结论的 Codex CLI 盲审，0 分歧、0 未决。剩余 4 条因实际输出不满足任务合同而失败。"], + ["报告阅读与耗时", "全部方法共同成功任务的跨批次耗时表默认展开，占据总榜下方较大篇幅；综合总榜没有同时给出完整 150 次调用的时延分布。", "共同成功任务耗时改为按需展开；综合总榜新增每种方法全部 150 次调用的均值、P50、P95，和通过率使用相同固定分母。", "时延列不参与通过率排序，也不选择成功子集；失败调用耗时保留在全调用分布中。"], + ["请求状态与空输出", "旧适配器把供应商调用成功但没有可验收正文，与供应商或目标请求失败混成同一种失败，导致请求成功率与任务通过率语义串线。", "Cloudflare 依据保存的 success、result 与 origin status 信封归一化；Moli 依据 1.1.4 的退出状态和完整响应错误信封归一化。空的成功响应仍是请求成功、任务失败；明确的目标非 2xx 仍是请求失败。", "14 条状态修正均由哈希绑定的原始证据离线重建，任务判定和分数不变，无 provider 调用。"], + ["动态提取与公开产物", "Crawl4AI 动态购物车请求声明了限定提取，但 worker 未传入对应配置并丢失卡片边界；OpenClaw 错误堆栈和 CowAgent 保存提示又会把本机路径带入公开证据。", "Crawl4AI worker 传入已声明的提取配置并保留卡片间边界；OpenClaw/DeepSeek 错误面只保留稳定消息，CowAgent 仅删除本机保存路径而保留抓取正文。", "Crawl4AI 该题认证重跑后通过，方法总分增至 119/150；OpenClaw 15 条和 CowAgent 11 条受影响产物认证重跑，任务总分不变，公开产物不再包含真实主机路径。"], + ["参考证据一致性", "四个完整正文样本的来源说明曾错误指向其他网站；NOAA 样本已有当日独立 200 响应、main 区域和完整正文哈希，却保留了“全文不可用”的旧备注。", "完整正文来源必须与题目 URL 同站；NOAA 元数据改为明确记录完整参考可用，稳定锚点继续决定二元验收，全文快照只提供诊断指标。新增数据集门禁拒绝有效全文与不可用声明并存。", "来源与备注修正不改变 provider 输出或任务分数；相关结果重新绑定并独立复核。"], + ["异常根因门禁", "仅比较标准答案与最终判定无法主动发现空输出、跨站相同错误、集中零分、接口误用、参考冲突和公开产物泄漏；按批次序号缓存又会在小修后错配审查结果。", "逐条盲审之后，对确定性异常信号执行七层调查；异常缓存按精确 attempt 范围而不是批次位置复用，范围变化时只重审变化组。", "最终矩阵 1,500/1,500 已核验，309 条异常触发结果全部完成调查，未决 0；缓存范围、参考修订和最终审查均有哈希绑定。"] + ] + , "i18n": { "en": { "title": "Fix root causes across the evaluation chain and require full-matrix review", "summary": "All 1,500 primary results across ten methods were independently reviewed with 0 unresolved decisions. The 309 anomaly-triggered results also completed seven-layer review covering input, reference, request, raw response, adapter, scorer, and final verdict. Shared fixes cover body segmentation, named links, script-state text, document landmarks, request status, dynamic-card boundaries, public path leakage, and reference metadata. Only declared scopes whose interface or artifact identity changed were rerun; all other corrections reused saved outputs offline. Moli remains on 1.1.4.", "changes": [ - [ - "Full result review", - "The previous release independently reviewed machine failures only. Machine passes had not been checked individually, leaving systematic false positives from permissive rules undetectable.", - "Machine passes and failures use the same packets with machine judgments hidden. Prior reviews remain valid only when both the task and reviewer-visible evidence hash are unchanged; only changed results are reviewed again.", - "Review covers 1,500/1,500 results in the current ten-method matrix. Before correction, the reviewer raised 56 disagreements with machine judgments; 2 confirmed scoring errors whose root causes were fixed first, and the remainder were adjudicated against actual artifacts, reference answers, and task contracts. There are 0 unresolved items. The same-window cohort remains latency-only and is outside result review." - ], - [ - "Body, dynamic-task, and challenge-page scoring", - "A report could keep reading stale scorer output after code changes; greedy body matching in reference order could let longer returned fragments consume the wrong region first; and serialized script state containing target strings could be mistaken for readable body text.", - "Report builds reject stale scorer output; body fragments now use global capacity allocation independent of reference-region order; and script or application-state shells cannot satisfy body tasks.", - "Both corrections have positive and negative regressions executed through the shared score_attempt and task-decision paths. Saved output is consistently rescored offline without provider-specific or row-specific exceptions." - ], - [ - "Exa named-link interface", - "The old adapter read only the bare links array, which could not prove name-to-URL ownership and omitted official richLinks with explicit anchor–URL pairs.", - "Only named-link tasks request extras.richLinks, and only explicit pairs with nonempty anchors and URLs are accepted. Body headings and bare links are never paired by position.", - "The interface-contract change affects only Exa's 10 challenge tasks, so the certified rerun was limited to that scope. 3 pass; 7 still fail because the actual artifacts do not satisfy the contract." - ], - [ - "Complete DOCX content", - "document-05 required only the title and section names, allowing narrative output that omitted table cells, calendar content, and the final notes to pass.", - "The task contract now includes table, calendar, and note landmarks from the frozen source document. Markdown table separators are treated as layout so complete tables are not falsely rejected.", - "Saved artifacts were consistently rescored offline: CowAgent's incomplete result now fails, while Firecrawl and Tavily retain passes for complete Markdown tables. No provider calls were needed." - ], - [ - "Image-resource scope", - "Image-URL tasks were briefly proposed as target-only output, which could fail a valid page fetch that found the target while retaining logos or recommendation images. Some responsive target aliases also require independent source evidence.", - "Resource acceptance requires a declared target or an evidenced valid alias. Additional page images remain a precision diagnostic and do not affect binary acceptance. Target lists, aliases, source observations, and resource contracts are all hash-bound.", - "This avoids treating a page-level fetch as if it had to return only one URL, while still rejecting long text or many unrelated images that never match the target. Stale scores cannot be published after a contract change." - ], - [ - "Recurrence prevention and evidence revision", - "Scores, references, datasets, and review receipts can change independently. Partial lifecycle validation could wrongly reuse an old review, while patching only a final verdict cannot prevent recurrence.", - "Each root cause has executable controls in both directions. Reference and scoring revisions separately chain complete input and output hashes, root causes, and regression IDs; report builds validate scorer identity, task content, resource contracts, and source evidence.", - "Scoring and reference revisions do not resend provider requests. Only predeclared scopes affected by an interface, version, input, or request-contract change are rerun, with traceability preserved between old and new calls." - ], - [ - "Firecrawl dynamic interface", - "The old adapter used actions, which the current account region does not allow. All 10 dynamic tasks failed at the service boundary, incorrectly producing a 0/10 capability result.", - "The adapter now uses waitFor, supported by the current public interface. A certified run with references frozen before and after calls reran only these 10 tasks; the other 140 Firecrawl results are unchanged.", - "The new results pass 6/10. Codex CLI blindly reviewed all 10 additions with machine judgments hidden, finding 0 disagreements and 0 unresolved items. The remaining 4 fail because their actual output does not satisfy the task contract." - ], - [ - "Report reading and latency", - "The cross-batch latency table for tasks passed by every method was expanded by default and occupied substantial space below the ranking; the overall ranking did not show latency over all 150 calls.", - "Shared-success latency is now collapsed on demand. The overall ranking adds mean, P50, and P95 over all 150 calls per method, using the same fixed denominator as coverage.", - "Latency columns do not affect coverage ordering or select a successful subset; failed-call elapsed time remains in the all-call distribution." - ], - [ - "Request status and empty output", - "The old adapters conflated a successful provider call with no acceptable body and a provider or target request failure, mixing request-success and task-pass semantics.", - "Cloudflare now normalizes saved success, result, and origin-status envelopes; Moli uses the 1.1.4 exit status and whole-response error envelope. An empty successful response remains request success but task failure, while an explicit target non-2xx remains request failure.", - "All 14 status corrections were reconstructed offline from hash-bound raw evidence. Task verdicts and scores did not change, and no provider was called." - ], - [ - "Dynamic extraction and public artifacts", - "The Crawl4AI dynamic-cart request declared bounded extraction, but the worker did not pass the configuration and lost card boundaries. OpenClaw error stacks and CowAgent save notices could also expose host-local paths in public evidence.", - "The Crawl4AI worker now passes the declared extraction configuration and preserves card boundaries. OpenClaw and DeepSeek error surfaces retain only stable messages, while CowAgent removes only host-local save paths and keeps fetched text.", - "The certified Crawl4AI rerun passes and raises its total to 119/150. Certified reruns covered 15 affected OpenClaw artifacts and 11 CowAgent artifacts without changing their task totals; public artifacts no longer contain real host paths." - ], - [ - "Reference-evidence consistency", - "Four complete-body samples had source descriptions pointing to unrelated sites. The NOAA sample had a same-day independent 200 response, main-region extraction, and complete-body hash but retained a stale note saying the full text was unavailable.", - "Complete-body evidence must use the task URL's site. NOAA metadata now explicitly records the complete reference as available; stable anchors still drive binary acceptance and the full snapshot supplies diagnostic metrics. A dataset guard rejects valid full text paired with an unavailable declaration.", - "Source and note corrections do not change provider output or task scores; affected results were rebound and independently reviewed." - ], - [ - "Anomaly root-cause gate", - "Comparing only a reference and final verdict cannot proactively find empty output, repeated cross-site errors, concentrated zero scores, interface misuse, reference conflicts, or public-artifact leakage. Caching by batch number also misaligns reviews after small revisions.", - "After row-level blind review, deterministic anomaly signals receive a seven-layer investigation. Anomaly caches are reused by exact attempt scope rather than batch position, so only changed groups are reviewed again.", - "The final matrix is reviewed 1,500/1,500; all 309 anomaly-triggered results completed investigation with 0 unresolved items. Cache scope, reference revisions, and final review are hash-bound." - ] + ["Full result review", "The previous release independently reviewed machine failures only. Machine passes had not been checked individually, leaving systematic false positives from permissive rules undetectable.", "Machine passes and failures use the same packets with machine judgments hidden. Prior reviews remain valid only when both the task and reviewer-visible evidence hash are unchanged; only changed results are reviewed again.", "Review covers 1,500/1,500 results in the current ten-method matrix. Before correction, the reviewer raised 56 disagreements with machine judgments; 2 confirmed scoring errors whose root causes were fixed first, and the remainder were adjudicated against actual artifacts, reference answers, and task contracts. There are 0 unresolved items. The same-window cohort remains latency-only and is outside result review."], + ["Body, dynamic-task, and challenge-page scoring", "A report could keep reading stale scorer output after code changes; greedy body matching in reference order could let longer returned fragments consume the wrong region first; and serialized script state containing target strings could be mistaken for readable body text.", "Report builds reject stale scorer output; body fragments now use global capacity allocation independent of reference-region order; and script or application-state shells cannot satisfy body tasks.", "Both corrections have positive and negative regressions executed through the shared score_attempt and task-decision paths. Saved output is consistently rescored offline without provider-specific or row-specific exceptions."], + ["Exa named-link interface", "The old adapter read only the bare links array, which could not prove name-to-URL ownership and omitted official richLinks with explicit anchor–URL pairs.", "Only named-link tasks request extras.richLinks, and only explicit pairs with nonempty anchors and URLs are accepted. Body headings and bare links are never paired by position.", "The interface-contract change affects only Exa's 10 challenge tasks, so the certified rerun was limited to that scope. 3 pass; 7 still fail because the actual artifacts do not satisfy the contract."], + ["Complete DOCX content", "document-05 required only the title and section names, allowing narrative output that omitted table cells, calendar content, and the final notes to pass.", "The task contract now includes table, calendar, and note landmarks from the frozen source document. Markdown table separators are treated as layout so complete tables are not falsely rejected.", "Saved artifacts were consistently rescored offline: CowAgent's incomplete result now fails, while Firecrawl and Tavily retain passes for complete Markdown tables. No provider calls were needed."], + ["Image-resource scope", "Image-URL tasks were briefly proposed as target-only output, which could fail a valid page fetch that found the target while retaining logos or recommendation images. Some responsive target aliases also require independent source evidence.", "Resource acceptance requires a declared target or an evidenced valid alias. Additional page images remain a precision diagnostic and do not affect binary acceptance. Target lists, aliases, source observations, and resource contracts are all hash-bound.", "This avoids treating a page-level fetch as if it had to return only one URL, while still rejecting long text or many unrelated images that never match the target. Stale scores cannot be published after a contract change."], + ["Recurrence prevention and evidence revision", "Scores, references, datasets, and review receipts can change independently. Partial lifecycle validation could wrongly reuse an old review, while patching only a final verdict cannot prevent recurrence.", "Each root cause has executable controls in both directions. Reference and scoring revisions separately chain complete input and output hashes, root causes, and regression IDs; report builds validate scorer identity, task content, resource contracts, and source evidence.", "Scoring and reference revisions do not resend provider requests. Only predeclared scopes affected by an interface, version, input, or request-contract change are rerun, with traceability preserved between old and new calls."], + ["Firecrawl dynamic interface", "The old adapter used actions, which the current account region does not allow. All 10 dynamic tasks failed at the service boundary, incorrectly producing a 0/10 capability result.", "The adapter now uses waitFor, supported by the current public interface. A certified run with references frozen before and after calls reran only these 10 tasks; the other 140 Firecrawl results are unchanged.", "The new results pass 6/10. Codex CLI blindly reviewed all 10 additions with machine judgments hidden, finding 0 disagreements and 0 unresolved items. The remaining 4 fail because their actual output does not satisfy the task contract."], + ["Report reading and latency", "The cross-batch latency table for tasks passed by every method was expanded by default and occupied substantial space below the ranking; the overall ranking did not show latency over all 150 calls.", "Shared-success latency is now collapsed on demand. The overall ranking adds mean, P50, and P95 over all 150 calls per method, using the same fixed denominator as coverage.", "Latency columns do not affect coverage ordering or select a successful subset; failed-call elapsed time remains in the all-call distribution."], + ["Request status and empty output", "The old adapters conflated a successful provider call with no acceptable body and a provider or target request failure, mixing request-success and task-pass semantics.", "Cloudflare now normalizes saved success, result, and origin-status envelopes; Moli uses the 1.1.4 exit status and whole-response error envelope. An empty successful response remains request success but task failure, while an explicit target non-2xx remains request failure.", "All 14 status corrections were reconstructed offline from hash-bound raw evidence. Task verdicts and scores did not change, and no provider was called."], + ["Dynamic extraction and public artifacts", "The Crawl4AI dynamic-cart request declared bounded extraction, but the worker did not pass the configuration and lost card boundaries. OpenClaw error stacks and CowAgent save notices could also expose host-local paths in public evidence.", "The Crawl4AI worker now passes the declared extraction configuration and preserves card boundaries. OpenClaw and DeepSeek error surfaces retain only stable messages, while CowAgent removes only host-local save paths and keeps fetched text.", "The certified Crawl4AI rerun passes and raises its total to 119/150. Certified reruns covered 15 affected OpenClaw artifacts and 11 CowAgent artifacts without changing their task totals; public artifacts no longer contain real host paths."], + ["Reference-evidence consistency", "Four complete-body samples had source descriptions pointing to unrelated sites. The NOAA sample had a same-day independent 200 response, main-region extraction, and complete-body hash but retained a stale note saying the full text was unavailable.", "Complete-body evidence must use the task URL's site. NOAA metadata now explicitly records the complete reference as available; stable anchors still drive binary acceptance and the full snapshot supplies diagnostic metrics. A dataset guard rejects valid full text paired with an unavailable declaration.", "Source and note corrections do not change provider output or task scores; affected results were rebound and independently reviewed."], + ["Anomaly root-cause gate", "Comparing only a reference and final verdict cannot proactively find empty output, repeated cross-site errors, concentrated zero scores, interface misuse, reference conflicts, or public-artifact leakage. Caching by batch number also misaligns reviews after small revisions.", "After row-level blind review, deterministic anomaly signals receive a seven-layer investigation. Anomaly caches are reused by exact attempt scope rather than batch position, so only changed groups are reviewed again.", "The final matrix is reviewed 1,500/1,500; all 309 anomaly-triggered results completed investigation with 0 unresolved items. Cache scope, reference revisions, and final review are hash-bound."] ] } } @@ -4390,9 +3240,7 @@ "id": "2026-09-10-twelve-method-refresh-contract", "publication_state": "published", "date": "2026-09-10", - "comparison_kinds": [ - "web_fetch_methods" - ], + "comparison_kinds": ["web_fetch_methods"], "scoring_revision_input_sha256": [ "f167652c1225f33829123a7ef9f75280a610d104e76ff4752f982c3ec3b89e0b", "0362d27e8bf8418c79a255e144fd146d7450699a753f4eb00b8f38963ab8b526", @@ -4401,108 +3249,29 @@ "title": "完成十二方法升级、影响范围重跑与失败项核验", "summary": "十二方法报告已重新绑定当前评测集、实现身份和独立发布目标。主矩阵按预声明影响范围新增 432 次调用，同窗测速新增 240 次；主矩阵 705 条机器失败结果完成独立核验，1,095 条机器通过结果未逐条审查，分歧逐条裁决，未决为 0。", "changes": [ - [ - "评测对象", - "默认报告一度指向七种浏览器方法，旧十二方法页面又保留了已替换的易变正文样本，无法作为本轮十二方法权威矩阵。", - "当前 profile 恢复十二种 Web Fetch 方法和 150 个任务；八个易变全文样本改为 RFC、固定 Git 提交、W3C 定版文档或 Wikipedia 固定修订，并保留明确的替换关系。", - "最终权威矩阵仍为 1,800 个方法—任务逻辑结果；被替换题目及另外五个受参考或实现影响的题目须按当前合同重跑，旧结果不直接沿用。" - ], - [ - "实现身份与前置门禁", - "旧证据没有完整固定 CowAgent、OpenClaw、DeepSeek Harness、LangChain 与 Crawl4AI 的源码或依赖身份，空路径或错误版本还可能在证据目录创建后才失败。", - "源码型方法固定仓库修订、Node 版本和必需文件；LangChain WebBaseLoader 0.3.30、Crawl4AI 0.9.3 及文档解析依赖进入 uv.lock。所有方法在创建运行证据前完成身份、命令和路径校验，身份摘要写入 manifest。", - "LangChain 与 Crawl4AI 因旧权威证据未记录库版本，各自完整重跑 150 题；源码型方法只有精确且干净的绑定实现进入本轮。" - ], - [ - "直接文档路线", - "通用 Moli 适配器把直接 PDF 或 DOCX 正文任务当作普通网页 Markdown，可能在取得原始文档前失败或把二进制响应序列化。", - "直接 PDF、DOCX 与文件任务预先选择 Moli 自动原始响应；正文任务在同一个 45 秒预算内解析文字并保留原始字节哈希，文件任务继续按字节验收，不增加跨方法下载后备。", - "受路线影响的文档结果已进入声明的重跑范围；评分与盲审均使用新保存输出。" - ], - [ - "偶发下载证据", - "正文抓取若同时触发浏览器下载，旧 runner 因二进制没有文件任务验收记录而中止整批，导致后续结果缺失。", - "调用边界现在会保存并哈希所有返回字节；只有文件任务可用这些字节通过验收，正文任务仍必须交付合格文字。触发故障的 Crawl4AI 批次已整批作废并重跑。", - "修复只保证证据完整和运行可继续，不把偶发下载当成正文能力，也不复用崩溃前的部分批次。" - ], - [ - "影响范围与测速", - "仅补跑旧失败或只重建页面会混入未知实现版本，也无法公平比较来自不同批次的十二方法耗时。", - "主矩阵预先声明 13 题 × 12 方法、LangChain 与 Crawl4AI 的其余完整方法 cohort，以及两个超时边界修正，共完成 432 次新调用；另以 provider-blind、域名去重的 20 题完成 240 次十二方法同窗测速。", - "主矩阵固定每个方法—任务一个权威逻辑结果，不按最好或最快挑选。测速保留完整预声明矩阵：全调用延迟按同一 20 题分母统计，成功调用延迟另作条件统计；十二方法共同通过为 1/20，只作为诊断，不据此筛样本或排名。" - ], - [ - "证据与失败项核验", - "旧组装器可能把 worker 工作树和符号链接带入汇总，又会漏掉重试物理调用返回的辅助文件；审查包若遗漏原生答案字段，也会制造系统性假分歧。", - "汇总只复制哈希绑定的公开文件，拒绝符号链接和工作树路径，并绑定主返回、辅助文件与每次物理重试。Codex gpt-5.6-sol 对主矩阵 705 条机器失败结果盲审，审查包同时提供白名单化的原生答案 metadata；2 条分歧逐条裁决。", - "最终证据为 12 × 150 的唯一主矩阵和 12 × 20 的唯一测速矩阵；主矩阵失败项核验 705/705，机器通过未逐条审查 1,095 条，测速结果不在本次审查范围，未决 0。裁决只确定审计结论，不回写或择优替换 provider 输出。" - ], - [ - "报告与发布隔离", - "旧生成器混用了 150 题、六类和七浏览器专属叙述，方法页重复 Exa、Tavily、Firecrawl 配置，通用报告还可能显示 headless 修订史、复现链接和中文延迟加载提示。", - "同一 renderer 改由当前 dataset、records 和实际收据生成数量、方法路线、修订说明与链接；十二方法与 headless 历史按 comparison kind 隔离，中英文虚拟路由分别显示对应加载及无障碍文案。当前十二方法构建绑定新的 dev 域站点，旧十二方法站点与 app 域浏览器参考报告保持不变。", - "发布器逐文件核验公开文件；服务端允许的 HTML 外壳注入与匿名字节一致性分层披露，不再把两者混称为全站字节相同。" - ], - [ - "环境导入", - "托管方法凭据已存在于用户级 ~/.env，但旧流程只读取仓库 .env，导致新会话会误报缺少变量。", - "新增显式 import-env：仅将九个允许变量从用户级文件补入仓库 .env，保留已有非空值、隐藏变量内容并强制 0600 权限；文档和测试固定该入口。", - "凭据值仍不进入 Git、证据、日志或报告；导入后六个托管方法的前置校验通过。" - ] - ], + ["评测对象", "默认报告一度指向七种浏览器方法，旧十二方法页面又保留了已替换的易变正文样本，无法作为本轮十二方法权威矩阵。", "当前 profile 恢复十二种 Web Fetch 方法和 150 个任务；八个易变全文样本改为 RFC、固定 Git 提交、W3C 定版文档或 Wikipedia 固定修订，并保留明确的替换关系。", "最终权威矩阵仍为 1,800 个方法—任务逻辑结果；被替换题目及另外五个受参考或实现影响的题目须按当前合同重跑，旧结果不直接沿用。"], + ["实现身份与前置门禁", "旧证据没有完整固定 CowAgent、OpenClaw、DeepSeek Harness、LangChain 与 Crawl4AI 的源码或依赖身份，空路径或错误版本还可能在证据目录创建后才失败。", "源码型方法固定仓库修订、Node 版本和必需文件；LangChain WebBaseLoader 0.3.30、Crawl4AI 0.9.3 及文档解析依赖进入 uv.lock。所有方法在创建运行证据前完成身份、命令和路径校验，身份摘要写入 manifest。", "LangChain 与 Crawl4AI 因旧权威证据未记录库版本，各自完整重跑 150 题；源码型方法只有精确且干净的绑定实现进入本轮。"], + ["直接文档路线", "通用 Moli 适配器把直接 PDF 或 DOCX 正文任务当作普通网页 Markdown，可能在取得原始文档前失败或把二进制响应序列化。", "直接 PDF、DOCX 与文件任务预先选择 Moli 自动原始响应；正文任务在同一个 45 秒预算内解析文字并保留原始字节哈希，文件任务继续按字节验收，不增加跨方法下载后备。", "受路线影响的文档结果已进入声明的重跑范围；评分与盲审均使用新保存输出。"], + ["偶发下载证据", "正文抓取若同时触发浏览器下载，旧 runner 因二进制没有文件任务验收记录而中止整批，导致后续结果缺失。", "调用边界现在会保存并哈希所有返回字节；只有文件任务可用这些字节通过验收，正文任务仍必须交付合格文字。触发故障的 Crawl4AI 批次已整批作废并重跑。", "修复只保证证据完整和运行可继续，不把偶发下载当成正文能力，也不复用崩溃前的部分批次。"], + ["影响范围与测速", "仅补跑旧失败或只重建页面会混入未知实现版本，也无法公平比较来自不同批次的十二方法耗时。", "主矩阵预先声明 13 题 × 12 方法、LangChain 与 Crawl4AI 的其余完整方法 cohort，以及两个超时边界修正，共完成 432 次新调用；另以 provider-blind、域名去重的 20 题完成 240 次十二方法同窗测速。", "主矩阵固定每个方法—任务一个权威逻辑结果，不按最好或最快挑选。测速保留完整预声明矩阵：全调用延迟按同一 20 题分母统计，成功调用延迟另作条件统计；十二方法共同通过为 1/20，只作为诊断，不据此筛样本或排名。"], + ["证据与失败项核验", "旧组装器可能把 worker 工作树和符号链接带入汇总，又会漏掉重试物理调用返回的辅助文件；审查包若遗漏原生答案字段，也会制造系统性假分歧。", "汇总只复制哈希绑定的公开文件，拒绝符号链接和工作树路径，并绑定主返回、辅助文件与每次物理重试。Codex gpt-5.6-sol 对主矩阵 705 条机器失败结果盲审，审查包同时提供白名单化的原生答案 metadata；2 条分歧逐条裁决。", "最终证据为 12 × 150 的唯一主矩阵和 12 × 20 的唯一测速矩阵；主矩阵失败项核验 705/705，机器通过未逐条审查 1,095 条，测速结果不在本次审查范围，未决 0。裁决只确定审计结论，不回写或择优替换 provider 输出。"], + ["报告与发布隔离", "旧生成器混用了 150 题、六类和七浏览器专属叙述，方法页重复 Exa、Tavily、Firecrawl 配置，通用报告还可能显示 headless 修订史、复现链接和中文延迟加载提示。", "同一 renderer 改由当前 dataset、records 和实际收据生成数量、方法路线、修订说明与链接；十二方法与 headless 历史按 comparison kind 隔离，中英文虚拟路由分别显示对应加载及无障碍文案。当前十二方法构建绑定新的 dev 域站点，旧十二方法站点与 app 域浏览器参考报告保持不变。", "发布器逐文件核验公开文件；服务端允许的 HTML 外壳注入与匿名字节一致性分层披露，不再把两者混称为全站字节相同。"], + ["环境导入", "托管方法凭据已存在于用户级 ~/.env，但旧流程只读取仓库 .env，导致新会话会误报缺少变量。", "新增显式 import-env：仅将九个允许变量从用户级文件补入仓库 .env，保留已有非空值、隐藏变量内容并强制 0600 权限；文档和测试固定该入口。", "凭据值仍不进入 Git、证据、日志或报告；导入后六个托管方法的前置校验通过。"] + ] + , "i18n": { "en": { "title": "Complete the twelve-method upgrade, impact-scoped reruns, and failure review", "summary": "The twelve-method report is now bound to the current evaluation set, implementation identities, and an independent publication target. The primary matrix adds 432 calls within the predeclared impact scope and the same-window latency cohort adds 240 calls. All 705 machine failures in the primary matrix received independent review; the 1,095 machine passes were not reviewed individually. Every disagreement was adjudicated, with 0 unresolved.", "changes": [ - [ - "Evaluation scope", - "The default report had temporarily pointed to the seven browser methods, while the old twelve-method site still contained volatile body-text tasks that have since been replaced. Neither could serve as the authoritative twelve-method matrix for this run.", - "The current profile restores twelve Web Fetch methods and 150 tasks. Eight volatile full-text tasks are replaced with RFCs, pinned Git commits, final W3C publications, or a fixed Wikipedia revision, with each replacement recorded explicitly.", - "The final authoritative matrix remains 1,800 logical method–task results. The replaced tasks and five additional tasks affected by reference or implementation changes must be rerun under the current contract; their old results are not carried forward directly." - ], - [ - "Implementation identity and preflight gates", - "The old evidence did not fully pin the source or dependency identities for CowAgent, OpenClaw, DeepSeek Harness, LangChain, and Crawl4AI. Empty paths or incorrect versions could also fail only after an evidence directory had been created.", - "Source-backed methods now pin repository revisions, Node versions, and required files. LangChain WebBaseLoader 0.3.30, Crawl4AI 0.9.3, and document-parsing dependencies are locked in uv.lock. Every method validates its identity, command, and paths before run evidence is created, and an identity summary is written to the manifest.", - "LangChain and Crawl4AI each reran all 150 tasks because the previous authoritative evidence did not record their library versions. For source-based methods, only exact, clean, bound implementations entered this run." - ], - [ - "Direct-document path", - "The general Moli adapter treated direct PDF and DOCX body-text tasks as ordinary web Markdown, which could fail before retrieving the original document or serialize a binary response.", - "Direct PDF, DOCX, and file tasks now preselect Moli's automatic raw response. Body-text tasks parse text within the same 45-second budget and retain the raw-byte hash; file tasks continue to be accepted by byte validation, with no cross-method download fallback.", - "Document results affected by the route change were included in the declared rerun scope; scoring and blind review both use the newly saved outputs." - ], - [ - "Incidental-download evidence", - "When a body fetch also triggered a browser download, the old runner aborted the whole batch because the binary had no file-task evaluation record, leaving later results missing.", - "The call boundary now persists and hashes every returned binary. Only file tasks may pass on those bytes; body tasks must still deliver acceptable text. The affected Crawl4AI batch was invalidated in full and rerun.", - "This fix preserves evidence and lets execution continue; it neither treats an incidental download as body capability nor reuses the partial pre-crash batch." - ], - [ - "Impact scope and latency cohort", - "Rerunning only old failures or merely rebuilding the pages would mix unknown implementation versions and would not support a fair latency comparison across twelve methods measured in different batches.", - "The primary matrix predeclared 13 tasks × 12 methods, the remaining full-method cohorts for LangChain and Crawl4AI, and two deadline-boundary corrections, for 432 new calls in total. A separate provider-blind, domain-deduplicated set of 20 tasks produced 240 same-window latency calls across all twelve methods.", - "The primary matrix fixes one authoritative logical result for every method–task pair and never selects the best or fastest result. The latency cohort retains its complete predeclared matrix: all-call latency uses the same 20-task denominator, while successful-call latency is reported separately as a conditional statistic. All twelve methods jointly passed 1/20 tasks; this is diagnostic only and is not used to select samples or rank methods." - ], - [ - "Evidence and failure review", - "The old assembler could pull worker trees and symbolic links into the aggregate while omitting auxiliary files returned by physical retry calls. A review packet that omitted native answer fields could also create systematic false disagreements.", - "Aggregation now copies only hash-bound public files, rejects symbolic links and worker-tree paths, and binds the primary response, auxiliary files, and every physical retry. Codex gpt-5.6-sol blindly reviewed the 705 machine failures in the primary matrix, with allowlisted native answer metadata included in the review packet; the 2 disagreements were adjudicated individually.", - "The final evidence consists of one 12 × 150 primary matrix and one 12 × 20 latency matrix. Failure review covers 705/705 primary-matrix failures; 1,095 machine passes were not reviewed individually, and latency results were outside this review scope. There are 0 unresolved items. Adjudication determines only the audit conclusion and never rewrites or selectively replaces provider output." - ], - [ - "Report and publication isolation", - "The old generator mixed fixed 150-task and six-family copy with seven-browser-specific explanations. Method pages repeated Exa, Tavily, and Firecrawl configuration, while the general report could also show headless changelog entries, reproduction links, and Chinese lazy-loading messages.", - "The shared renderer now derives counts, method paths, revision explanations, and links from the current dataset, records, and available receipts. Twelve-method and headless histories are separated by comparison kind, and Chinese and English virtual routes show localized loading and accessibility copy. The current twelve-method build is bound to a new dev-domain site, while the old twelve-method site and the app-domain browser reference report remain unchanged.", - "The publisher verifies public files individually. Server-permitted HTML shell injection and anonymous byte equality are disclosed as separate layers rather than being conflated as site-wide byte identity." - ], - [ - "Environment import", - "Credentials for hosted methods already existed in the user-level ~/.env, but the old workflow read only the repository .env, causing new sessions to report variables as missing incorrectly.", - "An explicit import-env command now copies only nine allowlisted variables from the user-level file into the repository .env, preserves existing non-empty values, hides variable contents, and enforces 0600 permissions. Documentation and tests make this the stable entry point.", - "Credential values still never enter Git, evidence, logs, or the report. Preflight validation passes for all six hosted methods after import." - ] + ["Evaluation scope", "The default report had temporarily pointed to the seven browser methods, while the old twelve-method site still contained volatile body-text tasks that have since been replaced. Neither could serve as the authoritative twelve-method matrix for this run.", "The current profile restores twelve Web Fetch methods and 150 tasks. Eight volatile full-text tasks are replaced with RFCs, pinned Git commits, final W3C publications, or a fixed Wikipedia revision, with each replacement recorded explicitly.", "The final authoritative matrix remains 1,800 logical method–task results. The replaced tasks and five additional tasks affected by reference or implementation changes must be rerun under the current contract; their old results are not carried forward directly."], + ["Implementation identity and preflight gates", "The old evidence did not fully pin the source or dependency identities for CowAgent, OpenClaw, DeepSeek Harness, LangChain, and Crawl4AI. Empty paths or incorrect versions could also fail only after an evidence directory had been created.", "Source-backed methods now pin repository revisions, Node versions, and required files. LangChain WebBaseLoader 0.3.30, Crawl4AI 0.9.3, and document-parsing dependencies are locked in uv.lock. Every method validates its identity, command, and paths before run evidence is created, and an identity summary is written to the manifest.", "LangChain and Crawl4AI each reran all 150 tasks because the previous authoritative evidence did not record their library versions. For source-based methods, only exact, clean, bound implementations entered this run."], + ["Direct-document path", "The general Moli adapter treated direct PDF and DOCX body-text tasks as ordinary web Markdown, which could fail before retrieving the original document or serialize a binary response.", "Direct PDF, DOCX, and file tasks now preselect Moli's automatic raw response. Body-text tasks parse text within the same 45-second budget and retain the raw-byte hash; file tasks continue to be accepted by byte validation, with no cross-method download fallback.", "Document results affected by the route change were included in the declared rerun scope; scoring and blind review both use the newly saved outputs."], + ["Incidental-download evidence", "When a body fetch also triggered a browser download, the old runner aborted the whole batch because the binary had no file-task evaluation record, leaving later results missing.", "The call boundary now persists and hashes every returned binary. Only file tasks may pass on those bytes; body tasks must still deliver acceptable text. The affected Crawl4AI batch was invalidated in full and rerun.", "This fix preserves evidence and lets execution continue; it neither treats an incidental download as body capability nor reuses the partial pre-crash batch."], + ["Impact scope and latency cohort", "Rerunning only old failures or merely rebuilding the pages would mix unknown implementation versions and would not support a fair latency comparison across twelve methods measured in different batches.", "The primary matrix predeclared 13 tasks × 12 methods, the remaining full-method cohorts for LangChain and Crawl4AI, and two deadline-boundary corrections, for 432 new calls in total. A separate provider-blind, domain-deduplicated set of 20 tasks produced 240 same-window latency calls across all twelve methods.", "The primary matrix fixes one authoritative logical result for every method–task pair and never selects the best or fastest result. The latency cohort retains its complete predeclared matrix: all-call latency uses the same 20-task denominator, while successful-call latency is reported separately as a conditional statistic. All twelve methods jointly passed 1/20 tasks; this is diagnostic only and is not used to select samples or rank methods."], + ["Evidence and failure review", "The old assembler could pull worker trees and symbolic links into the aggregate while omitting auxiliary files returned by physical retry calls. A review packet that omitted native answer fields could also create systematic false disagreements.", "Aggregation now copies only hash-bound public files, rejects symbolic links and worker-tree paths, and binds the primary response, auxiliary files, and every physical retry. Codex gpt-5.6-sol blindly reviewed the 705 machine failures in the primary matrix, with allowlisted native answer metadata included in the review packet; the 2 disagreements were adjudicated individually.", "The final evidence consists of one 12 × 150 primary matrix and one 12 × 20 latency matrix. Failure review covers 705/705 primary-matrix failures; 1,095 machine passes were not reviewed individually, and latency results were outside this review scope. There are 0 unresolved items. Adjudication determines only the audit conclusion and never rewrites or selectively replaces provider output."], + ["Report and publication isolation", "The old generator mixed fixed 150-task and six-family copy with seven-browser-specific explanations. Method pages repeated Exa, Tavily, and Firecrawl configuration, while the general report could also show headless changelog entries, reproduction links, and Chinese lazy-loading messages.", "The shared renderer now derives counts, method paths, revision explanations, and links from the current dataset, records, and available receipts. Twelve-method and headless histories are separated by comparison kind, and Chinese and English virtual routes show localized loading and accessibility copy. The current twelve-method build is bound to a new dev-domain site, while the old twelve-method site and the app-domain browser reference report remain unchanged.", "The publisher verifies public files individually. Server-permitted HTML shell injection and anonymous byte equality are disclosed as separate layers rather than being conflated as site-wide byte identity."], + ["Environment import", "Credentials for hosted methods already existed in the user-level ~/.env, but the old workflow read only the repository .env, causing new sessions to report variables as missing incorrectly.", "An explicit import-env command now copies only nine allowlisted variables from the user-level file into the repository .env, preserves existing non-empty values, hides variable contents, and enforces 0600 permissions. Documentation and tests make this the stable entry point.", "Credential values still never enter Git, evidence, logs, or the report. Preflight validation passes for all six hosted methods after import."] ] } } @@ -4514,36 +3283,17 @@ "title": "解读突出耗时减少比例并增加三张信息图", "summary": "首屏以 Chromium 为基准展示均值、P50、P95 的耗时减少比例；以结果总览和两张流程图替代重复总结与部分流程文字，标题改为直接的内容名称。", "changes": [ - [ - "耗时比较", - "首屏强调秒数，读者需要自行计算差距。", - "用未舍入数据计算相对 Chromium 的耗时减少比例：均值 42.8%、P50 46.7%、P95 44.3%，秒数作为次要依据保留。", - "仍为跨批次的 98 道共同成功任务；统计样本与测量边界不变。" - ], - [ - "信息图", - "概念流程与总结主要依靠文字。", - "增加结果总览、Web Fetch 与 Browser Use 对照、五种交付物流程图；提供完整图片下载，英文版使用对应英文总览。", - "流程图为说明性示意，六组案例仍展示原始证据。图片逐项核对并绑定哈希，不参与评分。" - ] - ], + ["耗时比较", "首屏强调秒数，读者需要自行计算差距。", "用未舍入数据计算相对 Chromium 的耗时减少比例：均值 42.8%、P50 46.7%、P95 44.3%，秒数作为次要依据保留。", "仍为跨批次的 98 道共同成功任务；统计样本与测量边界不变。"], + ["信息图", "概念流程与总结主要依靠文字。", "增加结果总览、Web Fetch 与 Browser Use 对照、五种交付物流程图；提供完整图片下载，英文版使用对应英文总览。", "流程图为说明性示意，六组案例仍展示原始证据。图片逐项核对并绑定哈希，不参与评分。"] + ] + , "i18n": { "en": { "title": "Emphasize latency reduction and add three explanatory figures", "summary": "The opening compares mean, P50, and P95 latency reductions against Chromium. A result overview and two flow diagrams replace repeated summaries and some process prose, with direct descriptive titles.", "changes": [ - [ - "Latency comparison", - "The opening emphasized seconds and required readers to calculate the differences themselves.", - "Use unrounded values to calculate reductions relative to Chromium: 42.8% for mean latency, 46.7% for P50, and 44.3% for P95, while retaining seconds as secondary context.", - "The population remains 98 shared successful tasks collected across batches; the sample and measurement boundary are unchanged." - ], - [ - "Explanatory figures", - "Conceptual flows and summaries relied mostly on prose.", - "Add a result overview, a Web Fetch versus Browser Use comparison, and a flow diagram for five deliverable types. Full-size images are downloadable, with a corresponding English overview.", - "The diagrams are explanatory. Six case groups still show original evidence; images are reviewed and hash-bound but do not affect scoring." - ] + ["Latency comparison", "The opening emphasized seconds and required readers to calculate the differences themselves.", "Use unrounded values to calculate reductions relative to Chromium: 42.8% for mean latency, 46.7% for P50, and 44.3% for P95, while retaining seconds as secondary context.", "The population remains 98 shared successful tasks collected across batches; the sample and measurement boundary are unchanged."], + ["Explanatory figures", "Conceptual flows and summaries relied mostly on prose.", "Add a result overview, a Web Fetch versus Browser Use comparison, and a flow diagram for five deliverable types. Full-size images are downloadable, with a corresponding English overview.", "The diagrams are explanatory. Six case groups still show original evidence; images are reviewed and hash-bound but do not affect scoring."] ] } } @@ -4555,48 +3305,19 @@ "title": "解读开篇增加综合通过率与六类结果", "summary": "五浏览器解读先展示综合通过率与六类任务成绩，正文解释差异，结尾归纳结论。浏览器配置改为对照表，章节统一编号。", "changes": [ - [ - "结果展示", - "总通过率与分组结果分散在正文中。", - "开篇集中展示五款浏览器的总通过率与六类通过数，随后解释文档、媒体和网页的差异。", - "沿用同一组 150 道任务和 Moli 1.1.1，评分与通过数不变。" - ], - [ - "耗时展示", - "解读同时展开两组不同条件的耗时比较，阅读主线分散。", - "解读集中展示全部 98 道共同成功主任务的均值、P50、P95，并保留跨批次条件。", - "原报告的同窗测速章节与证据继续保留，未删除或改写调用记录。" - ], - [ - "阅读结构", - "章节眉题、副标题和浏览器长段落增加阅读负担。", - "统一编号标题和字号层级，以对照表介绍五款浏览器，增加量化总结与有序后续计划。", - "保留 Web Fetch、Browser Use 背景、五类流程和六组真实案例。" - ] - ], + ["结果展示", "总通过率与分组结果分散在正文中。", "开篇集中展示五款浏览器的总通过率与六类通过数，随后解释文档、媒体和网页的差异。", "沿用同一组 150 道任务和 Moli 1.1.1，评分与通过数不变。"], + ["耗时展示", "解读同时展开两组不同条件的耗时比较，阅读主线分散。", "解读集中展示全部 98 道共同成功主任务的均值、P50、P95，并保留跨批次条件。", "原报告的同窗测速章节与证据继续保留，未删除或改写调用记录。"], + ["阅读结构", "章节眉题、副标题和浏览器长段落增加阅读负担。", "统一编号标题和字号层级，以对照表介绍五款浏览器，增加量化总结与有序后续计划。", "保留 Web Fetch、Browser Use 背景、五类流程和六组真实案例。"] + ] + , "i18n": { "en": { "title": "Interpretation opens with overall pass rates and six task groups", "summary": "The five-browser article opens with overall pass rates and results for six task groups, explains the differences in the body, and closes with findings. Browser configurations use a comparison table, with consistently numbered sections.", "changes": [ - [ - "Result presentation", - "Overall pass rates and group results were spread throughout the article.", - "The opening brings together overall pass rates and six group pass counts for the five browsers, followed by explanations of document, media, and web differences.", - "The same 150 tasks and Moli 1.1.1 are retained; scores and pass counts are unchanged." - ], - [ - "Latency presentation", - "The article expanded two latency comparisons under different conditions, dividing the reading flow.", - "The article focuses on mean latency, P50, and P95 for all 98 shared successful main tasks, retaining the collection-across-batches condition.", - "The original report retains its same-window latency section and evidence; call records are neither deleted nor rewritten." - ], - [ - "Reading structure", - "Section eyebrows, subtitles, and long browser paragraphs added reading overhead.", - "Use consistently numbered headings and type sizes, introduce the five browsers in a comparison table, and add quantified findings and an ordered follow-up plan.", - "Retain Web Fetch and Browser Use background, five workflow types, and real examples from six groups." - ] + ["Result presentation", "Overall pass rates and group results were spread throughout the article.", "The opening brings together overall pass rates and six group pass counts for the five browsers, followed by explanations of document, media, and web differences.", "The same 150 tasks and Moli 1.1.1 are retained; scores and pass counts are unchanged."], + ["Latency presentation", "The article expanded two latency comparisons under different conditions, dividing the reading flow.", "The article focuses on mean latency, P50, and P95 for all 98 shared successful main tasks, retaining the collection-across-batches condition.", "The original report retains its same-window latency section and evidence; call records are neither deleted nor rewritten."], + ["Reading structure", "Section eyebrows, subtitles, and long browser paragraphs added reading overhead.", "Use consistently numbered headings and type sizes, introduce the five browsers in a comparison table, and add quantified findings and an ordered follow-up plan.", "Retain Web Fetch and Browser Use background, five workflow types, and real examples from six groups."] ] } } @@ -4608,12 +3329,7 @@ "title": "增加五浏览器 Web Fetch 图文解读", "summary": "文章对比 Moli 1.1.1 与另外四款浏览器，明确 150 题中通过 135 题，完整展示均值、P50、P95，并用六个评测集的可切换案例并列标准答案与真实输出。原始调用、评分及榜单分母不变。", "changes": [ - [ - "加载恢复", - "短暂网络失败后，失败缓存使页面内重试无法重新请求。", - "清除失败缓存，为页面和证据内容提供重新加载入口。", - "仅修复报告读取行为，原始输出与评分不变。" - ], + ["加载恢复", "短暂网络失败后，失败缓存使页面内重试无法重新请求。", "清除失败缓存，为页面和证据内容提供重新加载入口。", "仅修复报告读取行为，原始输出与评分不变。"], [ "文章范围", "多版本结果与案例细节分散，难以独立理解 Moli 1.1.1 的表现。", @@ -4638,42 +3354,18 @@ "普通网页、动态网页、热门站点、文档、图片、视频各取一个案例，逐项核对独立参考与真实返回；保留原始 PNG，关键数字直接排版。", "明确结构检查不核验商品名称事实、文件长度门槛为 32 字节，以及正文规则经过统一离线修订。" ] - ], + ] + , "i18n": { "en": { "title": "Add an illustrated five-browser Web Fetch interpretation", "summary": "The article compares Moli 1.1.1 with four other browsers, states that it passed 135 of 150 tasks, presents mean latency, P50, and P95, and provides selectable examples from six dataset groups pairing references with actual outputs. Original calls, scores, and ranking denominators are unchanged.", "changes": [ - [ - "Loading recovery", - "After a temporary network failure, cached failures prevented in-page retries from making a new request.", - "Clear cached failures and provide reload controls for pages and evidence content.", - "Only report loading behavior changes; original outputs and scores are unchanged." - ], - [ - "Article scope", - "Results across versions and scattered case details made it difficult to understand Moli 1.1.1's performance independently.", - "Generate five-browser charts, a 120/30-task breakdown, a PDF path, and media examples from the same evidence; disclose that Moli 1.1.1 was selected after seeing version results.", - "The article presents 750 results from five methods; all 1,050 records across seven paths remain available, without task-by-task version selection." - ], - [ - "Basis for conclusions", - "The total pass count does not explain which input types contribute to the difference.", - "Show that, across 120 web tasks, Moli and Chromium each complete 111; across 30 document and media tasks, they complete 24 and 14 respectively.", - "PDF results include the local parser; file checks do not assess media semantics. Latency for the 98 shared successful main tasks spans batches. The five-browser same-window comparison uses all 9 shared successes, without changing the original seven-configuration latency results." - ], - [ - "Reading format", - "Report tables were not convenient for long-form reading.", - "The integrated article and standalone HTML share body text, inline graphics, and styles, with mobile and print layouts.", - "Presentation and explanation changes only: no new provider calls and no score or latency changes." - ], - [ - "Acceptance explained", - "The five task types previously had only conceptual flows, making the reasons for passing difficult to check.", - "One example each from general pages, dynamic pages, popular sites, documents, images, and videos compares independent references with actual outputs check by check. The original PNG is retained and key figures are rendered directly as text.", - "Clarify that structural checks do not verify product-name facts, the file-length threshold is 32 bytes, and body rules received shared offline revisions." - ] + ["Loading recovery", "After a temporary network failure, cached failures prevented in-page retries from making a new request.", "Clear cached failures and provide reload controls for pages and evidence content.", "Only report loading behavior changes; original outputs and scores are unchanged."], + ["Article scope", "Results across versions and scattered case details made it difficult to understand Moli 1.1.1's performance independently.", "Generate five-browser charts, a 120/30-task breakdown, a PDF path, and media examples from the same evidence; disclose that Moli 1.1.1 was selected after seeing version results.", "The article presents 750 results from five methods; all 1,050 records across seven paths remain available, without task-by-task version selection."], + ["Basis for conclusions", "The total pass count does not explain which input types contribute to the difference.", "Show that, across 120 web tasks, Moli and Chromium each complete 111; across 30 document and media tasks, they complete 24 and 14 respectively.", "PDF results include the local parser; file checks do not assess media semantics. Latency for the 98 shared successful main tasks spans batches. The five-browser same-window comparison uses all 9 shared successes, without changing the original seven-configuration latency results."], + ["Reading format", "Report tables were not convenient for long-form reading.", "The integrated article and standalone HTML share body text, inline graphics, and styles, with mobile and print layouts.", "Presentation and explanation changes only: no new provider calls and no score or latency changes."], + ["Acceptance explained", "The five task types previously had only conceptual flows, making the reasons for passing difficult to check.", "One example each from general pages, dynamic pages, popular sites, documents, images, and videos compares independent references with actual outputs check by check. The original PNG is retained and key figures are rendered directly as text.", "Clarify that structural checks do not verify product-name facts, the file-length threshold is 32 bytes, and body rules received shared offline revisions."] ] } } @@ -4721,48 +3413,19 @@ "对同一四题、三个版本做三轮轮换，统一采用第三轮的 12 个耗时并重算均值、P50、P95；不按结果择优。", "全部通过任务中，1.1.3 的均值由 4,000 降至 3,706 ms、P50 由 3,444 降至 3,340 ms、P95 由 9,833 降至 7,983 ms；1.0.6 与 1.1.1 也按同一规则更新。共保留 36 次物理调用；正文、评分和任务判定未改变。" ] - ], + ] + , "i18n": { "en": { "title": "Document on-demand publishing, complete setup guidance, and strengthen the public-release security gate", "summary": "The report now distinguishes logical routes from deployed physical files and adds route-aligned bilingual access, setup instructions for all seven methods, and a stronger public-release security gate. It also corrects 12 anomalous latency values across four tasks and three Moli versions using one fixed final round. Task outputs, verdicts, and scores are unchanged; latency aggregates were recomputed.", "changes": [ - [ - "Publishing architecture", - "Calling 1,218 auditable routes deployed HTML files suggested duplicated deployment and eager loading.", - "Clarified that the local build produces logical routes, while the Sites bundle uses one HTML shell, a shared runtime, and content-addressed compressed blocks; pages and large evidence load on demand.", - "The publication manifest records logical routes, source files, compressed blocks, and per-file hashes. Evidence is not reduced and download bytes do not change." - ], - [ - "Access & setup", - "The old page showed only Cloudflare, the sole provider account requirement, making acquisition and binary configuration for local methods appear to be missing.", - "Changed the page to an access and installation checklist: Cloudflare lists account and credential variables, while the six local methods list official acquisition sources and binary variables.", - "Publish only variable names and official sources; never retain values, local paths, cookies, or personal sessions." - ], - [ - "Review attribution", - "The ranking said only that results were reviewed, without identifying the reviewer.", - "Each method's review column now shows 100/100 · reviewed by Codex and continues to link to the shared review record.", - "Presentation copy only; review bindings, coverage, and unresolved counts are unchanged." - ], - [ - "Chinese and English editions", - "The former English entry was a one-page summary whose routes, sections, and per-attempt evidence explanations did not align with the Chinese report.", - "Generate route-aligned Chinese and English sites from the same report model. Navigation follows dataset, methodology, analysis, review, interpretation, methods, reproduction, changelog, and setup; switching language stays on the current route.", - "Validate one-to-one parity for routes, sections, tables, values, filters, and evidence links. UI and explanations are rewritten for natural English, while task names, IDs, URLs, reference answers, and raw outputs remain unchanged and are not rescored." - ], - [ - "Public-release security gate", - "The previous release scan covered common Google, Artifact Hub, GitHub, and OpenAI key patterns, but did not explicitly block Authorization values, local user-directory paths, or AWS and Slack credentials.", - "Every rebuilt public file is now scanned uniformly, adding checks for Authorization values, user-directory paths, AWS and Slack credentials, and private-key patterns. Existing redaction of Google-shaped secrets in third-party HTML retains both hashes.", - "Pattern scanning cannot prove that every unknown secret is absent. Structured credential values remain prohibited in evidence and reports, and published files still require byte-level verification." - ], - [ - "Moli latency correction", - "A few anomalous legacy latency values made 1.1.3 appear to have an unproven overall performance regression.", - "Run three order-rotated rounds for the same four tasks and three versions, use the 12 third-round latency values uniformly, and recompute mean, P50, and P95 without selecting favorable results.", - "Across passing tasks, the 1.1.3 mean fell from 4,000 to 3,706 ms, P50 from 3,444 to 3,340 ms, and P95 from 9,833 to 7,983 ms. Versions 1.0.6 and 1.1.1 were updated under the same rule. All 36 physical calls are retained; body text, scores, and task verdicts are unchanged." - ] + ["Publishing architecture", "Calling 1,218 auditable routes deployed HTML files suggested duplicated deployment and eager loading.", "Clarified that the local build produces logical routes, while the Sites bundle uses one HTML shell, a shared runtime, and content-addressed compressed blocks; pages and large evidence load on demand.", "The publication manifest records logical routes, source files, compressed blocks, and per-file hashes. Evidence is not reduced and download bytes do not change."], + ["Access & setup", "The old page showed only Cloudflare, the sole provider account requirement, making acquisition and binary configuration for local methods appear to be missing.", "Changed the page to an access and installation checklist: Cloudflare lists account and credential variables, while the six local methods list official acquisition sources and binary variables.", "Publish only variable names and official sources; never retain values, local paths, cookies, or personal sessions."], + ["Review attribution", "The ranking said only that results were reviewed, without identifying the reviewer.", "Each method's review column now shows 100/100 · reviewed by Codex and continues to link to the shared review record.", "Presentation copy only; review bindings, coverage, and unresolved counts are unchanged."], + ["Chinese and English editions", "The former English entry was a one-page summary whose routes, sections, and per-attempt evidence explanations did not align with the Chinese report.", "Generate route-aligned Chinese and English sites from the same report model. Navigation follows dataset, methodology, analysis, review, interpretation, methods, reproduction, changelog, and setup; switching language stays on the current route.", "Validate one-to-one parity for routes, sections, tables, values, filters, and evidence links. UI and explanations are rewritten for natural English, while task names, IDs, URLs, reference answers, and raw outputs remain unchanged and are not rescored."], + ["Public-release security gate", "The previous release scan covered common Google, Artifact Hub, GitHub, and OpenAI key patterns, but did not explicitly block Authorization values, local user-directory paths, or AWS and Slack credentials.", "Every rebuilt public file is now scanned uniformly, adding checks for Authorization values, user-directory paths, AWS and Slack credentials, and private-key patterns. Existing redaction of Google-shaped secrets in third-party HTML retains both hashes.", "Pattern scanning cannot prove that every unknown secret is absent. Structured credential values remain prohibited in evidence and reports, and published files still require byte-level verification."], + ["Moli latency correction", "A few anomalous legacy latency values made 1.1.3 appear to have an unproven overall performance regression.", "Run three order-rotated rounds for the same four tasks and three versions, use the 12 third-round latency values uniformly, and recompute mean, P50, and P95 without selecting favorable results.", "Across passing tasks, the 1.1.3 mean fell from 4,000 to 3,706 ms, P50 from 3,444 to 3,340 ms, and P95 from 9,833 to 7,983 ms. Versions 1.0.6 and 1.1.1 were updated under the same rule. All 36 physical calls are retained; body text, scores, and task verdicts are unchanged."] ] } } @@ -4798,36 +3461,17 @@ "区分两条路径：错误命令、版本、裁判或参考按明确范围 supersede 并重测；普通失败可在测量前声明最多重试一次，保留两次物理证据且最后一次无条件生效。", "当前 1,050 条结果没有启用重试，issue #66 的诊断复测也不替换正式结果；后续协议可按需启用。" ] - ], + ] + , "i18n": { "en": { "title": "Unify method-combination comparisons and add Moli latency diagnostics", "summary": "The analysis page now uses one comparator for both two-method and multi-method comparisons, with a clearer task-and-result layout. After rotating request order on the public web and checking against a fixed local page, the regression signal in issue #66 remains a long-tail effect that cannot be attributed to a single cause. Official tasks, scores, and latency values are unchanged.", "changes": [ - [ - "Unified comparator", - "Two-method and multi-method comparisons were split across two large cards and two result tables, duplicating controls and crowding the page.", - "Keep one method A selector, a multi-select for comparison methods, and one shared set of relationship filters; render only one matched-task result table.", - "Supports disagreements, A passing or failing, all methods passing or failing, and only the selected methods passing; no new score is produced." - ], - [ - "Result layout", - "The two-column layout squeezed the task-family and method columns, wrapping Chinese family names one character at a time and making horizontal reading difficult.", - "The component now uses the full width. Task family appears as a badge inside the task cell, and passing and failing methods appear as wrapping status labels.", - "The table can still scroll horizontally on narrow screens, while task names, IDs, and evidence links remain complete." - ], - [ - "Moli latency signal", - "The official single-run results show 1.1.3 markedly slower than earlier versions on a few tasks, but cannot separate version effects from network and page-wait effects.", - "Retest four high-difference tasks on distinct paths in three rotating rounds, then use five rounds per version on a fixed local page to measure fixed overhead.", - "Per-task medians on the public web move in both directions and have overlapping ranges. Fixed local overhead shows no 1.1.3 regression, so issue #66 is not promoted to a confirmed regression." - ], - [ - "Corrections and failure retries", - "Banning every later call would prevent correction of proven errors; selecting a better response after failure would instead create best-of bias.", - "Separate two paths: an incorrect command, version, judge, or reference supersedes the declared scope and is rerun; an ordinary failure may use at most one retry declared before measurement, retaining both physical calls and unconditionally using the final one.", - "Retries are not enabled for the current 1,050 results, and the diagnostic reruns for issue #66 do not replace official results; future protocols may enable retries when needed." - ] + ["Unified comparator", "Two-method and multi-method comparisons were split across two large cards and two result tables, duplicating controls and crowding the page.", "Keep one method A selector, a multi-select for comparison methods, and one shared set of relationship filters; render only one matched-task result table.", "Supports disagreements, A passing or failing, all methods passing or failing, and only the selected methods passing; no new score is produced."], + ["Result layout", "The two-column layout squeezed the task-family and method columns, wrapping Chinese family names one character at a time and making horizontal reading difficult.", "The component now uses the full width. Task family appears as a badge inside the task cell, and passing and failing methods appear as wrapping status labels.", "The table can still scroll horizontally on narrow screens, while task names, IDs, and evidence links remain complete."], + ["Moli latency signal", "The official single-run results show 1.1.3 markedly slower than earlier versions on a few tasks, but cannot separate version effects from network and page-wait effects.", "Retest four high-difference tasks on distinct paths in three rotating rounds, then use five rounds per version on a fixed local page to measure fixed overhead.", "Per-task medians on the public web move in both directions and have overlapping ranges. Fixed local overhead shows no 1.1.3 regression, so issue #66 is not promoted to a confirmed regression."], + ["Corrections and failure retries", "Banning every later call would prevent correction of proven errors; selecting a better response after failure would instead create best-of bias.", "Separate two paths: an incorrect command, version, judge, or reference supersedes the declared scope and is rerun; an ordinary failure may use at most one retry declared before measurement, retaining both physical calls and unconditionally using the final one.", "Retries are not enabled for the current 1,050 results, and the diagnostic reruns for issue #66 do not replace official results; future protocols may enable retries when needed."] ] } } @@ -4857,30 +3501,16 @@ "三轮配对诊断将瞬时请求、页面不稳定和当前主榜成绩分开；京东保留为网络与会话敏感的挑战样本。", "当前主榜未启用失败重试，也不用诊断调用挑选更好响应；后续可预先声明失败重试，已证实的错误可按完整范围纠正并替换权威逻辑结果。" ] - ], + ] + , "i18n": { "en": { "title": "Add matched-task method comparisons and fix stalls on large outputs", "summary": "The analysis page now includes two-method differences and multi-method intersections. Complete evidence larger than 256 KiB is no longer inserted automatically and instead loads on demand or downloads directly. Official tasks, calls, verdicts, scores, and latency values are unchanged.", "changes": [ - [ - "Matched-task method comparison", - "The old page could only filter 1,050 calls by method or result, making it difficult to find tasks where A passed and B failed, both passed, both failed, or several methods disagreed.", - "Add two-method differences and multi-method intersections. Each row reuses the reviewed primary verdict for the same task and links to every reference answer, actual output, and failure reason.", - "This is set analysis: it produces no new score and does not mix latency from different passing samples." - ], - [ - "Large-evidence loading", - "Automatically downloading an output of more than a million characters and inserting it into a pre element occupied the main thread for a long time, making the page appear frozen.", - "Text larger than 256 KiB first shows an on-demand control. After a click, the complete content is placed in a read-only text box; the original-file download is always retained.", - "Only browser presentation timing and controls change; evidence bytes and checksums in the release bundle are unchanged." - ], - [ - "Version and sample boundaries", - "Moli 1.1.3 passing two fewer tasks than 1.1.1 could be read directly as a capability regression, while JD.com occasionally opening could be used to dismiss the official failure.", - "Three paired diagnostic rounds separate transient requests and page instability from the current primary ranking. JD.com remains a challenge sample sensitive to network and session state.", - "The current primary ranking does not enable failure retries or use diagnostic calls to select better responses. Future runs may predeclare failure retries, while proven errors may be corrected across their complete scope and replace the authoritative logical results." - ] + ["Matched-task method comparison", "The old page could only filter 1,050 calls by method or result, making it difficult to find tasks where A passed and B failed, both passed, both failed, or several methods disagreed.", "Add two-method differences and multi-method intersections. Each row reuses the reviewed primary verdict for the same task and links to every reference answer, actual output, and failure reason.", "This is set analysis: it produces no new score and does not mix latency from different passing samples."], + ["Large-evidence loading", "Automatically downloading an output of more than a million characters and inserting it into a pre element occupied the main thread for a long time, making the page appear frozen.", "Text larger than 256 KiB first shows an on-demand control. After a click, the complete content is placed in a read-only text box; the original-file download is always retained.", "Only browser presentation timing and controls change; evidence bytes and checksums in the release bundle are unchanged."], + ["Version and sample boundaries", "Moli 1.1.3 passing two fewer tasks than 1.1.1 could be read directly as a capability regression, while JD.com occasionally opening could be used to dismiss the official failure.", "Three paired diagnostic rounds separate transient requests and page instability from the current primary ranking. JD.com remains a challenge sample sensitive to network and session state.", "The current primary ranking does not enable failure retries or use diagnostic calls to select better responses. Future runs may predeclare failure retries, while proven errors may be corrected across their complete scope and replace the authoritative logical results."] ] } } @@ -4910,30 +3540,16 @@ "新增 benchmark-report Skill，并由根 AGENTS.md 强制路由；规则适用于 Web Fetch、Browser Use 及其他系统对比，不绑定本报告页面结构。", "这是流程和维护门禁；本轮主结果仍为 1,050 条，同窗测速仍为 140 条，未重新调用 provider。" ] - ], + ] + , "i18n": { "en": { "title": "Migrate to Sites and formalize the benchmark-report workflow", "summary": "The report moved to a new Sites domain while retaining the original sites. The repository now includes a report skill spanning benchmark types and covering evaluation design, evidence, independent review, impact-scoped reruns, and publication gates. Tasks, outputs, scores, and latency values are unchanged.", "changes": [ - [ - "Publication target", - "The old publication script hard-coded the artifact-hub.dev domain, so a new session could continue updating an old site.", - "The current report configuration binds both the Sites service address and site ID. Publishing, verification, and body links read the same configuration, and an update is rejected if the targets disagree.", - "The new site is UUAhhQMnrGwm; neither old site, bWkRP157EtF6 or UeZE1vfcEOrM, was overwritten." - ], - [ - "Publishing identity", - "An old publication token in the repository .env could take precedence over the new user-level Sites token, causing 401 responses even after authorization.", - "Publishing and verification scripts no longer load the evaluation .env; they read only the explicit process environment or user-level token, and device authorization backs up a differing old token.", - "Only the source of publication credentials changes; no provider call or report data changes." - ], - [ - "Cross-benchmark workflow", - "The existing benchmark-editorial skill covers only formal interpretation articles and cannot govern evaluation design, score review, selective reruns, or report publication.", - "Add the benchmark-report skill and require routing through it from the root AGENTS.md. Its rules apply to Web Fetch, Browser Use, and other system comparisons without depending on this report's page structure.", - "This is a workflow and maintenance gate. The run still contains 1,050 primary results and 140 same-window latency results; no provider was called again." - ] + ["Publication target", "The old publication script hard-coded the artifact-hub.dev domain, so a new session could continue updating an old site.", "The current report configuration binds both the Sites service address and site ID. Publishing, verification, and body links read the same configuration, and an update is rejected if the targets disagree.", "The new site is UUAhhQMnrGwm; neither old site, bWkRP157EtF6 or UeZE1vfcEOrM, was overwritten."], + ["Publishing identity", "An old publication token in the repository .env could take precedence over the new user-level Sites token, causing 401 responses even after authorization.", "Publishing and verification scripts no longer load the evaluation .env; they read only the explicit process environment or user-level token, and device authorization backs up a differing old token.", "Only the source of publication credentials changes; no provider call or report data changes."], + ["Cross-benchmark workflow", "The existing benchmark-editorial skill covers only formal interpretation articles and cannot govern evaluation design, score review, selective reruns, or report publication.", "Add the benchmark-report skill and require routing through it from the root AGENTS.md. Its rules apply to Web Fetch, Browser Use, and other system comparisons without depending on this report's page structure.", "This is a workflow and maintenance gate. The run still contains 1,050 primary results and 140 same-window latency results; no provider was called again."] ] } } @@ -4964,30 +3580,16 @@ "核验轮次改为独立的结构化记录并在构建时校验；Moli 直接文件统一使用裸 fetch；文档命令统一使用锁定的 uv 运行入口。", "页面只展示每轮范围、发现、纠正和未完成项；不增加核验结果下载。" ] - ], + ] + , "i18n": { "en": { "title": "Pin the default report evidence and measured routes", "summary": "Default rebuilds now accept only the current evidence for seven methods, 150 tasks, and 1,050 primary calls. Machine-readable routes now reflect the native Markdown, Moli automatic raw responses, and node-html-markdown 2.0.0 actually used in this run. Scores, call outputs, and latency values are unchanged.", "changes": [ - [ - "Default rebuild", - "The default evidence directory still pointed to the old twelve-method evaluation, so a new session could silently generate a different report.", - "Set the reviewed seven-method evidence as current and add a separate current-report configuration file. Build and evidence checks reject evidence whose methods, counts, deadlines, or routes do not match.", - "Primary-result review remains 1,050/1,050 and same-window latency review remains 140/140; old evidence is retained under the research directory." - ], - [ - "Route metadata", - "Download data still identified Chromium's converter as the retired pyhtml2md 1.7.0 and did not state how the three Moli versions handled direct documents and media.", - "Route descriptions now share the runner's current definitions: ordinary pages use each method's native body capability, Moli uses automatic raw responses for direct documents and media, and Chromium uses node-html-markdown 2.0.0.", - "Only non-verdict metadata is corrected; tasks, references, outputs, scores, and latency values are unchanged." - ], - [ - "Regression-prevention gate", - "Review rounds were attached to one release record, so adding a release could empty the review page; the generic Moli file adapter could also still append --dump html to binary responses.", - "Review rounds are now independent structured records validated at build time. Moli direct files consistently use bare fetch, and document commands consistently use the locked uv runtime entry point.", - "The page shows only each round's scope, findings, corrections, and outstanding items; no review-result download is added." - ] + ["Default rebuild", "The default evidence directory still pointed to the old twelve-method evaluation, so a new session could silently generate a different report.", "Set the reviewed seven-method evidence as current and add a separate current-report configuration file. Build and evidence checks reject evidence whose methods, counts, deadlines, or routes do not match.", "Primary-result review remains 1,050/1,050 and same-window latency review remains 140/140; old evidence is retained under the research directory."], + ["Route metadata", "Download data still identified Chromium's converter as the retired pyhtml2md 1.7.0 and did not state how the three Moli versions handled direct documents and media.", "Route descriptions now share the runner's current definitions: ordinary pages use each method's native body capability, Moli uses automatic raw responses for direct documents and media, and Chromium uses node-html-markdown 2.0.0.", "Only non-verdict metadata is corrected; tasks, references, outputs, scores, and latency values are unchanged."], + ["Regression-prevention gate", "Review rounds were attached to one release record, so adding a release could empty the review page; the generic Moli file adapter could also still append --dump html to binary responses.", "Review rounds are now independent structured records validated at build time. Moli direct files consistently use bare fetch, and document commands consistently use the locked uv runtime entry point.", "The page shows only each round's scope, findings, corrections, and outstanding items; no review-result download is added."] ] } } @@ -5048,54 +3650,20 @@ "环境和复现合并，只保留版本、二进制哈希、部署边界、配置文件和锁定命令；账号页仅列所需账号、变量名和官方获取入口。", "正式密钥只从仓库本地 .env 读取，不进入 HTML、Git 或发布包；Kitesurf 的托管后端版本和本地部署不可复现边界单独披露。" ] - ], + ] + , "i18n": { "en": { "title": "Complete seven-method measurement, full review, and same-window latency testing", "summary": "All seven methods completed the same 150 tasks for 1,050 primary calls, plus 20 tasks × 7 methods for 140 same-window latency calls. A Codex task agent reviewed 1,190/1,190 outputs without seeing machine verdicts or scores; the final decisions agree with current scoring, with 0 unresolved.", "changes": [ - [ - "Complete seven-method matrix", - "After Moli 1.0.6 was added, the old six-method denominator and version comparison were incomplete.", - "Each of the seven methods runs the same 150 tasks, retaining one authoritative logical result per method-task pair; Moli 1.0.6, 1.1.1, and 1.1.3 are listed separately.", - "Pass counts are Moli 1.1.1 135, Moli 1.0.6 134, Moli 1.1.3 133, Chromium 125, Lightpanda 119, Obscura 117, and Kitesurf 112. The totals show coverage of this dataset only and do not replace the six family rankings." - ], - [ - "Independent-review ledger", - "Reviewing only failures would miss false positives, and old detail pages also mistook layout differences for missing content.", - "A Codex task agent read every task contract, reference answer, and complete actual output. Primary-result review covers 1,050/1,050 and same-window latency review covers 140/140; machine verdicts and scores were hidden until after the independent decision.", - "Current automated false negatives are 0, false positives are 0, and unresolved items are 0. Historical issues including challenge-08, dynamic-quote field boundaries, the Feishu table, and direct-document routes were rescored or rerun under the current shared rules." - ], - [ - "Same-window latency test", - "The primary matrix spans several execution batches, so remote networking, queuing, and local resource contention prevent fair cross-batch latency ranking.", - "Use 20 predeclared pages on distinct domains and run all seven methods in parallel within the same window. Latency is compared only on the same 8 tasks passed by every method.", - "Moli 1.0.6 has the lowest sample P50. The 8 tasks are a small exploratory sample, and P95 is only a sample diagnostic, not an estimate of stable tail latency or an SLA." - ], - [ - "Reference and route corrections", - "live-025 incorrectly used the initial HTTP bytes as the reference for a dynamic page. For direct PDFs and media, forcing --dump on Moli fails before the artifact is retrieved.", - "live-025 now uses visible body text rendered by an independent browser, and all seven methods were rerun. The three Moli versions use bare fetch for direct PDF, DOCX, and media URLs; body tasks include local parsing, while file tasks validate bytes directly.", - "References, input-output targets, and the scoring entry point are identical for every method; each method may still use its fastest official native-artifact route." - ], - [ - "document-03 reference replacement", - "The old W3C PDF URL became a Cloudflare verification page during this run; continuing to use it would mistake site blocking for PDF-parsing capability.", - "Replace it with a three-page PDF from a pinned Mozilla PDF.js Git commit and call all seven methods again; the unchanged 1,043 primary results and their reviews remain intact.", - "Moli 1.0.6 and 1.1.1 retrieved all three pages of text and passed. Moli 1.1.3 had a single request failure, while the other methods did not retrieve readable body text. This single result cannot by itself establish a version regression." - ], - [ - "Title diagnostics", - "The title metric previously showed a definition followed by an empty table, leaving its inputs, alignment, and aggregation process invisible.", - "Show four steps: read records, align by detail URL, compare each title, and aggregate comparable results. When no titles can be compared, explicitly show not applicable instead of generating an empty table.", - "Title diagnostics still do not determine task acceptance and do not represent ranking recall, freshness, or whole-page body quality." - ], - [ - "Environment and reproduction", - "The old environment page repeated every batch, while the account page could imply that production credentials should be stored in the report.", - "Combine environment and reproduction, retaining only versions, binary hashes, deployment boundaries, configuration files, and locked commands. The account page lists only required accounts, variable names, and official acquisition sources.", - "Production credentials are read only from the repository-local .env and never enter HTML, Git, or the release bundle. Kitesurf's hosted backend version and the non-reproducibility of local deployment are disclosed separately." - ] + ["Complete seven-method matrix", "After Moli 1.0.6 was added, the old six-method denominator and version comparison were incomplete.", "Each of the seven methods runs the same 150 tasks, retaining one authoritative logical result per method-task pair; Moli 1.0.6, 1.1.1, and 1.1.3 are listed separately.", "Pass counts are Moli 1.1.1 135, Moli 1.0.6 134, Moli 1.1.3 133, Chromium 125, Lightpanda 119, Obscura 117, and Kitesurf 112. The totals show coverage of this dataset only and do not replace the six family rankings."], + ["Independent-review ledger", "Reviewing only failures would miss false positives, and old detail pages also mistook layout differences for missing content.", "A Codex task agent read every task contract, reference answer, and complete actual output. Primary-result review covers 1,050/1,050 and same-window latency review covers 140/140; machine verdicts and scores were hidden until after the independent decision.", "Current automated false negatives are 0, false positives are 0, and unresolved items are 0. Historical issues including challenge-08, dynamic-quote field boundaries, the Feishu table, and direct-document routes were rescored or rerun under the current shared rules."], + ["Same-window latency test", "The primary matrix spans several execution batches, so remote networking, queuing, and local resource contention prevent fair cross-batch latency ranking.", "Use 20 predeclared pages on distinct domains and run all seven methods in parallel within the same window. Latency is compared only on the same 8 tasks passed by every method.", "Moli 1.0.6 has the lowest sample P50. The 8 tasks are a small exploratory sample, and P95 is only a sample diagnostic, not an estimate of stable tail latency or an SLA."], + ["Reference and route corrections", "live-025 incorrectly used the initial HTTP bytes as the reference for a dynamic page. For direct PDFs and media, forcing --dump on Moli fails before the artifact is retrieved.", "live-025 now uses visible body text rendered by an independent browser, and all seven methods were rerun. The three Moli versions use bare fetch for direct PDF, DOCX, and media URLs; body tasks include local parsing, while file tasks validate bytes directly.", "References, input-output targets, and the scoring entry point are identical for every method; each method may still use its fastest official native-artifact route."], + ["document-03 reference replacement", "The old W3C PDF URL became a Cloudflare verification page during this run; continuing to use it would mistake site blocking for PDF-parsing capability.", "Replace it with a three-page PDF from a pinned Mozilla PDF.js Git commit and call all seven methods again; the unchanged 1,043 primary results and their reviews remain intact.", "Moli 1.0.6 and 1.1.1 retrieved all three pages of text and passed. Moli 1.1.3 had a single request failure, while the other methods did not retrieve readable body text. This single result cannot by itself establish a version regression."], + ["Title diagnostics", "The title metric previously showed a definition followed by an empty table, leaving its inputs, alignment, and aggregation process invisible.", "Show four steps: read records, align by detail URL, compare each title, and aggregate comparable results. When no titles can be compared, explicitly show not applicable instead of generating an empty table.", "Title diagnostics still do not determine task acceptance and do not represent ranking recall, freshness, or whole-page body quality."], + ["Environment and reproduction", "The old environment page repeated every batch, while the account page could imply that production credentials should be stored in the report.", "Combine environment and reproduction, retaining only versions, binary hashes, deployment boundaries, configuration files, and locked commands. The account page lists only required accounts, variable names, and official acquisition sources.", "Production credentials are read only from the repository-local .env and never enter HTML, Git, or the release bundle. Kitesurf's hosted backend version and the non-reproducibility of local deployment are disclosed separately."] ] } } @@ -5125,30 +3693,16 @@ "主结果 900/900 全量盲审，包括 Moli 1.1.1 和 1.1.3 各 150 条；184 个机器失败项全部有非空独立审查证据。", "各方法失败核验覆盖均为 100%，无未裁定结果。" ] - ], + ] + , "i18n": { "en": { "title": "Distinguish text presence from intact field boundaries", "summary": "Review two Moli outputs for the asynchronous Quotes list: all 10 quotations and authors were retrieved, but adjacent fields and multiple tags lacked readable separators. The task remains failed, while the detail page now presents quotations, authors, and tags field by field instead of describing lost boundaries broadly as missing text.", "changes": [ - [ - "Field-level evidence", - "The old detail page marked an entire stable record as unmatched even when its quotation and author appeared in full, obscuring that the actual failure was the tag-list boundary.", - "For each record, show the reference, parsed value, and verdict for quotation, author, and tags; when all tag characters are present but separators disappear, explicitly mark the field boundary invalid.", - "Only the failure explanation and classification are corrected; the task contract, raw output, latency, and score do not change." - ], - [ - "Review of two Moli versions", - "Looking only at Moli 1.1.3 would miss whether the earlier version had the same problem.", - "The outputs from Moli 1.1.1 and 1.1.3 for this task are byte-for-byte identical, each with 10 complete unique records. In 8 records, multiple tags run together, and the last record also includes Loading text.", - "Both versions remain failed. Single-tag record 9 matches normally, showing that the rule checks tag boundaries rather than whether the quotation appears." - ], - [ - "Failure-review coverage", - "Readers need to distinguish full review from checks of only selected versions.", - "All 900/900 primary results were blindly reviewed, including Moli 1.1.1 and 1.1.3 with 150 each. Every one of the 184 machine failures has nonempty independent review evidence.", - "Failure-review coverage is 100% for every method, with no unadjudicated results." - ] + ["Field-level evidence", "The old detail page marked an entire stable record as unmatched even when its quotation and author appeared in full, obscuring that the actual failure was the tag-list boundary.", "For each record, show the reference, parsed value, and verdict for quotation, author, and tags; when all tag characters are present but separators disappear, explicitly mark the field boundary invalid.", "Only the failure explanation and classification are corrected; the task contract, raw output, latency, and score do not change."], + ["Review of two Moli versions", "Looking only at Moli 1.1.3 would miss whether the earlier version had the same problem.", "The outputs from Moli 1.1.1 and 1.1.3 for this task are byte-for-byte identical, each with 10 complete unique records. In 8 records, multiple tags run together, and the last record also includes Loading text.", "Both versions remain failed. Single-tag record 9 matches normally, showing that the rule checks tag boundaries rather than whether the quotation appears."], + ["Failure-review coverage", "Readers need to distinguish full review from checks of only selected versions.", "All 900/900 primary results were blindly reviewed, including Moli 1.1.1 and 1.1.3 with 150 each. Every one of the 184 machine failures has nonempty independent review evidence.", "Failure-review coverage is 100% for every method, with no unadjudicated results."] ] } } @@ -5191,42 +3745,18 @@ "删除与目标无关的表格语法限制；继续用 3 个稳定事实、五段正文覆盖和完整文本质量验收，同一规则作用于全部方法。", "沿用原始输出和耗时离线重评分，不重跑 provider；3 次结果由失败改为通过。" ] - ], + ] + , "i18n": { "en": { "title": "Audit document references and correct equivalent-table misclassification", "summary": "Audit all 10 document tasks across their frozen sources, complete body references, body-region contracts, and 60 actual outputs, and recheck the Feishu help-document table verdict. No document reference was truncated. The Feishu task had incorrectly treated Markdown table syntax as a content requirement, causing 3 misclassifications that are now corrected under the same input-output target.", "changes": [ - [ - "Coverage presentation", - "Region details placed a short inspection snippet beside the long reference body, making it easy to mistake the snippet for the only text used in machine scoring.", - "The result page now explains the denominator, numerator, and complete input in place: coverage is computed from the complete actual output above, while the locator snippet only helps readers inspect one match.", - "The scoring implementation, saved output, latency, and score are unchanged." - ], - [ - "Minimal PDF scope", - "The source body for document-03 contains only Dummy PDF file; reusing the long-document description would make the reference look incomplete.", - "The report explicitly labels this a minimal document-parsing sample: the complete extractable body of the frozen source is 14 characters, and the task checks only whether short PDF text can be obtained.", - "This task cannot establish support for multiple pages, tables, or appendices. None of the six methods retrieved these 14 characters in this run, so all results remain failed." - ], - [ - "Document-reference audit", - "When an embedded-document task retains only the surrounding HTML, readers cannot trace which PDF supplied the complete body.", - "document-07 records the PDF actually loaded by the viewer, the existing frozen-file path, and two hashes. document-08 records the embedded URL in the frozen HTML, the independent-download hash, and the extracted-body hash.", - "All 10 document references and body-region contracts passed review; no score changed because of a truncated reference." - ], - [ - "Moli DOCX description", - "The report said the direct DOCX was obtained as a raw response, but Moli 1.1.3 actually serialized that DOCX as HTML containing package bytes in this run.", - "The method page now describes automatic selection accurately: the PDF became a raw response and was parsed, while the DOCX did not become a parseable original file and failed.", - "No external fallback such as curl is added, and another downloader's success is not credited to Moli." - ], - [ - "Equivalent table output", - "The original rule for the Feishu help document accepted only a Markdown pipe table or an HTML table. Obscura and two Moli versions returned all three column headings and the complete table content, but expanded it as plain text in reading order and were therefore misclassified as failures.", - "Remove the table-syntax restriction, which is unrelated to the target, while retaining acceptance based on 3 stable facts, five-region body coverage, and complete-text quality. The same rule applies to every method.", - "Rescore the existing raw outputs and latency offline without rerunning the provider; 3 results change from fail to pass." - ] + ["Coverage presentation", "Region details placed a short inspection snippet beside the long reference body, making it easy to mistake the snippet for the only text used in machine scoring.", "The result page now explains the denominator, numerator, and complete input in place: coverage is computed from the complete actual output above, while the locator snippet only helps readers inspect one match.", "The scoring implementation, saved output, latency, and score are unchanged."], + ["Minimal PDF scope", "The source body for document-03 contains only Dummy PDF file; reusing the long-document description would make the reference look incomplete.", "The report explicitly labels this a minimal document-parsing sample: the complete extractable body of the frozen source is 14 characters, and the task checks only whether short PDF text can be obtained.", "This task cannot establish support for multiple pages, tables, or appendices. None of the six methods retrieved these 14 characters in this run, so all results remain failed."], + ["Document-reference audit", "When an embedded-document task retains only the surrounding HTML, readers cannot trace which PDF supplied the complete body.", "document-07 records the PDF actually loaded by the viewer, the existing frozen-file path, and two hashes. document-08 records the embedded URL in the frozen HTML, the independent-download hash, and the extracted-body hash.", "All 10 document references and body-region contracts passed review; no score changed because of a truncated reference."], + ["Moli DOCX description", "The report said the direct DOCX was obtained as a raw response, but Moli 1.1.3 actually serialized that DOCX as HTML containing package bytes in this run.", "The method page now describes automatic selection accurately: the PDF became a raw response and was parsed, while the DOCX did not become a parseable original file and failed.", "No external fallback such as curl is added, and another downloader's success is not credited to Moli."], + ["Equivalent table output", "The original rule for the Feishu help document accepted only a Markdown pipe table or an HTML table. Obscura and two Moli versions returned all three column headings and the complete table content, but expanded it as plain text in reading order and were therefore misclassified as failures.", "Remove the table-syntax restriction, which is unrelated to the target, while retaining acceptance based on 3 stable facts, five-region body coverage, and complete-text quality. The same rule applies to every method.", "Rescore the existing raw outputs and latency offline without rerunning the provider; 3 results change from fail to pass."] ] } } @@ -5292,60 +3822,21 @@ "组装器保留源证据顺序，并将参考修订、评分修订和结果核验记录随报告发布。", "900/900 指标可从保存输出复现；150 个任务和 6 种方法均无缺项。" ] - ], + ] + , "i18n": { "en": { "title": "Correct document and resource routes and independently review all 900 results", "summary": "Across 150 tasks × 6 methods, all 900 calls have explicit results. Moli 1.1.3 now uses automatic raw responses for direct documents and media. Codex gpt-5.6-sol blindly reviewed all 900 saved outputs; every disagreement was adjudicated and none remain unresolved. Task detail pages show the reference answer, actual output, and verdict evidence directly.", "changes": [ - [ - "Correct route for Moli 1.1.3", - "Using --dump html/markdown for a direct PDF errors before body retrieval and cannot measure the raw-response capability of 1.1.3 accurately.", - "Ordinary pages continue to use native Markdown. Direct documents and media use bare fetch, allowing the CLI to select a raw response or serialized page automatically. PDF, PNG, WebM, and MP4 returned original bytes in this run; DOCX was serialized as HTML and failed.", - "Only the 7 direct-document calls and 3 direct-file calls affected by the implementation are replaced. Moli 1.1.3 ultimately passes 133/150, while the older Moli 1.1.1 passes 123/150." - ], - [ - "Separate document-body and file tracks", - "Downloading a PDF or DOCX is not the same as obtaining its body text; conversely, a body task should not force every method through the same intermediate format.", - "Body tasks uniformly require readable text: direct body output is accepted, while an original file is parsed locally with parsing time included. Original-file tasks validate format, length, and target SHA-256 without an unrelated Markdown conversion.", - "Current coverage includes PDF, DOCX, TXT, and embedded documents; it does not claim support for legacy DOC, OCR, table semantics, or arbitrary office formats." - ], - [ - "Independent review of every result", - "Reviewing only automated failures would miss false positives. Giving the agent only an empty text placeholder for file tasks would also create false failures.", - "Codex gpt-5.6-sol reviewed 900/900 results without seeing machine verdicts or scores. File tasks also supplied real binary evidence, and the 3 batches affected by a protocol defect were reviewed separately.", - "Every disagreement has observable evidence and a final adjudication, with 0 unresolved; all 120/120 same-window latency results were reviewed separately." - ], - [ - "Shared scoring correction", - "Markdown link boundaries could make an existing Reddit community description appear missing, and percent-encoded relative download URLs could also be missed.", - "Restore visible link text at layout boundaries. Resolve and URL-decode only angle-bracketed relative targets with file extensions against the source page; do not treat ordinary filename prose as a URL.", - "False negatives for two Moli versions on challenge-08 are corrected, and Chromium's Blender download-package URL is changed to pass. Raw output, call status, latency, and content hashes remain unchanged." - ], - [ - "Corrupted-text detection", - "A successful request and long output do not establish readable text; run-9b35581eb0fdbbdea4a6 contains many replacement characters.", - "Consistently measure the count and share of U+FFFD, classify the body as decode-corrupted at a material threshold, and show the count, ratio, and raw output on the detail page.", - "The call remains failed, but its reason changes from generic missing content to verifiable decode corruption; garbled text is not treated as valid body text." - ], - [ - "Task-detail redesign", - "The old page labeled five regions from one answer as body-region, while hiding the reference answer and actual output behind numerous tables and disclosure controls.", - "The first screen shows task ID, method, run ID, and conclusion, followed by the complete reference answer and actual output side by side. Region checks retain only a summary table, source evidence expands on demand, and run fields are grouped in a disclosure at the end.", - "Only evidence presentation changes; no task, output, score, or latency value is modified." - ], - [ - "Version traceability", - "When the runtime version string is missing, a product name alone cannot establish which build was tested.", - "Local implementations are pinned by binary SHA-256: Moli 1.1.1 was matched byte for byte to the official archive afterward; Moli 1.1.3 and Chromium retain runtime versions and hashes; Lightpanda is identified by its exact nightly hash.", - "Kitesurf is a hosted service whose server build is unpublished; the report marks it unavailable instead of inventing a version." - ], - [ - "Evidence packaging", - "Changing JSONL row order while reassembling the same records invalidates independent-review hashes.", - "The assembler preserves source-evidence order and publishes reference revisions, scoring revisions, and result-review records with the report.", - "The 900/900 metric is reproducible from saved outputs; none of the 150 tasks or 6 methods is missing." - ] + ["Correct route for Moli 1.1.3", "Using --dump html/markdown for a direct PDF errors before body retrieval and cannot measure the raw-response capability of 1.1.3 accurately.", "Ordinary pages continue to use native Markdown. Direct documents and media use bare fetch, allowing the CLI to select a raw response or serialized page automatically. PDF, PNG, WebM, and MP4 returned original bytes in this run; DOCX was serialized as HTML and failed.", "Only the 7 direct-document calls and 3 direct-file calls affected by the implementation are replaced. Moli 1.1.3 ultimately passes 133/150, while the older Moli 1.1.1 passes 123/150."], + ["Separate document-body and file tracks", "Downloading a PDF or DOCX is not the same as obtaining its body text; conversely, a body task should not force every method through the same intermediate format.", "Body tasks uniformly require readable text: direct body output is accepted, while an original file is parsed locally with parsing time included. Original-file tasks validate format, length, and target SHA-256 without an unrelated Markdown conversion.", "Current coverage includes PDF, DOCX, TXT, and embedded documents; it does not claim support for legacy DOC, OCR, table semantics, or arbitrary office formats."], + ["Independent review of every result", "Reviewing only automated failures would miss false positives. Giving the agent only an empty text placeholder for file tasks would also create false failures.", "Codex gpt-5.6-sol reviewed 900/900 results without seeing machine verdicts or scores. File tasks also supplied real binary evidence, and the 3 batches affected by a protocol defect were reviewed separately.", "Every disagreement has observable evidence and a final adjudication, with 0 unresolved; all 120/120 same-window latency results were reviewed separately."], + ["Shared scoring correction", "Markdown link boundaries could make an existing Reddit community description appear missing, and percent-encoded relative download URLs could also be missed.", "Restore visible link text at layout boundaries. Resolve and URL-decode only angle-bracketed relative targets with file extensions against the source page; do not treat ordinary filename prose as a URL.", "False negatives for two Moli versions on challenge-08 are corrected, and Chromium's Blender download-package URL is changed to pass. Raw output, call status, latency, and content hashes remain unchanged."], + ["Corrupted-text detection", "A successful request and long output do not establish readable text; run-9b35581eb0fdbbdea4a6 contains many replacement characters.", "Consistently measure the count and share of U+FFFD, classify the body as decode-corrupted at a material threshold, and show the count, ratio, and raw output on the detail page.", "The call remains failed, but its reason changes from generic missing content to verifiable decode corruption; garbled text is not treated as valid body text."], + ["Task-detail redesign", "The old page labeled five regions from one answer as body-region, while hiding the reference answer and actual output behind numerous tables and disclosure controls.", "The first screen shows task ID, method, run ID, and conclusion, followed by the complete reference answer and actual output side by side. Region checks retain only a summary table, source evidence expands on demand, and run fields are grouped in a disclosure at the end.", "Only evidence presentation changes; no task, output, score, or latency value is modified."], + ["Version traceability", "When the runtime version string is missing, a product name alone cannot establish which build was tested.", "Local implementations are pinned by binary SHA-256: Moli 1.1.1 was matched byte for byte to the official archive afterward; Moli 1.1.3 and Chromium retain runtime versions and hashes; Lightpanda is identified by its exact nightly hash.", "Kitesurf is a hosted service whose server build is unpublished; the report marks it unavailable instead of inventing a version."], + ["Evidence packaging", "Changing JSONL row order while reassembling the same records invalidates independent-review hashes.", "The assembler preserves source-evidence order and publishes reference revisions, scoring revisions, and result-review records with the report.", "The 900/900 metric is reproducible from saved outputs; none of the 150 tasks or 6 methods is missing."] ] } } @@ -5399,54 +3890,20 @@ "发布目标固定为独立站点 bWkRP157EtF6。", "旧站点 UeZE1vfcEOrM 保持不变。" ] - ], + ] + , "i18n": { "en": { "title": "Complete 750 measured calls on the fastest Markdown routes of five browsers", "summary": "Kitesurf, Lightpanda, Obscura, and Moli use native Markdown, while Chromium uses a rendered DOM with node-html-markdown 2.0.0. All 150 tasks × 5 methods were executed, and the independent-reference lifecycle and offline rescoring both passed.", "changes": [ - [ - "Method contract", - "The old draft forced all five methods through the same HTML conversion chain, obscuring the advantage of native interfaces.", - "Align URLs, Markdown output, task set, acceptance rules, and the 45-second deadline; each method uses its fastest official or built-in Markdown route.", - "Kitesurf uses the account-level Cloudflare /markdown route; three local CLIs use native Markdown; only Chromium, which lacks native Markdown, converts the DOM." - ], - [ - "Chromium route", - "The old pyhtml2md route converted large pages slowly and passed one fewer ordinary web task.", - "Rescore candidate converters on the same saved DOM batch, adopt node-html-markdown 2.0.0, and retain the HTML, version, hash, and conversion latency.", - "Chromium passes 98/100 ordinary web tasks in this run. Conversion is included in client end-to-end latency and is not presented as a native browser capability." - ], - [ - "Historical-verdict review", - "The old rules missed line breaks across Markdown links, native-interface link layout, layout whitespace, and some dynamic expressions, and had also accepted incomplete body text.", - "Compare all 750 saved outputs individually with their reference answers and task contracts, correcting 43 false negatives and 1 false positive under shared rules; run-f4e6f2ee641e8baa9e3d is confirmed as misclassified.", - "The corrected rules apply to every method without provider-specific exceptions; raw outputs, call identities, and latency values do not change." - ], - [ - "Sync with main", - "Upstream task-rule and report-defect corrections can affect earlier conclusions.", - "After merging the latest main, review the affected scope. Because the Django tutorial reference changed, rerun the matched task for all five methods; reuse other results only when reference bytes and evaluation contracts are equivalent.", - "The new Moli call for this task exits the browser, leaving its ordinary-web result at 93/100; the old pass is not retained." - ], - [ - "Fairness", - "Remote Kitesurf, local processes, differing readiness conditions, and conversion time could be misread as pure engine performance.", - "Pair speed only on identical URLs passed by both methods and state the client end-to-end boundary explicitly.", - "The results cannot establish same-machine CPU/RSS, network-excluded engine speed, sustained throughput, cost, or an SLA." - ], - [ - "Task identification", - "Task detail pages showed only titles, so readers could not confirm task IDs from the page.", - "Both task pages and per-call pages now show stable task IDs immediately; run IDs, rules, references, and outputs remain traceable.", - "The presentation change does not alter tasks, results, or scores." - ], - [ - "Publication isolation", - "This report serves a different purpose from the old report, and overwriting the old site would break historical links.", - "The publication target is pinned to the separate site bWkRP157EtF6.", - "The old site UeZE1vfcEOrM remains unchanged." - ] + ["Method contract", "The old draft forced all five methods through the same HTML conversion chain, obscuring the advantage of native interfaces.", "Align URLs, Markdown output, task set, acceptance rules, and the 45-second deadline; each method uses its fastest official or built-in Markdown route.", "Kitesurf uses the account-level Cloudflare /markdown route; three local CLIs use native Markdown; only Chromium, which lacks native Markdown, converts the DOM."], + ["Chromium route", "The old pyhtml2md route converted large pages slowly and passed one fewer ordinary web task.", "Rescore candidate converters on the same saved DOM batch, adopt node-html-markdown 2.0.0, and retain the HTML, version, hash, and conversion latency.", "Chromium passes 98/100 ordinary web tasks in this run. Conversion is included in client end-to-end latency and is not presented as a native browser capability."], + ["Historical-verdict review", "The old rules missed line breaks across Markdown links, native-interface link layout, layout whitespace, and some dynamic expressions, and had also accepted incomplete body text.", "Compare all 750 saved outputs individually with their reference answers and task contracts, correcting 43 false negatives and 1 false positive under shared rules; run-f4e6f2ee641e8baa9e3d is confirmed as misclassified.", "The corrected rules apply to every method without provider-specific exceptions; raw outputs, call identities, and latency values do not change."], + ["Sync with main", "Upstream task-rule and report-defect corrections can affect earlier conclusions.", "After merging the latest main, review the affected scope. Because the Django tutorial reference changed, rerun the matched task for all five methods; reuse other results only when reference bytes and evaluation contracts are equivalent.", "The new Moli call for this task exits the browser, leaving its ordinary-web result at 93/100; the old pass is not retained."], + ["Fairness", "Remote Kitesurf, local processes, differing readiness conditions, and conversion time could be misread as pure engine performance.", "Pair speed only on identical URLs passed by both methods and state the client end-to-end boundary explicitly.", "The results cannot establish same-machine CPU/RSS, network-excluded engine speed, sustained throughput, cost, or an SLA."], + ["Task identification", "Task detail pages showed only titles, so readers could not confirm task IDs from the page.", "Both task pages and per-call pages now show stable task IDs immediately; run IDs, rules, references, and outputs remain traceable.", "The presentation change does not alter tasks, results, or scores."], + ["Publication isolation", "This report serves a different purpose from the old report, and overwriting the old site would break historical links.", "The publication target is pinned to the separate site bWkRP157EtF6.", "The old site UeZE1vfcEOrM remains unchanged."] ] } } @@ -5494,48 +3951,19 @@ "六类任务概览前置，直接列出样本数、完整判分的方法数、最高任务通过率及对应方法；详细榜单留在后方。", "不同类别任务要求不同，最高分不能跨类别直接比较。" ] - ], + ] + , "i18n": { "en": { "title": "Complete file-task evaluation and establish a report-completeness publication gate", "summary": "All 36 file-task calls were actually made and judged: Moli and Crawl4AI each pass 3, for 6 passes, 30 failures, and 0 unmeasured. Every one of the report's 1,800 planned calls has a unique result and explicit verdict.", "changes": [ - [ - "File tasks", - "Earlier records did not retain original-file bytes, so retrieval of direct PNG, WebM, and MP4 files could not be judged.", - "Each of the twelve methods runs 3 file tasks; all 36 calls in the final configuration are validated by format, length, and target SHA-256.", - "Moli and Crawl4AI each pass 3 calls, while the other 30 fail. Failures remain real results, and valid difficult tasks are not removed." - ], - [ - "Native-interface scope", - "Testing only body-text routes would miss publicly offered native file capabilities and mistake a partial configuration for product capability.", - "Moli file tasks output original-file bytes. Crawl4AI file tasks preselect its native HTTP crawler, while body tasks retain the browser route.", - "Evaluate the public interface fixed before the task begins, without post-failure fallback or presenting an ordinary HTTP download as a provider result." - ], - [ - "Timing boundary", - "Latency did not clearly distinguish server time, downloading, rendering, local processing, and report processing.", - "Use one client end-to-end timing boundary: from starting the provider call until a scoreable final answer exists. Downloading, rendering, extraction, input conversion, and file hashing are included.", - "Scoring, evidence persistence, rate-limit waits, and report generation are excluded; provider server-side latency is not ranked." - ], - [ - "Completeness gate", - "Using actual calls as the denominator could make 8/8 appear complete despite missing planned calls; two coverage columns were also redundant.", - "Change to “judgement coverage = explicitly judged calls / planned calls” and require exactly one current result for every provider × case.", - "Any missing, duplicate, or unjudgeable result marks the entire report unreliable; completeness checks, report validation, and release packaging all block publication of incomplete results." - ], - [ - "Evidence presentation", - "Successful files were retained in the bundle but difficult to verify directly from call pages, while the changelog stacked several same-day revisions.", - "Successful-call pages provide file downloads, format, byte count, and SHA-256. Same-day revisions are merged into one official release, with earlier releases collapsed.", - "Only audit access and reading structure improve; original files, scores, and latency values do not change." - ], - [ - "Overview structure", - "The family overview was hidden after detailed rankings, forcing readers through large tables first.", - "Move the six-family overview forward and list sample count, methods with complete judgements, highest task pass rate, and the corresponding methods directly; detailed rankings remain later.", - "Task requirements differ by family, so top scores cannot be compared directly across families." - ] + ["File tasks", "Earlier records did not retain original-file bytes, so retrieval of direct PNG, WebM, and MP4 files could not be judged.", "Each of the twelve methods runs 3 file tasks; all 36 calls in the final configuration are validated by format, length, and target SHA-256.", "Moli and Crawl4AI each pass 3 calls, while the other 30 fail. Failures remain real results, and valid difficult tasks are not removed."], + ["Native-interface scope", "Testing only body-text routes would miss publicly offered native file capabilities and mistake a partial configuration for product capability.", "Moli file tasks output original-file bytes. Crawl4AI file tasks preselect its native HTTP crawler, while body tasks retain the browser route.", "Evaluate the public interface fixed before the task begins, without post-failure fallback or presenting an ordinary HTTP download as a provider result."], + ["Timing boundary", "Latency did not clearly distinguish server time, downloading, rendering, local processing, and report processing.", "Use one client end-to-end timing boundary: from starting the provider call until a scoreable final answer exists. Downloading, rendering, extraction, input conversion, and file hashing are included.", "Scoring, evidence persistence, rate-limit waits, and report generation are excluded; provider server-side latency is not ranked."], + ["Completeness gate", "Using actual calls as the denominator could make 8/8 appear complete despite missing planned calls; two coverage columns were also redundant.", "Change to “judgement coverage = explicitly judged calls / planned calls” and require exactly one current result for every provider × case.", "Any missing, duplicate, or unjudgeable result marks the entire report unreliable; completeness checks, report validation, and release packaging all block publication of incomplete results."], + ["Evidence presentation", "Successful files were retained in the bundle but difficult to verify directly from call pages, while the changelog stacked several same-day revisions.", "Successful-call pages provide file downloads, format, byte count, and SHA-256. Same-day revisions are merged into one official release, with earlier releases collapsed.", "Only audit access and reading structure improve; original files, scores, and latency values do not change."], + ["Overview structure", "The family overview was hidden after detailed rankings, forcing readers through large tables first.", "Move the six-family overview forward and list sample count, methods with complete judgements, highest task pass rate, and the corresponding methods directly; detailed rankings remain later.", "Task requirements differ by family, so top scores cannot be compared directly across families."] ] } } @@ -5590,54 +4018,20 @@ "增加共同通过子集诊断，并明确单次、跨批次与参考范围限制。", "不能从本轮样本外推全网能力、稳定性或生产 SLA。" ] - ], + ] + , "i18n": { "en": { "title": "Unify acceptance criteria and correct references, scores, and report conclusions", "summary": "Retain 150 tasks and 1,800 original calls. Verdicts change for 92 tasks, and 36 historical records lacking file-byte evidence change from incorrectly recorded failures to unmeasured. Raw outputs and latency values are unchanged.", "changes": [ - [ - "Body references", - "The CSS reference body's table of contents was incorrectly removed, while the Django document included surrounding site content.", - "Redetermine body boundaries from independently saved source pages and record provenance and reference hashes.", - "Full-text F1 changes with the corrected references; earlier low scores cannot be attributed directly to provider body-text noise." - ], - [ - "Shared scoring", - "Explicit name-link fields and valid Markdown soft line breaks were not handled consistently in acceptance.", - "All methods share one answer transformation and scoring entry point, excluding request echoes and diagnostic logs.", - "Correct format-induced false failures without scoring branches keyed by provider name." - ], - [ - "Body coverage", - "Checking only opening keywords and minimum length could let a very short excerpt pass a long-document task.", - "For tasks with a complete independent reference, add token-coverage checks across up to five contiguous regions.", - "This is a minimum content-coverage requirement, not a replacement for full-text F1 or a guarantee of semantic or factual correctness." - ], - [ - "Dynamic fields", - "Required installment and delivery fields in the product task were not fully validated.", - "Check field ownership, count, type, and uniqueness against independent references; volatile prices and ordering are not frozen as permanent answers.", - "Complete fields do not establish that volatile values are accurate or current." - ], - [ - "Deadline", - "A result exceeding the registered budget could still count as successful.", - "New calls use a shared end-to-end deadline; historical records are rejudged only from their registered latency and budget.", - "The report does not claim that historical calls were originally controlled by the new hard deadline." - ], - [ - "Unmeasured files", - "36 file records without byte evidence were treated as failures.", - "First mark them unmeasured and remove them from the pass-rate denominator, then fill them with real provider calls on 2026-09-04.", - "Do not substitute a URL, text, or another method's download for the original method's file evidence." - ], - [ - "Interpreting results", - "Conditional latency from different passing subsets was treated as matched-task speed, and some article conclusions exceeded the evidence.", - "Add diagnostics for the jointly passing subset and state single-run, cross-batch, and reference-scope limitations explicitly.", - "This sample cannot establish internet-wide capability, stability, or a production SLA." - ] + ["Body references", "The CSS reference body's table of contents was incorrectly removed, while the Django document included surrounding site content.", "Redetermine body boundaries from independently saved source pages and record provenance and reference hashes.", "Full-text F1 changes with the corrected references; earlier low scores cannot be attributed directly to provider body-text noise."], + ["Shared scoring", "Explicit name-link fields and valid Markdown soft line breaks were not handled consistently in acceptance.", "All methods share one answer transformation and scoring entry point, excluding request echoes and diagnostic logs.", "Correct format-induced false failures without scoring branches keyed by provider name."], + ["Body coverage", "Checking only opening keywords and minimum length could let a very short excerpt pass a long-document task.", "For tasks with a complete independent reference, add token-coverage checks across up to five contiguous regions.", "This is a minimum content-coverage requirement, not a replacement for full-text F1 or a guarantee of semantic or factual correctness."], + ["Dynamic fields", "Required installment and delivery fields in the product task were not fully validated.", "Check field ownership, count, type, and uniqueness against independent references; volatile prices and ordering are not frozen as permanent answers.", "Complete fields do not establish that volatile values are accurate or current."], + ["Deadline", "A result exceeding the registered budget could still count as successful.", "New calls use a shared end-to-end deadline; historical records are rejudged only from their registered latency and budget.", "The report does not claim that historical calls were originally controlled by the new hard deadline."], + ["Unmeasured files", "36 file records without byte evidence were treated as failures.", "First mark them unmeasured and remove them from the pass-rate denominator, then fill them with real provider calls on 2026-09-04.", "Do not substitute a URL, text, or another method's download for the original method's file evidence."], + ["Interpreting results", "Conditional latency from different passing subsets was treated as matched-task speed, and some article conclusions exceeded the evidence.", "Add diagnostics for the jointly passing subset and state single-run, cross-batch, and reference-scope limitations explicitly.", "This sample cannot establish internet-wide capability, stability, or a production SLA."] ] } } From 3c19b47d8f4e4c78631b80789b73994c3d504baf Mon Sep 17 00:00:00 2001 From: lanyue-llk <270302213+lanyue-llk@users.noreply.github.com> Date: Sat, 10 Oct 2026 19:17:45 +0800 Subject: [PATCH 005/172] fix(publication): avoid rehashing unchanged payloads for every staging packet --- harness/reporting/publication.py | 51 +++++++++++++++++++++++++---- harness/web_fetch/report_publish.py | 13 +++++--- tests/test_reporting_publication.py | 39 +++++++++++++++++++++- 3 files changed, 91 insertions(+), 12 deletions(-) diff --git a/harness/reporting/publication.py b/harness/reporting/publication.py index 55c2d9e49..c95792775 100644 --- a/harness/reporting/publication.py +++ b/harness/reporting/publication.py @@ -7,12 +7,13 @@ import os import re import shutil +import stat import tempfile import urllib.parse import zipfile from collections.abc import Callable, Sequence from contextlib import contextmanager -from dataclasses import dataclass +from dataclasses import dataclass, field from pathlib import Path from harness.web_fetch.safety import ( @@ -27,22 +28,58 @@ class PublicationSnapshot: archive: Path sha256: str sources: dict + signatures: dict = field(default_factory=dict) def check(self) -> None: - if any(_identity(path) != value for path, value in self.sources.items()): - raise ValueError('publication inputs changed after validation') - if _identity(self.archive) != self.sha256: - raise ValueError('frozen publication archive changed') + for path, value in {**self.sources, self.archive: self.sha256}.items(): + signature = _file_signature(path) + # Windows ctime is creation time, so retain byte checks there. + if os.name != 'nt' and self.signatures.get(path) == signature: + continue + if _identity(path) != value or _file_signature(path) != signature: + raise ValueError('publication inputs changed after validation') + self.signatures[path] = signature + + def verify(self) -> None: + """Rehash the complete bytes at the final publication boundary.""" + for path, value in {**self.sources, self.archive: self.sha256}.items(): + if _identity(path) != value: + raise ValueError('publication inputs changed after validation') + + +def _file_signature(path: Path): + """Kernel change times prevent same-size writes or restored mtimes hiding edits.""" + entries = {} + + def visit(current: Path, relative: str): + info = current.lstat() + if stat.S_ISLNK(info.st_mode): + raise ValueError('Publication inputs contain a symlink') + entries[relative] = (info.st_dev, info.st_ino, info.st_mode, + info.st_size, info.st_mtime_ns, info.st_ctime_ns) + if stat.S_ISDIR(info.st_mode): + with os.scandir(current) as children: + for child in children: + visit(Path(child.path), relative + '/' + child.name) + elif not stat.S_ISREG(info.st_mode): + raise ValueError('Publication inputs must be regular files or directories') + + visit(path, '') + return entries @contextmanager def snapshot_publication(archive: Path, *, inputs: Sequence[Path]): """Upload a private byte-exact copy while guarding the release authority.""" - sources = {path: _identity(path) for path in dict.fromkeys((archive, *inputs))} + paths = dict.fromkeys((archive, *inputs)) + signatures = {path: _file_signature(path) for path in paths} + sources = {path: _identity(path) for path in paths} + if any(_file_signature(path) != signatures[path] for path in paths): + raise ValueError('publication inputs changed during snapshot creation') with tempfile.TemporaryDirectory(prefix='publication-snapshot-') as work: frozen = Path(work) / 'archive.zip' shutil.copyfile(archive, frozen) - snapshot = PublicationSnapshot(frozen, sources[archive], sources) + snapshot = PublicationSnapshot(frozen, sources[archive], sources, signatures) snapshot.check() yield snapshot diff --git a/harness/web_fetch/report_publish.py b/harness/web_fetch/report_publish.py index a5c4d7bef..021884dc3 100644 --- a/harness/web_fetch/report_publish.py +++ b/harness/web_fetch/report_publish.py @@ -188,7 +188,9 @@ def finish(value): if config.mode == "reuse": if package_check is None: return finish(result) - with snapshot_publication(config.archive, inputs=[config.profile, config.source, *validation_inputs]) as release: + with snapshot_publication( + config.archive, inputs=[config.profile, config.source, *validation_inputs] + ) as release: release_guard = release.check package_check(release.archive, config.source) release.check() @@ -198,14 +200,17 @@ def finish(value): for path in (config.archive, config.receipt, config.journal): if path.resolve().is_relative_to(config.source.resolve()): raise ValueError("Publication outputs must stay outside the source report") - with snapshot_publication(config.archive, inputs=[config.profile, config.source, *validation_inputs]) as release: + with snapshot_publication( + config.archive, inputs=[config.profile, config.source, *validation_inputs] + ) as release: release_guard = release.check (package_check or check)(release.archive, config.source) release.check() original_send = send def guarded_send(*args, **kwargs): - release.check() + if '/commit' in args[0] or '/versions' in args[0]: + release.verify() return original_send(*args, **kwargs) send = guarded_send @@ -284,7 +289,7 @@ def staged_send(path, data=None, content_type=None, *, method=None): ) save(result) summary = finish(result) - release.check() + release.verify() operations.clear(identity) return summary diff --git a/tests/test_reporting_publication.py b/tests/test_reporting_publication.py index e09d2b5c4..985fcb9fd 100644 --- a/tests/test_reporting_publication.py +++ b/tests/test_reporting_publication.py @@ -1,11 +1,48 @@ """Release archives must contain exactly the validated bytes or preserve the prior release.""" import json +import os import zipfile from pathlib import Path import pytest -from harness.reporting.publication import package_archive +from harness.reporting.publication import package_archive, snapshot_publication + + +@pytest.mark.skipif(os.name == 'nt', reason='Windows lacks Unix change-time semantics') +def test_packet_guards_do_not_reread_unchanged_report_payloads(tmp_path, monkeypatch): + report, authority, output, _ = fixture(tmp_path) + with snapshot_publication(output, inputs=[report, authority]) as snapshot: + with monkeypatch.context() as guards: + guards.setattr(Path, 'read_bytes', lambda _: pytest.fail('unchanged payload reread')) + for _ in range(100): + snapshot.check() + snapshot.verify() + + +@pytest.mark.parametrize('damage', ['page', 'profile', 'source-archive', 'frozen-archive', + 'new-file', 'deleted-file', 'replacement', 'symlink']) +def test_packet_guards_detect_input_edits_even_with_restored_mtime(tmp_path, damage): + report, authority, output, _ = fixture(tmp_path) + with snapshot_publication(output, inputs=[report, authority]) as snapshot: + target = {'page': report / 'index.html', 'profile': authority, + 'source-archive': output, 'frozen-archive': snapshot.archive}.get(damage) + if target is not None: + before = target.stat() + target.write_bytes(target.read_bytes()[::-1]) + os.utime(target, ns=(before.st_atime_ns, before.st_mtime_ns)) + elif damage == 'new-file': + (report / 'extra.txt').write_text('unexpected') + elif damage == 'deleted-file': + (report / 'index.html').unlink() + elif damage == 'replacement': + page = report / 'index.html' + page.unlink() + page.write_text('different page') + else: + (report / 'linked').symlink_to(authority) + with pytest.raises(ValueError): + snapshot.check() def fixture(tmp_path): From de620327901f6a527d7771895b3b773e9c1c37c2 Mon Sep 17 00:00:00 2001 From: lanyue-llk <270302213+lanyue-llk@users.noreply.github.com> Date: Sat, 10 Oct 2026 19:21:29 +0800 Subject: [PATCH 006/172] fix(publication): recover uncommitted operations from HTTP 404 responses --- harness/reporting/publication.py | 9 ++++++++- tests/test_reporting_publication.py | 21 +++++++++++++++++++++ 2 files changed, 29 insertions(+), 1 deletion(-) diff --git a/harness/reporting/publication.py b/harness/reporting/publication.py index c95792775..a7ea5b604 100644 --- a/harness/reporting/publication.py +++ b/harness/reporting/publication.py @@ -15,6 +15,7 @@ from contextlib import contextmanager from dataclasses import dataclass, field from pathlib import Path +from urllib.error import HTTPError from harness.web_fetch.safety import ( contained_path, @@ -393,7 +394,13 @@ def pending(self, identity: dict, *, allow_create: bool) -> tuple[dict, bool]: return pending, False def completed(self, key: str, *, request: Callable, decode: Callable): - status, raw = request('/api/operations/' + urllib.parse.quote(key)) + try: + status, raw = request('/api/operations/' + urllib.parse.quote(key)) + except HTTPError as error: + if error.code != 404: + raise + error.close() + return None if status == 404: return None operation = decode(status, raw, operation='operation recovery') diff --git a/tests/test_reporting_publication.py b/tests/test_reporting_publication.py index 985fcb9fd..f1e8fa41a 100644 --- a/tests/test_reporting_publication.py +++ b/tests/test_reporting_publication.py @@ -237,3 +237,24 @@ def retarget(): run(report, authority, alias / output.name, alias / receipt.name, retarget) assert (output.read_bytes(), receipt.read_bytes()) == before assert not list(other.iterdir()) + + +@pytest.mark.parametrize('status', [404, 403, 410, 500]) +def test_operation_recovery_handles_real_http_errors_without_replacing_key(tmp_path, status): + from urllib.error import HTTPError + + from harness.reporting.publication import PublicationOperation + operation = PublicationOperation(tmp_path / 'pending.json', 'test-') + identity = {'slug': 'existing', 'archive_sha256': 'bound'} + pending, _ = operation.pending(identity, allow_create=True) + before = operation.journal.read_bytes() + error = HTTPError('https://example.test/api/operations/key', status, 'status', {}, None) + def request(*args, **kwargs): + raise error + if status == 404: + assert operation.completed(pending['key'], request=request, decode=lambda *_: None) is None + else: + with pytest.raises(HTTPError) as caught: + operation.completed(pending['key'], request=request, decode=lambda *_: None) + assert caught.value.code == status + assert operation.journal.read_bytes() == before From 31329555b3c431f3c1e3e5018441ef83b0a71bdf Mon Sep 17 00:00:00 2001 From: lanyue-llk <270302213+lanyue-llk@users.noreply.github.com> Date: Sat, 10 Oct 2026 19:39:38 +0800 Subject: [PATCH 007/172] chore(webfetch): bind the published six-configuration report version --- evaluation/browser-native-430/current-report.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/evaluation/browser-native-430/current-report.json b/evaluation/browser-native-430/current-report.json index 43ca2a7fb..960f105d6 100644 --- a/evaluation/browser-native-430/current-report.json +++ b/evaluation/browser-native-430/current-report.json @@ -511,6 +511,6 @@ "native_task_route_contract": 4, "artifact_hub_base_url": "https://artifact-site.lexmount.cn", "artifact_hub_site": "jGXOOJwkdBK_", - "artifact_hub_expected_version": "ver_840272a174e344269e4ebbcc9d114e6a", + "artifact_hub_expected_version": "ver_be2aa3273e2d4e1eb171f6a7b93c964c", "artifact_hub_share_url": "https://artifact-site.lexmount.cn/v/IqGpMUYHWat-dw4n8usQttTptoPzboAo" } From 3ee2f78e6a534f50fc57c84415b8fbee822be912 Mon Sep 17 00:00:00 2001 From: lanyue-llk <270302213+lanyue-llk@users.noreply.github.com> Date: Sat, 10 Oct 2026 20:08:53 +0800 Subject: [PATCH 008/172] fix(report): classify risk-controlled body metrics consistently --- harness/web_fetch/report.py | 2 +- harness/web_fetch/report_check.py | 18 ++++++++++++++++++ research/report-changes.json | 12 ++++++++++++ tests/test_report_body_scope.py | 16 ++++++++++++++-- tests/test_report_check_owner.py | 25 +++++++++++++++++++++++++ 5 files changed, 70 insertions(+), 3 deletions(-) diff --git a/harness/web_fetch/report.py b/harness/web_fetch/report.py index 71b4c7e79..5088b91a2 100644 --- a/harness/web_fetch/report.py +++ b/harness/web_fetch/report.py @@ -1544,7 +1544,7 @@ def ranking(records,cases,track,base='',id=None,compact=False,audit=None): from .report_components import comparable_body_reference title_scope=track=='challenge' and any('title_quality' in r for r in records if cases[r['case_id']].get('track')==track) selected={k:c for k,c in cases.items() if c.get('track','web')==track} - text_track=track in {'web','dynamic','document','challenge'} and any(c.get('task_kind','body') in {'body','structure'} for c in selected.values()) + text_track=track in {'web','dynamic','document','challenge','risk_control'} and any(c.get('task_kind','body') in {'body','structure'} for c in selected.values()) groups=defaultdict(list) for r in records: if cases[r['case_id']].get('track','web')==track:groups[r['provider']].append(r) diff --git a/harness/web_fetch/report_check.py b/harness/web_fetch/report_check.py index cd8eef6e4..061d81e21 100644 --- a/harness/web_fetch/report_check.py +++ b/harness/web_fetch/report_check.py @@ -633,6 +633,24 @@ def check_revision(out: Path, *, authority_root: Path): raise ValueError((path, width)) for table in soup.select("table.ranking"): headers = table.select("thead th") + body_families = {"web", "dynamic", "challenge", "risk_control", "document"} + family = table.get("id", "").removeprefix("ranking-") + if family in body_families and any( + c.get("track", "web") == family and c.get("task_kind", "body") in {"body", "structure"} + for c in dataset["cases"] + ): + labels = [h.get_text() for h in headers] + required = ( + ["Recall", "Precision", "F1"] if is_english else ["完整度", "纯净度", "F1"] + ) + forbidden = ( + ["Resource hit rate", "File retrieval rate"] + if is_english else ["资源命中率", "文件取得率"] + ) + if not all(any(label in h for h in labels) for label in required) or any( + label in h for label in forbidden for h in labels + ): + raise ValueError((path, "Body ranking metric columns", family)) sort = int(table["data-col"]) expected = ( (["Matched repositories"] if is_english else ["命中仓库数"]) diff --git a/research/report-changes.json b/research/report-changes.json index 1c3e5bc57..e564f36ec 100644 --- a/research/report-changes.json +++ b/research/report-changes.json @@ -1323,6 +1323,12 @@ "稳定版与源码分支的调用发生在不同日期", "分别展示完整调用与各方法成功调用的耗时及样本数", "这些批次的差值不作为 PR 导致提速的证据；分支未加入独立的资源测量。" + ], + [ + "正文专项指标", + "风控网站正文表误用了资源和文件指标列", + "按正文任务展示完整度、纯净度、F1 和文字核对覆盖；缺少全文参考时显示不可用", + "只修正展示，成绩、耗时、任务数和浏览器输出不变。" ] ], "i18n": { @@ -1365,6 +1371,12 @@ "Stable and source-branch calls were collected on different dates", "Show all-call and each method’s successful-call latency with population counts", "Batch differences are not evidence of PR-induced speed improvements. The branch is not part of the independent resource cohort." + ], + [ + "Body metrics", + "The risk-controlled website table used resource and file columns", + "Show completeness, purity, F1 and registered-text coverage for body tasks; full-text metrics remain unavailable without a complete reference", + "Presentation only. Scores, latency, task counts and browser outputs are unchanged." ] ] } diff --git a/tests/test_report_body_scope.py b/tests/test_report_body_scope.py index cabbe668c..eab167095 100644 --- a/tests/test_report_body_scope.py +++ b/tests/test_report_body_scope.py @@ -258,9 +258,9 @@ def test_text_coverage_repeated_denominator_removed_but_different_scopes_disclos self.assertEqual(row[7],'100.0%') self.assertNotIn('· 1/2 例',rendered) - def test_four_text_tracks_share_columns_and_text_coverage_semantics(self): + def test_all_text_tracks_share_columns_and_text_coverage_semantics(self): headers=[] - for track in ('web','dynamic','challenge','document'): + for track in ('web','dynamic','challenge','risk_control','document'): c=case('example','body');c['track']=track r=record(c,passed=False) r['metrics']['fact_results']=[{'id':'a','pass':True},{'id':'b','pass':False}] @@ -273,6 +273,18 @@ def test_four_text_tracks_share_columns_and_text_coverage_semantics(self): headers.append([re.sub(r'<[^>]*>','',h) for h in re.findall(r']*>(.*?)',rendered,re.S)]) self.assertTrue(all(h==headers[0] for h in headers)) + def test_body_without_full_reference_keeps_metrics_unavailable(self): + for track in ('web','dynamic','challenge','risk_control','document'): + with self.subTest(track=track): + c=case('example','body',track) + c['ground_truth_main_content']='' + r=record(c) + rendered=ranking([r],{'example':c},track) + row=table_cells(rendered,'ranking-'+track)[0] + self.assertEqual(row[4:7],['—']*3) + self.assertNotIn('资源命中率',rendered) + self.assertNotIn('直接文件必须',rendered) + def test_nonliteral_snapshot_not_promoted_into_invariant_body_quality(self): c=case('example','body'); c['track']='dynamic' c['acceptance_contract']={'schema':'product','minimum_records':3} diff --git a/tests/test_report_check_owner.py b/tests/test_report_check_owner.py index b6845811a..90101cd0d 100644 --- a/tests/test_report_check_owner.py +++ b/tests/test_report_check_owner.py @@ -320,3 +320,28 @@ def test_registered_browser_build_has_its_own_interface_panel(tmp_path, defect): with pytest.raises(ValueError):check_revision(root,authority_root=authority) else: assert check_revision(root,authority_root=authority).startswith('PASS:') + + +@pytest.mark.parametrize('defect', [False, True]) +def test_body_family_rankings_reject_resource_columns(tmp_path, defect): + from harness.web_fetch.report import ranking + from harness.web_fetch.report_check import check_revision + from tests.report_check_fixture import create_revision_report + from tests.test_report_body_scope import case, record + + root, authority = create_revision_report(tmp_path) + dataset=json.loads((root/'dataset.json').read_text()) + for family in ('web','dynamic','challenge','risk_control','document'): + c=case(family,'body',family) + dataset['cases'].append(c) + rendered=ranking([record(c)],{family:c},family) + if defect: + rendered=rendered.replace('完整度','资源命中率') + path=root/'index.html' + path.write_text(path.read_text()+rendered) + write_json(root/'dataset.json',dataset) + if defect: + with pytest.raises(ValueError,match='Body ranking metric columns'): + check_revision(root,authority_root=authority) + else: + assert check_revision(root,authority_root=authority).startswith('PASS:') From c6be370a04257e35bfbfe8a04eaf64b326985a42 Mon Sep 17 00:00:00 2001 From: lanyue-llk <270302213+lanyue-llk@users.noreply.github.com> Date: Sat, 10 Oct 2026 20:09:34 +0800 Subject: [PATCH 009/172] test(report): enforce each body-family publication boundary --- tests/test_report_check_owner.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/tests/test_report_check_owner.py b/tests/test_report_check_owner.py index 90101cd0d..3f1716ad0 100644 --- a/tests/test_report_check_owner.py +++ b/tests/test_report_check_owner.py @@ -322,8 +322,9 @@ def test_registered_browser_build_has_its_own_interface_panel(tmp_path, defect): assert check_revision(root,authority_root=authority).startswith('PASS:') +@pytest.mark.parametrize('family', ['web','dynamic','challenge','risk_control','document']) @pytest.mark.parametrize('defect', [False, True]) -def test_body_family_rankings_reject_resource_columns(tmp_path, defect): +def test_body_family_rankings_reject_resource_columns(tmp_path, family, defect): from harness.web_fetch.report import ranking from harness.web_fetch.report_check import check_revision from tests.report_check_fixture import create_revision_report @@ -331,14 +332,13 @@ def test_body_family_rankings_reject_resource_columns(tmp_path, defect): root, authority = create_revision_report(tmp_path) dataset=json.loads((root/'dataset.json').read_text()) - for family in ('web','dynamic','challenge','risk_control','document'): - c=case(family,'body',family) - dataset['cases'].append(c) - rendered=ranking([record(c)],{family:c},family) - if defect: - rendered=rendered.replace('完整度','资源命中率') - path=root/'index.html' - path.write_text(path.read_text()+rendered) + c=case(family,'body',family) + dataset['cases'].append(c) + rendered=ranking([record(c)],{family:c},family) + if defect: + rendered=rendered.replace('完整度','资源命中率') + path=root/'index.html' + path.write_text(path.read_text()+rendered) write_json(root/'dataset.json',dataset) if defect: with pytest.raises(ValueError,match='Body ranking metric columns'): From c1385fcb457921c5af47e244a91d3698a711965c Mon Sep 17 00:00:00 2001 From: lanyue-llk <270302213+lanyue-llk@users.noreply.github.com> Date: Sat, 10 Oct 2026 20:27:37 +0800 Subject: [PATCH 010/172] chore(report): bind verified body-metrics publication --- evaluation/browser-native-430/current-report.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/evaluation/browser-native-430/current-report.json b/evaluation/browser-native-430/current-report.json index 960f105d6..c0f2c6925 100644 --- a/evaluation/browser-native-430/current-report.json +++ b/evaluation/browser-native-430/current-report.json @@ -511,6 +511,6 @@ "native_task_route_contract": 4, "artifact_hub_base_url": "https://artifact-site.lexmount.cn", "artifact_hub_site": "jGXOOJwkdBK_", - "artifact_hub_expected_version": "ver_be2aa3273e2d4e1eb171f6a7b93c964c", + "artifact_hub_expected_version": "ver_94084cc8ebaa48268cb29efaddff0902", "artifact_hub_share_url": "https://artifact-site.lexmount.cn/v/IqGpMUYHWat-dw4n8usQttTptoPzboAo" } From 71b6ca7cbba3c14e98b5cfca798f73c0f462c317 Mon Sep 17 00:00:00 2001 From: lanyue-llk <270302213+lanyue-llk@users.noreply.github.com> Date: Mon, 12 Oct 2026 11:00:40 +0800 Subject: [PATCH 011/172] fix(report): distinguish measured Moli release and source-build versions --- harness/web_fetch/methods.py | 45 ++++++++++++++++++++++-- harness/web_fetch/report.py | 7 +++- harness/web_fetch/report_capabilities.py | 2 ++ harness/web_fetch/report_check.py | 18 ++++++++++ harness/web_fetch/report_explainer.py | 15 +++++--- harness/web_fetch/report_interfaces.py | 4 +-- harness/web_fetch/report_methods.py | 4 +-- research/report-changes.json | 35 ++++++++++++++++++ tests/test_report_check_owner.py | 22 ++++++++++++ tests/test_report_methods.py | 32 +++++++++++++++++ 10 files changed, 171 insertions(+), 13 deletions(-) diff --git a/harness/web_fetch/methods.py b/harness/web_fetch/methods.py index 53fa0f6c3..2e0350f87 100644 --- a/harness/web_fetch/methods.py +++ b/harness/web_fetch/methods.py @@ -3,7 +3,8 @@ import json import importlib.metadata import re -from functools import lru_cache +from functools import lru_cache, wraps +from contextvars import ContextVar from pathlib import Path from typing import Any @@ -30,7 +31,44 @@ def native_browser_engine(name): def browser_method_label(name): - return browser_method_registry().get(name, {}).get("label", name) + return _MEASURED_LABELS.get().get( + name, browser_method_registry().get(name, {}).get("label", name) + ) + + +_MEASURED_LABELS = ContextVar("webfetch_measured_labels", default={}) + + +def measured_method_label(name, fallback): + return _MEASURED_LABELS.get().get(name, fallback) + + +def with_measured_method_labels(render): + """Scope Moli version labels to the actual report's hash-bound binaries.""" + @wraps(render) + def wrapped(out, dataset, records, *args, **kwargs): + path = Path(out) / "implementation-provenance.json" + provenance = json.loads(path.read_text()) if path.is_file() else {} + identities = { + (item["provider"], item["binary_sha256"]): item["version"] + for item in provenance.get("implementations", []) + if item.get("version") and item.get("binary_sha256") + } + labels = {} + for name in {r["provider"] for r in records}: + if native_browser_engine(name) != "moli": + continue + subset = [r for r in records if r["provider"] == name] + hashes = {r.get("provider_metadata", {}).get("binary_sha256") for r in subset} + if hashes and all((name, value) in identities for value in hashes): + versions = sorted({identities[name, value] for value in hashes}) + labels[name] = "Moli " + " / ".join(versions) + token = _MEASURED_LABELS.set(labels) + try: + return render(out, dataset, records, *args, **kwargs) + finally: + _MEASURED_LABELS.reset(token) + return wrapped METHOD_REGISTRY: dict[str, dict[str, Any]] = { @@ -104,7 +142,8 @@ def method_page_model(name: str, label: str, records: list[dict[str, Any]]) -> d version_rule["pattern"], version_rule.get("replacement", ""), versions[0], flags=re.I, ) - title += f" · {version}" + if version not in title: + title += f" · {version}" model = { "title": title, "contract": info["contract"], diff --git a/harness/web_fetch/report.py b/harness/web_fetch/report.py index 5088b91a2..05738f2ed 100644 --- a/harness/web_fetch/report.py +++ b/harness/web_fetch/report.py @@ -298,7 +298,9 @@ def score_digits(values, direction='max'): page(lang,f'cases/{case}.html',case,body) from harness.report_languages import write_language_contract write_language_contract(out, include_english=include_english) -def provider_label(name):return PROVIDER_LABELS.get(name,browser_method_label(name)) +def provider_label(name): + from .methods import measured_method_label + return measured_method_label(name,PROVIDER_LABELS.get(name,browser_method_label(name))) def case_track_label(case): if case.get('track','web')=='web': return {'file':'文件交付','resource':'资源发现','structure':'结构化内容'}.get(case.get('task_kind','body'),TRACKS['web']) @@ -1831,6 +1833,9 @@ def local_audit_banner(dataset): return '
本地实测版

全部方法使用同一组已提交案例和判分规则，每例一次调用，统一45秒外部时限，不设逐例等待或提取参数，不重试。结果为机器判定，未进行独立人工审核；本报告仅供本地查看，不代表正式发布。

' return '
本地审核版

本页合并两个不同日期的 Moli 调用批次，保留每例的真实调用时间。本次仅更换报告展示代码，没有新增 Moli 调用，也不代表正式发布。

' +from .methods import with_measured_method_labels + +@with_measured_method_labels def generate(out:Path,dataset:dict,records:list,manifests:list,*,include_english:bool=False,authored_interpretation:bool=True,presentation_only:bool=False): global _RELIABILITY_BANNER,_ENGLISH_GENERATED _ENGLISH_GENERATED=include_english diff --git a/harness/web_fetch/report_capabilities.py b/harness/web_fetch/report_capabilities.py index 1e83b363c..a72fba74f 100644 --- a/harness/web_fetch/report_capabilities.py +++ b/harness/web_fetch/report_capabilities.py @@ -261,6 +261,8 @@ def task_capabilities(out, names, records): def render_capabilities(out, names, *, table, cell, link, records=()): labels={'moli-1.0.6':'Moli 1.0.6','moli':'Moli','moli-1.1.3':'Moli 1.1.3','moli-1.1.7':'Moli 1.1.7','moli-1.1.11':'Moli 1.1.11'} + from .methods import measured_method_label + labels.update({name:measured_method_label(name,labels.get(name,name)) for name in names}) configured=task_capabilities(out,names,records) capabilities=configured or CAPABILITIES unknown = sorted(set(names) - set(capabilities)) diff --git a/harness/web_fetch/report_check.py b/harness/web_fetch/report_check.py index 061d81e21..4122430d3 100644 --- a/harness/web_fetch/report_check.py +++ b/harness/web_fetch/report_check.py @@ -592,6 +592,24 @@ def check_revision(out: Path, *, authority_root: Path): return ( "PASS: WebMainBench preselected-200 release scope and report acceptance checks verified" ) + provenance_path = out / "implementation-provenance.json" + if provenance_path.is_file(): + from .methods import native_browser_engine + provenance = json.loads(provenance_path.read_text()) + records = load_jsonl(out / "results.jsonl") + home = BeautifulSoup((out / "index.html").read_text(), "html.parser") + for item in provenance.get("implementations", []): + name = item.get("provider") + if native_browser_engine(name) != "moli" or not item.get("version"): + continue + hashes = {r.get("provider_metadata", {}).get("binary_sha256") + for r in records if r["provider"] == name} + if hashes != {item.get("binary_sha256")}: + continue + expected = "Moli " + item["version"] + links = home.select('a[href="providers/' + name + '.html"]') + if not links or any(anchor.get_text() != expected for anchor in links): + raise ValueError("Measured method version label: " + name) check_language_contract(out) has_english = "en" in generated_languages(out) for path in out.rglob("*.html"): diff --git a/harness/web_fetch/report_explainer.py b/harness/web_fetch/report_explainer.py index 113e9f2a9..dd16a044c 100644 --- a/harness/web_fetch/report_explainer.py +++ b/harness/web_fetch/report_explainer.py @@ -37,6 +37,11 @@ 'moli-1.0.6': 'Moli 1.0.6', 'moli': 'Moli 1.1.5', 'moli-pr919': 'Moli · PR #919', 'moli-1.1.3': 'Moli 1.1.3', 'moli-1.1.11': 'Moli 1.1.11'} +def provider_name(name, fallback): + from .methods import measured_method_label + return measured_method_label(name, PROVIDER_NAMES.get(name, fallback)) + + METHODOLOGY_FIGURES = ( ('annotation', '混合标注：稳定文字、易变字段与全文参考', '页面中稳定标题、栏目与说明作为内容核对点；变化条目检查数量、字段类型和结构；全文参考单独声明范围。', @@ -302,7 +307,7 @@ def num(value): if speed.get('schema_version', 1) == 1: table = _table(['方法', '共同成功样本', 'P50 ms', '样本 P95 ms'], [ - (escape(PROVIDER_NAMES.get(row['provider'], row['provider'])), + (escape(provider_name(row['provider'], row['provider'])), str(row['attempts']), num(row['p50_ms']), num(row['p95_ms'])) for row in rows], f'同窗速度子集：预先声明 {len(planned_cases)} 题，{provider_count} 种方法共同通过 ' @@ -319,7 +324,7 @@ def num(value): table = _table( ['方法', '计划调用', '任务通过', '通过率', '全调用 P50 ms', '全调用 P95 ms', '通过调用 P50 ms', '通过调用 P95 ms'], [ - (escape(PROVIDER_NAMES.get(row['provider'], row['provider'])), + (escape(provider_name(row['provider'], row['provider'])), str(row['planned_attempts']), str(row['successful_attempts']), _percent(row['task_pass_rate']), num(row['all_calls_p50_ms']), num(row['all_calls_p95_ms']), num(row['successful_calls_p50_ms']), @@ -434,7 +439,7 @@ def _dynamic_section(data, cases): if sample: stats = scope['providers'][sample['provider']] metrics = sample['metrics'] - mechanism += (f'

{escape(PROVIDER_NAMES.get(sample["provider"], sample["provider"]))} 在动态网站正文中通过 ' + mechanism += (f'

{escape(provider_name(sample["provider"], sample["provider"]))} 在动态网站正文中通过 ' f'{stats["task_pass"]}/{stats["records"]}，平均 F1 却为 {_percent(stats["mean_f1"])}。' f'其中{_attempt(sample, cases[sample["case_id"]]["name"])}的 F1 为 {_percent(metrics["word_f1"])}，' '但没有满足该例任务规则。' @@ -600,7 +605,7 @@ def _render_article_speed(population): best, top = min(value for _, value in values), max(value for _, value in values) html += f'

{title}

{explanation}

' for provider, value in values: - label = escape(PROVIDER_NAMES.get(provider, provider.title())) + label = escape(provider_name(provider, provider.title())) html += (f'
{label}{value/1000:.2f} s' '
' f'' @@ -647,7 +652,7 @@ def passed(r): pass_ids = {n: {cid for cid, r in rows[n].items() if passed(r)} for n in selected} totals = {n: len(pass_ids[n]) for n in selected} def label(n): - return escape(PROVIDER_NAMES.get(n, n.title())) + return escape(provider_name(n, n.title())) def bars(values, unit='', maximum=None): top = maximum or max(v for _, v in values) or 1 return '
' + ''.join( diff --git a/harness/web_fetch/report_interfaces.py b/harness/web_fetch/report_interfaces.py index 1fe1320ba..631caf45b 100644 --- a/harness/web_fetch/report_interfaces.py +++ b/harness/web_fetch/report_interfaces.py @@ -3,7 +3,7 @@ import re from html import escape from .safety import contained_path -from .methods import native_browser_engine, browser_method_registry +from .methods import native_browser_engine, browser_method_registry, browser_method_label URL='https://example.com/' HEADLESS_ACCESS={ @@ -282,7 +282,7 @@ def fields(rows):return '
'+''.join('
# Explicit whitelist: no local paths, request IDs, credentials or session metadata. meta=r.get('provider_metadata',{}) if r else {};sample['metadata']={k:meta[k] for k in ['engine','upstream_status','links_count','images_count','extractor','truncated','document_count'] if k in meta} dest=out/'adoption'/'interfaces';dest.mkdir(exist_ok=True);(dest/(name+'.json')).write_text(json.dumps({'interface':s,'access':access,'sample':sample,'attempt_id':r['attempt_id'] if r else None},ensure_ascii=False,indent=2)+'\n') - body+=f'
'+escape(name)+'' + body+=f'
'+escape(browser_method_label(name))+'' body+='

'+escape(s['entry'])+'

'+escape(s['auth'])+'

' if access: body+='

账号与获取入口

'+escape(access['account'])+'

'+escape(access['configuration'])+'

'+ ' · '.join(''+escape(label)+'' for label,url in access['links'])+'

实际版本和接口见运行记录；下方链接指向官方说明。

' diff --git a/harness/web_fetch/report_methods.py b/harness/web_fetch/report_methods.py index 30aa26ddd..d14333fc1 100644 --- a/harness/web_fetch/report_methods.py +++ b/harness/web_fetch/report_methods.py @@ -1,7 +1,7 @@ """Provider-centred introduction and reproducible, sample-scoped adoption counts.""" import json from pathlib import Path -from .methods import method_info, method_page_model, is_browser_comparison, browser_method_label +from .methods import method_info, method_page_model, is_browser_comparison, browser_method_label, measured_method_label from .report_components import tabs,cards from .report_interfaces import render_interfaces from .report_capabilities import render_capabilities @@ -31,7 +31,7 @@ 'crawl4ai':('本地开源库','本地浏览器渲染，再抽取 Markdown、媒体及链接信息。','https://docs.crawl4ai.com/'), } DISPLAY_NAMES={'moli-1.0.6':'Moli 1.0.6','moli':'Moli','moli-1.1.3':'Moli 1.1.3','moli-1.1.7':'Moli 1.1.7','moli-1.1.11':'Moli 1.1.11'} -def display_name(name):return DISPLAY_NAMES.get(name,browser_method_label(name)) +def display_name(name):return measured_method_label(name,DISPLAY_NAMES.get(name,browser_method_label(name))) def adoption_counts(data,names): result=[] diff --git a/research/report-changes.json b/research/report-changes.json index e564f36ec..3e07866d2 100644 --- a/research/report-changes.json +++ b/research/report-changes.json @@ -1275,6 +1275,41 @@ } ], "releases": [ + { + "id": "2026-10-12-moli-measured-method-versions", + "publication_state": "published", + "date": "2026-10-12", + "comparison_kinds": [ + "headless_markdown" + ], + "scoring_revision_input_sha256": [ + "0bbbe0264c9715da5438d05cbd5d140fe0e1a5abc4c80625e078e7b4548c3774" + ], + "title": "区分两个 Moli 方法并标明受测版本", + "summary": "总榜、专项、筛选、详情和接口说明分别标注 Moli 1.1.15 与 Moli 1.1.15 · PR #919 源码构建（98be1f10）。版本与受测二进制的校验值对应，2,580 条成绩及耗时不变。", + "changes": [ + [ + "方法名称", + "此前仅显示 Moli 与 Moli · PR #919", + "按实际受测二进制标注版本和源码构建提交", + "未新增调用；旧成绩不会改挂到后来 rebase 的新提交上。" + ] + ], + "i18n": { + "en": { + "title": "Identify both measured Moli methods by version", + "summary": "Overall and family tables, filters, result details and interface descriptions distinguish Moli 1.1.15 from the Moli 1.1.15 PR #919 source build at 98be1f10. Labels match the measured binary hashes. All 2,580 scores and timings are unchanged.", + "changes": [ + [ + "Method labels", + "Previously shown as Moli and Moli · PR #919", + "Use the measured binary version and source-build commit", + "No new calls. Earlier results are not attributed to a later rebased commit." + ] + ] + } + } + }, { "id": "2026-10-10-moli-pr919-native-430-and-complete-resource-scope", "publication_state": "published", diff --git a/tests/test_report_check_owner.py b/tests/test_report_check_owner.py index 3f1716ad0..eeb85c6e6 100644 --- a/tests/test_report_check_owner.py +++ b/tests/test_report_check_owner.py @@ -345,3 +345,25 @@ def test_body_family_rankings_reject_resource_columns(tmp_path, family, defect): check_revision(root,authority_root=authority) else: assert check_revision(root,authority_root=authority).startswith('PASS:') + +@pytest.mark.parametrize('label', ['Moli 1.1.15', 'Moli', 'Moli 1.1.16']) +def test_release_rejects_missing_or_relabelled_measured_version(tmp_path,label): + from harness.web_fetch.report_check import check_revision + from tests.report_check_fixture import create_revision_report + root,authority=create_revision_report(tmp_path) + row=json.loads((root/'results.jsonl').read_text()) + row['provider']='moli' + row['provider_metadata']={'binary_sha256':'measured-binary'} + (root/'results.jsonl').write_text(json.dumps(row)+'\n') + counts=json.loads((root/'adoption/counts.json').read_text()) + counts['methods']=[item for item in counts['methods'] if item['provider']!='alpha'] + write_json(root/'adoption/counts.json',counts) + write_json(root/'implementation-provenance.json',{'implementations':[ + {'provider':'moli','binary_sha256':'measured-binary','version':'1.1.15'}]}) + p=root/'index.html' + p.write_text(p.read_text()+''+label+'') + if label=='Moli 1.1.15': + assert check_revision(root,authority_root=authority).startswith('PASS:') + else: + with pytest.raises(ValueError,match='Measured method version label'): + check_revision(root,authority_root=authority) diff --git a/tests/test_report_methods.py b/tests/test_report_methods.py index 8f88e37a1..f4c1a9f55 100644 --- a/tests/test_report_methods.py +++ b/tests/test_report_methods.py @@ -111,3 +111,35 @@ def test_metric_definitions_cover_each_ranking(self): for title in ['请求成功率','任务通过率','结构达标率','资源命中率','文件取得率','信息通过率','完整度','纯净度','F1','均值','P50','P95']: self.assertIn('
'+title+'
',metric_legend([title])) self.assertNotIn('条件通过率',metric_guide()) + +def test_report_labels_distinguish_release_and_source_build_with_same_version(tmp_path): + import json + from harness.web_fetch.methods import with_measured_method_labels + from harness.web_fetch.report import provider_label + from harness.web_fetch.report_methods import display_name + from harness.web_fetch.report_explainer import provider_name + + identities={'implementations':[ + {'provider':'moli','binary_sha256':'stable','version':'1.1.15'}, + {'provider':'moli-pr919','binary_sha256':'branch','version':'1.1.15 · PR #919 源码构建（98be1f10）'}]} + (tmp_path/'implementation-provenance.json').write_text(json.dumps(identities)) + records=[{'provider':x['provider'],'provider_metadata':{'binary_sha256':x['binary_sha256']}} + for x in identities['implementations']] + original=provider_label('moli') + + @with_measured_method_labels + def render(out,dataset,rows): + for label in (provider_label,display_name,lambda n:provider_name(n,n)): + assert label('moli')=='Moli 1.1.15' + assert label('moli-pr919')=='Moli 1.1.15 · PR #919 源码构建（98be1f10）' + + render(tmp_path,{},records) + assert provider_label('moli')==original + records[0]['provider_metadata']['binary_sha256']='different-build' + + @with_measured_method_labels + def unrelated(out,dataset,rows): + assert provider_label('moli')==original + + unrelated(tmp_path,{},records) + assert provider_label('moli')==original From 7286b01ee41a9f66b7aa4edaa0aff891a6859dd1 Mon Sep 17 00:00:00 2001 From: lanyue-llk <270302213+lanyue-llk@users.noreply.github.com> Date: Mon, 12 Oct 2026 11:12:47 +0800 Subject: [PATCH 012/172] fix(report): keep presentation release separate from scoring revision --- research/report-changes.json | 3 --- 1 file changed, 3 deletions(-) diff --git a/research/report-changes.json b/research/report-changes.json index 3e07866d2..974d92acd 100644 --- a/research/report-changes.json +++ b/research/report-changes.json @@ -1282,9 +1282,6 @@ "comparison_kinds": [ "headless_markdown" ], - "scoring_revision_input_sha256": [ - "0bbbe0264c9715da5438d05cbd5d140fe0e1a5abc4c80625e078e7b4548c3774" - ], "title": "区分两个 Moli 方法并标明受测版本", "summary": "总榜、专项、筛选、详情和接口说明分别标注 Moli 1.1.15 与 Moli 1.1.15 · PR #919 源码构建（98be1f10）。版本与受测二进制的校验值对应，2,580 条成绩及耗时不变。", "changes": [ From 49e360a9a46ededcca34afc1790d973765dd922e Mon Sep 17 00:00:00 2001 From: lanyue-llk <270302213+lanyue-llk@users.noreply.github.com> Date: Mon, 12 Oct 2026 11:21:49 +0800 Subject: [PATCH 013/172] fix(report): localize reliability state labels after rebase --- harness/web_fetch/report_i18n.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/harness/web_fetch/report_i18n.py b/harness/web_fetch/report_i18n.py index 38bb17c8c..81d4f7ede 100644 --- a/harness/web_fetch/report_i18n.py +++ b/harness/web_fetch/report_i18n.py @@ -110,6 +110,9 @@ '方法名称': 'Method name', '工具版本': 'Tool version', 'Web Fetch · 评测报告': 'Web Fetch · Evaluation report', + 'Web Fetch · 合成示例': 'Web Fetch · Synthetic example', + 'Web Fetch · 诊断运行': 'Web Fetch · Diagnostic run', + 'Web Fetch · 实测报告': 'Web Fetch · Measured report', '合成示例 · 非正式评测': 'Synthetic example · not a formal benchmark', '诊断运行 · 非正式评测': 'Diagnostic run · not a formal benchmark', '文件交付': 'File delivery', From b0234b9deb54d77bffbb7879506d3983249893ee Mon Sep 17 00:00:00 2001 From: lanyue-llk <270302213+lanyue-llk@users.noreply.github.com> Date: Mon, 12 Oct 2026 11:23:31 +0800 Subject: [PATCH 014/172] fix(report): translate raw output label in result details --- harness/web_fetch/report_i18n.py | 1 + 1 file changed, 1 insertion(+) diff --git a/harness/web_fetch/report_i18n.py b/harness/web_fetch/report_i18n.py index 81d4f7ede..dc131b8ab 100644 --- a/harness/web_fetch/report_i18n.py +++ b/harness/web_fetch/report_i18n.py @@ -109,6 +109,7 @@ '本报告的方法由调用方的显式配置提供。': 'The caller’s explicit configuration supplies the methods in this report.', '方法名称': 'Method name', '工具版本': 'Tool version', + '原始输出': 'Raw output', 'Web Fetch · 评测报告': 'Web Fetch · Evaluation report', 'Web Fetch · 合成示例': 'Web Fetch · Synthetic example', 'Web Fetch · 诊断运行': 'Web Fetch · Diagnostic run', From 172f4b58db967c23a819f59d79747dd9ce563937 Mon Sep 17 00:00:00 2001 From: lanyue-llk <270302213+lanyue-llk@users.noreply.github.com> Date: Mon, 12 Oct 2026 11:25:40 +0800 Subject: [PATCH 015/172] fix(report): localize failure labels and report all language gaps --- harness/web_fetch/report_i18n.py | 2 ++ tests/test_public_report_language.py | 8 ++++++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/harness/web_fetch/report_i18n.py b/harness/web_fetch/report_i18n.py index dc131b8ab..914017595 100644 --- a/harness/web_fetch/report_i18n.py +++ b/harness/web_fetch/report_i18n.py @@ -110,6 +110,8 @@ '方法名称': 'Method name', '工具版本': 'Tool version', '原始输出': 'Raw output', + '程序错误': 'Program error', + '分类：': 'Classification: ', 'Web Fetch · 评测报告': 'Web Fetch · Evaluation report', 'Web Fetch · 合成示例': 'Web Fetch · Synthetic example', 'Web Fetch · 诊断运行': 'Web Fetch · Diagnostic run', diff --git a/tests/test_public_report_language.py b/tests/test_public_report_language.py index 45b5a286f..269a5fd0e 100644 --- a/tests/test_public_report_language.py +++ b/tests/test_public_report_language.py @@ -164,8 +164,12 @@ def test_default_english_has_localized_ui_and_preserved_task_text(tmp_path): source_text = {"中文", "确定性安装测试", "固定"} source_text.update(case["name"] for case in dataset["cases"]) source_text.update(case["task_prompt"] for case in dataset["cases"]) - for page in (run / "report/en").rglob("*.html"): - assert not untranslated_english_narrative(page.read_text(), source_text), page + untranslated = { + str(page.relative_to(run / "report/en")): fragments + for page in (run / "report/en").rglob("*.html") + if (fragments := untranslated_english_narrative(page.read_text(), source_text)) + } + assert not untranslated, untranslated for case in dataset["cases"]: assert case["task_prompt"] in (run / f"report/en/datasets/{case['id']}.html").read_text() From 041db954e6311c556bc80aad9b7c4217237c7698 Mon Sep 17 00:00:00 2001 From: lanyue-llk <270302213+lanyue-llk@users.noreply.github.com> Date: Mon, 12 Oct 2026 11:27:25 +0800 Subject: [PATCH 016/172] fix(report): retain one measured-report translation --- harness/web_fetch/report_i18n.py | 1 - 1 file changed, 1 deletion(-) diff --git a/harness/web_fetch/report_i18n.py b/harness/web_fetch/report_i18n.py index 914017595..b8b54cbfc 100644 --- a/harness/web_fetch/report_i18n.py +++ b/harness/web_fetch/report_i18n.py @@ -115,7 +115,6 @@ 'Web Fetch · 评测报告': 'Web Fetch · Evaluation report', 'Web Fetch · 合成示例': 'Web Fetch · Synthetic example', 'Web Fetch · 诊断运行': 'Web Fetch · Diagnostic run', - 'Web Fetch · 实测报告': 'Web Fetch · Measured report', '合成示例 · 非正式评测': 'Synthetic example · not a formal benchmark', '诊断运行 · 非正式评测': 'Diagnostic run · not a formal benchmark', '文件交付': 'File delivery', From bb3885fa260fb68aeaa2b33e6ed2e3741350f894 Mon Sep 17 00:00:00 2001 From: lanyue-llk <270302213+lanyue-llk@users.noreply.github.com> Date: Mon, 12 Oct 2026 11:46:47 +0800 Subject: [PATCH 017/172] chore(report): bind verified Moli method-version publication --- evaluation/browser-native-430/current-report.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/evaluation/browser-native-430/current-report.json b/evaluation/browser-native-430/current-report.json index c0f2c6925..62ac04848 100644 --- a/evaluation/browser-native-430/current-report.json +++ b/evaluation/browser-native-430/current-report.json @@ -511,6 +511,6 @@ "native_task_route_contract": 4, "artifact_hub_base_url": "https://artifact-site.lexmount.cn", "artifact_hub_site": "jGXOOJwkdBK_", - "artifact_hub_expected_version": "ver_94084cc8ebaa48268cb29efaddff0902", + "artifact_hub_expected_version": "ver_a10d677a5efb4553a96ef201dae9faec", "artifact_hub_share_url": "https://artifact-site.lexmount.cn/v/IqGpMUYHWat-dw4n8usQttTptoPzboAo" } From 5fd5e811480bf79327acc6199dc9c3337526f4c4 Mon Sep 17 00:00:00 2001 From: lanyue-llk <270302213+lanyue-llk@users.noreply.github.com> Date: Mon, 12 Oct 2026 12:02:59 +0800 Subject: [PATCH 018/172] Fix scoped Markdown list and product acceptance --- evaluation/scoring-regressions.json | 407 +++++++++++++++++++++++++++- harness/web_fetch/dynamic_lists.py | 20 +- tests/test_scoring_regressions.py | 2 + 3 files changed, 425 insertions(+), 4 deletions(-) diff --git a/evaluation/scoring-regressions.json b/evaluation/scoring-regressions.json index d7c001a1d..2f2b142f1 100644 --- a/evaluation/scoring-regressions.json +++ b/evaluation/scoring-regressions.json @@ -21926,7 +21926,412 @@ {"id":"wrong-body-content-policy-dynamic-final-page-mismatch-cannot-be-counted-as-a-pass","category":"acceptance-contract","root_cause":"dynamic_page_identity_alignment","failure_mode":"false-positive","case":{"url":"https://example.test/start","evaluation_url":"https://example.test/start","expected_final_url":"https://example.test/en/catalog","target_language":"en-US","target_region":"US","track":"dynamic","task_kind":"body","acceptance_contract":{"version":1,"schema":"body_text","minimum_records":0,"stable_text_anchors":[{"id":"identity","alternatives":["Catalog"]}]}},"artifact":"Unrelated page text","ok":true,"final_url":"https://example.test/fr/catalog","metadata":{"route":"dynamic_selected_text","document_language":"fr"},"expected_task_pass":false,"expected_metrics":{"page_identity_status":"mismatch","final_url_consistent":false,"language_consistent":false,"page_identity_required":false}}, {"id":"wrong-body-content-policy-dynamic-search-query-mismatch-is-a-different-page","category":"acceptance-contract","root_cause":"dynamic_page_query_identity","failure_mode":"false-positive","case":{"url":"https://example.test/search?q=Mario","evaluation_url":"https://example.test/search?q=Mario","expected_final_url":"https://example.test/search?q=Mario","target_language":"ja-JP","target_region":"JP","track":"dynamic","task_kind":"body","acceptance_contract":{"version":1,"schema":"body_text","minimum_records":0,"stable_text_anchors":[{"id":"identity","alternatives":["Search results"]}]}},"artifact":"Unrelated page text","ok":true,"final_url":"https://example.test/search?q=Zelda","metadata":{"route":"dynamic_selected_text","document_language":"ja"},"expected_task_pass":false,"expected_metrics":{"page_identity_status":"mismatch","final_url_consistent":false,"page_identity_required":false}}, {"id":"wrong-body-content-policy-dynamic-source-document-language-mismatch-is-not-verified","category":"acceptance-contract","root_cause":"declared_document_language_vs_visible_language","failure_mode":"false-positive","case":{"url":"https://example.test/rtl","evaluation_url":"https://example.test/rtl","expected_final_url":"https://example.test/rtl","target_language":"ar","expected_document_language":"en","target_region":"US","track":"dynamic","task_kind":"body","acceptance_contract":{"version":1,"schema":"body_text","minimum_records":0,"stable_text_anchors":[{"id":"arabic-content","alternatives":["الشخص"]}]}},"artifact":"Unrelated page text","ok":true,"final_url":"https://example.test/rtl","metadata":{"route":"dynamic_selected_text","document_language":"fr"},"expected_task_pass":false,"expected_metrics":{"page_identity_status":"mismatch","language_consistent":false,"page_identity_required":false}}, - {"id":"wrong-body-content-policy-dynamic-undeclared-source-language-rejects-declared-language","category":"acceptance-contract","root_cause":"source_document_language_absent","failure_mode":"false-positive","case":{"url":"https://example.test/untagged","evaluation_url":"https://example.test/untagged","expected_final_url":"https://example.test/untagged","target_language":"en","target_region":"US","expected_document_language_present":false,"track":"dynamic","task_kind":"body","acceptance_contract":{"version":1,"schema":"body_text","minimum_records":0,"stable_text_anchors":[{"id":"content","alternatives":["Visible content"]}]}},"artifact":"Unrelated page text","ok":true,"final_url":"https://example.test/untagged","metadata":{"route":"dynamic_selected_text","document_language":"en"},"expected_task_pass":false,"expected_metrics":{"page_identity_status":"mismatch","language_consistent":false,"page_identity_required":false}} + {"id":"wrong-body-content-policy-dynamic-undeclared-source-language-rejects-declared-language","category":"acceptance-contract","root_cause":"source_document_language_absent","failure_mode":"false-positive","case":{"url":"https://example.test/untagged","evaluation_url":"https://example.test/untagged","expected_final_url":"https://example.test/untagged","target_language":"en","target_region":"US","expected_document_language_present":false,"track":"dynamic","task_kind":"body","acceptance_contract":{"version":1,"schema":"body_text","minimum_records":0,"stable_text_anchors":[{"id":"content","alternatives":["Visible content"]}]}},"artifact":"Unrelated page text","ok":true,"final_url":"https://example.test/untagged","metadata":{"route":"dynamic_selected_text","document_language":"en"},"expected_task_pass":false,"expected_metrics":{"page_identity_status":"mismatch","language_consistent":false,"page_identity_required":false}}, + { + "id": "scoped-markdown-empty-bookmark-keeps-owned-records", + "category": "format-normalization", + "root_cause": "scoped_markdown_bookmark_format_detection", + "failure_mode": "false-negative", + "case": { + "url": "https://example.test/catalog", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "example.test" + ], + "entry_path": "/item/\\d+", + "entry_scope": { + "html_selector": ".catalog a", + "start_pattern": "(?m)^Recommended[ \\t]*$", + "end_pattern": "(?m)^Other[ \\t]*$" + } + } + }, + "artifact": "\nRecommended\n[First item](/item/1)\n[Second item](/item/2)\nOther", + "ok": true, + "expected_task_pass": true, + "expected_metrics": { + "stable_structure_pass": true + } + }, + { + "id": "scoped-markdown-bookmark-cannot-import-outside-links", + "category": "format-normalization", + "root_cause": "scoped_markdown_bookmark_format_detection", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/catalog", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "example.test" + ], + "entry_path": "/item/\\d+", + "entry_scope": { + "html_selector": ".catalog a", + "start_pattern": "(?m)^Recommended[ \\t]*$", + "end_pattern": "(?m)^Other[ \\t]*$" + } + } + }, + "artifact": "\n[First item](/item/1)\n[Second item](/item/2)\nRecommended\n[One item](/item/3)\nOther", + "ok": true, + "expected_task_pass": false, + "expected_metrics": { + "stable_structure_pass": false + } + }, + { + "id": "scoped-html-missing-owner-cannot-fall-back-to-text", + "category": "format-normalization", + "root_cause": "scoped_markdown_bookmark_format_detection", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/catalog", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "example.test" + ], + "entry_path": "/item/\\d+", + "entry_scope": { + "html_selector": ".catalog a", + "start_pattern": "(?m)^Recommended[ \\t]*$", + "end_pattern": "(?m)^Other[ \\t]*$" + } + } + }, + "artifact": "
Recommended\n[First item](/item/1)\n[Second item](/item/2)\nOther
", + "ok": true, + "expected_task_pass": false, + "expected_metrics": { + "stable_structure_pass": false + } + }, + { + "id": "scoped-html-owns-records-despite-bookmark", + "category": "format-normalization", + "root_cause": "scoped_markdown_bookmark_format_detection", + "failure_mode": "false-negative", + "case": { + "url": "https://example.test/catalog", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "example.test" + ], + "entry_path": "/item/\\d+", + "entry_scope": { + "html_selector": ".catalog a", + "start_pattern": "(?m)^Recommended[ \\t]*$", + "end_pattern": "(?m)^Other[ \\t]*$" + } + } + }, + "artifact": "
First itemSecond item
", + "ok": true, + "expected_task_pass": true, + "expected_metrics": { + "stable_structure_pass": true + } + }, + { + "id": "scoped-markdown-bullet-boundary-keeps-records", + "category": "format-normalization", + "root_cause": "scoped_markdown_list_boundary_semantics", + "failure_mode": "false-negative", + "case": { + "url": "https://example.test/catalog", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "example.test" + ], + "entry_path": "/item/\\d+", + "entry_scope": { + "html_selector": ".catalog a", + "start_pattern": "(?m)^Recommended[ \\t]*$", + "end_pattern": "(?m)^Other[ \\t]*$" + } + } + }, + "artifact": "- Recommended\n[First item](/item/1)\n[Second item](/item/2)\n- Other\n[Outside item](/item/3)", + "ok": true, + "expected_task_pass": true, + "expected_metrics": { + "stable_structure_pass": true + } + }, + { + "id": "scoped-markdown-numbered-boundary-keeps-records", + "category": "format-normalization", + "root_cause": "scoped_markdown_list_boundary_semantics", + "failure_mode": "false-negative", + "case": { + "url": "https://example.test/catalog", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "example.test" + ], + "entry_path": "/item/\\d+", + "entry_scope": { + "html_selector": ".catalog a", + "start_pattern": "(?m)^Recommended[ \\t]*$", + "end_pattern": "(?m)^Other[ \\t]*$" + } + } + }, + "artifact": "1. Recommended\n[First item](/item/1)\n[Second item](/item/2)\n- Other\n[Outside item](/item/3)", + "ok": true, + "expected_task_pass": true, + "expected_metrics": { + "stable_structure_pass": true + } + }, + { + "id": "scoped-markdown-linked-bullet-boundary-keeps-records", + "category": "format-normalization", + "root_cause": "scoped_markdown_list_boundary_semantics", + "failure_mode": "false-negative", + "case": { + "url": "https://example.test/catalog", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "example.test" + ], + "entry_path": "/item/\\d+", + "entry_scope": { + "html_selector": ".catalog a", + "start_pattern": "(?m)^Recommended[ \\t]*$", + "end_pattern": "(?m)^Other[ \\t]*$" + } + } + }, + "artifact": "- [Recommended](/category)\n[First item](/item/1)\n[Second item](/item/2)\n- Other\n[Outside item](/item/3)", + "ok": true, + "expected_task_pass": true, + "expected_metrics": { + "stable_structure_pass": true + } + }, + { + "id": "scoped-markdown-bullet-end-excludes-outside-records", + "category": "format-normalization", + "root_cause": "scoped_markdown_list_boundary_semantics", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/catalog", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "example.test" + ], + "entry_path": "/item/\\d+", + "entry_scope": { + "html_selector": ".catalog a", + "start_pattern": "(?m)^Recommended[ \\t]*$", + "end_pattern": "(?m)^Other[ \\t]*$" + } + } + }, + "artifact": "- Recommended\n[One item](/item/1)\n- Other\n[Outside item](/item/2)", + "ok": true, + "expected_task_pass": false, + "expected_metrics": { + "stable_structure_pass": false + } + }, + { + "id": "scoped-markdown-prose-is-not-a-list-boundary", + "category": "format-normalization", + "root_cause": "scoped_markdown_list_boundary_semantics", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/catalog", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "example.test" + ], + "entry_path": "/item/\\d+", + "entry_scope": { + "html_selector": ".catalog a", + "start_pattern": "(?m)^Recommended[ \\t]*$", + "end_pattern": "(?m)^Other[ \\t]*$" + } + } + }, + "artifact": "We discuss Recommended here.\n[First item](/item/1)\n[Second item](/item/2)\nOther", + "ok": true, + "expected_task_pass": false, + "expected_metrics": { + "stable_structure_pass": false + } + }, + { + "id": "scoped-markdown-bullet-prefix-does-not-match-prose", + "category": "format-normalization", + "root_cause": "scoped_markdown_list_boundary_semantics", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/catalog", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "example.test" + ], + "entry_path": "/item/\\d+", + "entry_scope": { + "html_selector": ".catalog a", + "start_pattern": "(?m)^Recommended[ \\t]*$", + "end_pattern": "(?m)^Other[ \\t]*$" + } + } + }, + "artifact": "- Recommended reading elsewhere\n[First item](/item/1)\n[Second item](/item/2)\nOther", + "ok": true, + "expected_task_pass": false, + "expected_metrics": { + "stable_structure_pass": false + } + }, + { + "id": "scoped-markdown-bookmark-keeps-owned-product-prices", + "category": "format-normalization", + "root_cause": "scoped_markdown_bookmark_format_detection", + "failure_mode": "false-negative", + "case": { + "url": "https://example.test/catalog", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "linked_product", + "minimum_records": 2, + "hosts": [ + "example.test" + ], + "entry_path": "/item/\\d+", + "entry_scope": { + "html_selector": ".catalog a", + "start_pattern": "(?m)^Recommended[ \\t]*$", + "end_pattern": "(?m)^Other[ \\t]*$", + "price_selector": ".price" + }, + "price_patterns": [ + "\\$\\d+\\.\\d{2}" + ], + "price_exclusion_patterns": [] + } + }, + "artifact": "\n- Recommended\n[First item](/item/1)\n$10.00\n[Second item](/item/2)\n$20.00\n- Other", + "ok": true, + "expected_task_pass": true, + "expected_metrics": { + "stable_structure_pass": true + } + }, + { + "id": "scoped-markdown-product-scope-excludes-outside-prices", + "category": "format-normalization", + "root_cause": "scoped_markdown_bookmark_format_detection", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/catalog", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "linked_product", + "minimum_records": 2, + "hosts": [ + "example.test" + ], + "entry_path": "/item/\\d+", + "entry_scope": { + "html_selector": ".catalog a", + "start_pattern": "(?m)^Recommended[ \\t]*$", + "end_pattern": "(?m)^Other[ \\t]*$", + "price_selector": ".price" + }, + "price_patterns": [ + "\\$\\d+\\.\\d{2}" + ], + "price_exclusion_patterns": [] + } + }, + "artifact": "\nRecommended\n[First item](/item/1)\n$10.00\nOther\n[Second item](/item/2)\n$20.00", + "ok": true, + "expected_task_pass": false, + "expected_metrics": { + "stable_structure_pass": false + } + }, + { + "id": "scoped-markdown-product-missing-section-cannot-pass", + "category": "format-normalization", + "root_cause": "scoped_markdown_bookmark_format_detection", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/catalog", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "linked_product", + "minimum_records": 2, + "hosts": [ + "example.test" + ], + "entry_path": "/item/\\d+", + "entry_scope": { + "html_selector": ".catalog a", + "start_pattern": "(?m)^Recommended[ \\t]*$", + "end_pattern": "(?m)^Other[ \\t]*$", + "price_selector": ".price" + }, + "price_patterns": [ + "\\$\\d+\\.\\d{2}" + ], + "price_exclusion_patterns": [] + } + }, + "artifact": "[First item](/item/1)\n$10.00\n[Second item](/item/2)\n$20.00", + "ok": true, + "expected_task_pass": false, + "expected_metrics": { + "stable_structure_pass": false + } + } ], "superseded_cases": { "dynamic-final-page-mismatch-cannot-be-counted-as-a-pass": "content-policy-dynamic-final-page-mismatch-cannot-be-counted-as-a-pass", diff --git a/harness/web_fetch/dynamic_lists.py b/harness/web_fetch/dynamic_lists.py index e326c6e40..53020ca07 100644 --- a/harness/web_fetch/dynamic_lists.py +++ b/harness/web_fetch/dynamic_lists.py @@ -226,6 +226,7 @@ def _text_scope_boundary(content, pattern): for line in content.splitlines(keepends=True): visible=visible_markdown_links(line).strip() visible=re.sub(r'^#{1,6}\s+','',visible) + visible=re.sub(r'^(?:[-*+]|\d+[.)])\s+','',visible) visible=re.sub(r'([*_]{1,2})(.+?)\1',r'\2',visible) visible=re.sub(r'\s+#+\s*$','',visible).strip() if re.match(pattern,visible,re.M): @@ -234,10 +235,20 @@ def _text_scope_boundary(content, pattern): return min(candidates) if candidates else None +def _has_scoped_html(content): + # Native Markdown may retain empty bookmark anchors. They provide no HTML + # record ownership and must not select the HTML-only CSS scope parser. + without_bookmarks = re.sub(r']*>\s*', '', content, flags=re.I) + return bool(re.search( + r'<(?:html|body|main|div|section|article|table|ul|ol|a|p|h[1-6])\b', + without_bookmarks, re.I, + )) + + def _scope_named_content(content, scope): """Apply one declared HTML or text scope and return owned name/link pairs.""" owned_items = [] - if re.search(r'<(?:html|body|main|div|section|article|table|ul|ol|a|p|h[1-6])\b', content, re.I): + if _has_scoped_html(content): soup = _scoped_html(content) owners = soup.select(scope['html_selector']) if not scope.get('title_selector'): @@ -466,8 +477,7 @@ def extract_named_entries(content,contract,base_url,links=None,diagnostics=None) def extract_linked_products(content, contract, base_url, diagnostics=None): """Extract title/link/price triples from one owned product segment.""" scope = contract.get('entry_scope') - if scope and scope.get('price_selector') and re.search( - r'<(?:html|body|main|div|section|article|table|ul|ol|a|p|h[1-6])\b', content, re.I): + if scope and scope.get('price_selector') and _has_scoped_html(content): soup = _scoped_html(content) allowed = [re.compile(p, re.I) for p in contract['price_patterns']] excluded = [re.compile(p, re.I) for p in contract.get('price_exclusion_patterns', [])] @@ -494,6 +504,10 @@ def extract_linked_products(content, contract, base_url, diagnostics=None): **entry, 'price': None, 'price_status': 'not_shown_on_page', 'price_unavailable_text': hint} return list(records.values()) + if scope: + content, _, matched = _scope_named_content(content, scope) + if not matched: + return [] scannable = re.sub(r"!\[([^\]]*)\]\([^)]*\)", r"\1", content) link_pattern = re.compile( r"\[([^\]\n]+)\]\((https?://[^\s)]+|/[^\s)]+)\)" diff --git a/tests/test_scoring_regressions.py b/tests/test_scoring_regressions.py index 947346953..a768da14b 100644 --- a/tests/test_scoring_regressions.py +++ b/tests/test_scoring_regressions.py @@ -148,6 +148,8 @@ def test_every_registered_false_verdict_control_runs_through_shared_scorer(self) paired.add("declared_full_page_stable_text_scope") paired.add("inline_emphasis_in_structured_fields") paired.add("markdown_list_prefix_ownership") + paired.add("scoped_markdown_bookmark_format_detection") + paired.add("scoped_markdown_list_boundary_semantics") for root_cause in paired: self.assertEqual(root_modes.get(root_cause), {"false-positive", "false-negative"}) From 7ec325abbd81d511efdaa5fa54e579fb458809c2 Mon Sep 17 00:00:00 2001 From: lanyue-llk <270302213+lanyue-llk@users.noreply.github.com> Date: Mon, 12 Oct 2026 12:21:26 +0800 Subject: [PATCH 019/172] test(report): select Chinese for bound Chinese article fixture --- tests/test_headless_report.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/test_headless_report.py b/tests/test_headless_report.py index 9ab9ddb72..0e017820c 100644 --- a/tests/test_headless_report.py +++ b/tests/test_headless_report.py @@ -813,9 +813,11 @@ def test_public_builder_materializes_bound_prose_and_preserves_scores(self): profile_path.write_text(json.dumps(profile)) cli_report=root/'cli-report' build=subprocess.run([sys.executable,'scripts/build_report.py','--profile',str(profile_path), - '--evidence',str(source),'--output',str(cli_report)], + '--evidence',str(source),'--output',str(cli_report),'--language','zh'], cwd=repository,capture_output=True,text=True) self.assertEqual(build.returncode,0,build.stderr) + self.assertIn('新一轮浏览器结果',(cli_report/'index.html').read_text()) + self.assertIn('data-english-not-generated',(cli_report/'en/index.html').read_text()) gate=subprocess.run([sys.executable,'scripts/check_report.py',str(cli_report), '--profile',str(profile_path)],cwd=repository,capture_output=True,text=True) self.assertEqual(gate.returncode,0,gate.stderr+gate.stdout) From ac5914be73a1c80adc32befea49601fd1d3c491d Mon Sep 17 00:00:00 2001 From: lanyue-llk <270302213+lanyue-llk@users.noreply.github.com> Date: Mon, 12 Oct 2026 12:36:00 +0800 Subject: [PATCH 020/172] fix(reference): exclude fully transparent rendered text --- .github/workflows/ci.yml | 6 +++ harness/web_fetch/browser_observations.py | 46 ++++++++++++++++++--- tests/test_browser_observation_contracts.py | 10 +++++ 3 files changed, 56 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5a6de6306..e9df22ead 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -44,3 +44,9 @@ jobs: cache: npm - run: make setup-test - run: make ci + - name: Exercise rendered reference capture + run: | + uv sync --frozen --extra test --extra headless-eval + uv run --frozen --no-sync playwright install --with-deps chromium + export BROWSER_EVAL_TEST_BROWSER_BINARY="$(uv run --frozen --no-sync python -c 'from playwright.sync_api import sync_playwright; p = sync_playwright().start(); print(p.chromium.executable_path); p.stop()')" + uv run --frozen --no-sync python -m pytest tests/test_browser_observation_contracts.py -q diff --git a/harness/web_fetch/browser_observations.py b/harness/web_fetch/browser_observations.py index 22083a4e3..6fac00fe4 100644 --- a/harness/web_fetch/browser_observations.py +++ b/harness/web_fetch/browser_observations.py @@ -27,6 +27,37 @@ from harness.web_fetch.safety import safe_identifier from harness.web_fetch.source_capture import extract_html +# innerText includes opacity-zero content. Descend only through ancestors of +# transparent subtrees; unaffected branches retain the browser's text layout. +PAINTED_TEXT = r"""(root) => { + for (let n = root; n; n = n.parentElement) { + if (Number(getComputedStyle(n).opacity) === 0) return ''; + } + const affected = new Set(); + for (const node of root.querySelectorAll('*')) { + if (Number(getComputedStyle(node).opacity) !== 0) continue; + for (let n = node.parentElement; n && !affected.has(n); n = n.parentElement) { + affected.add(n); + if (n === root) break; + } + } + const read = node => { + if (node.nodeType === Node.TEXT_NODE) { + return getComputedStyle(node.parentElement).visibility === 'visible' + ? node.textContent : ''; + } + if (node.nodeType !== Node.ELEMENT_NODE) return ''; + const style = getComputedStyle(node); + if (style.display === 'none' || Number(style.opacity) === 0) return ''; + if (!affected.has(node)) return node.innerText || ''; + const block = !['inline', 'inline-block', 'contents'].includes(style.display); + const text = Array.from(node.childNodes, read).join(''); + return block ? '\n' + text + '\n' : text; + }; + return read(root).trim(); +}""" + + CONTRACTS = Path(__file__).parent / "data/browser-observation-contracts.json" @@ -312,11 +343,11 @@ def capture( ) observation_method = "rendered_dom_text_including_preloaded_hidden_panels" elif policy.text_mode == "visible": - content = content_locator.inner_text(timeout=remaining_ms(begin, timeout_ms)) + content = content_locator.evaluate(PAINTED_TEXT, timeout=remaining_ms(begin, timeout_ms)) observation_method = ( - "visible_whole_page_inner_text" + "painted_whole_page_text" if observation_scope == "visible_page" - else "visible_inner_text" + else "painted_visible_text" ) else: content = extract_pruned_selector_text(outer_html, case["url"]) @@ -328,14 +359,17 @@ def capture( try: current, shown = frame, True while current.parent_frame is not None: - if not current.frame_element().is_visible(): + if not current.frame_element().evaluate( + "element => element.checkVisibility({opacityProperty: true, " + "visibilityProperty: true})" + ): shown = False break current = current.parent_frame if not shown: continue - text = frame.locator("body").inner_text( - timeout=remaining_ms(begin, timeout_ms) + text = frame.locator("body").evaluate( + PAINTED_TEXT, timeout=remaining_ms(begin, timeout_ms) ) frames.append( {"url": frame.url, "status": "success", "content_text": text} diff --git a/tests/test_browser_observation_contracts.py b/tests/test_browser_observation_contracts.py index 2af350803..8ccfbe7c9 100644 --- a/tests/test_browser_observation_contracts.py +++ b/tests/test_browser_observation_contracts.py @@ -206,6 +206,10 @@ class Page(BaseHTTPRequestHandler): def do_GET(self): body = b"""
Hidden panel
+
Transparent menu
+ Visible duplicate
Visible duplicate
+

Translucent text

+

Inline formatting preserved

Footer only
", - "ok": true, - "expected_task_pass": true - }, - { - "id": "forbidden-html-surface-visible-challenge", - "category": "answer-surface", - "root_cause": "dormant_html_templates_not_blocking_notices", - "failure_mode": "false-positive", - "case": { - "url": "https://example.test/catalog", - "task_kind": "body", - "expected": [ - "Game catalog" - ], - "forbidden_any": [ - "CAPTCHA", - "Access denied" - ], - "min_content_chars": 1 - }, - "artifact": "

Game catalog

CAPTCHA required

", - "ok": true, - "expected_task_pass": false - }, - { - "id": "forbidden-html-surface-explicitly-hidden", - "category": "answer-surface", - "root_cause": "dormant_html_templates_not_blocking_notices", - "failure_mode": "false-negative", - "case": { - "url": "https://example.test/catalog", - "task_kind": "body", - "expected": [ - "Game catalog" - ], - "forbidden_any": [ - "CAPTCHA", - "Access denied" - ], - "min_content_chars": 1 - }, - "artifact": "
Game catalog
", - "ok": true, - "expected_task_pass": true - }, - { - "id": "forbidden-html-surface-actual-denial", - "category": "answer-surface", - "root_cause": "dormant_html_templates_not_blocking_notices", - "failure_mode": "false-positive", - "case": { - "url": "https://example.test/catalog", - "task_kind": "body", - "expected": [ - "Game catalog" - ], - "forbidden_any": [ - "CAPTCHA", - "Access denied" - ], - "min_content_chars": 1 - }, - "artifact": "
Game catalog
Access denied
", - "ok": true, - "expected_task_pass": false - }, - { - "id": "forbidden-html-surface-plain-challenge", - "category": "answer-surface", - "root_cause": "dormant_html_templates_not_blocking_notices", - "failure_mode": "false-positive", - "case": { - "url": "https://example.test/catalog", - "task_kind": "body", - "expected": [ - "Game catalog" - ], - "forbidden_any": [ - "CAPTCHA", - "Access denied" - ], - "min_content_chars": 1 - }, - "artifact": "Game catalog\nCAPTCHA required", - "ok": true, - "expected_task_pass": false - }, - { - "id": "nintendo-markdown-section-actual-game-section", - "category": "structured-extraction", - "root_cause": "game_card_announcements_not_navigation_news", - "failure_mode": "false-negative", - "case": { - "url": "https://www.nintendo.com/", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 2, - "hosts": [ - "www.nintendo.com", - "store-jp.nintendo.com" - ], - "entry_path": "(?:/item/software/[A-Za-z0-9]+|/jp/games/(?:switch2/[A-Za-z0-9-]+|zelda/(?:botw|totk)/edition|feature/nintendo-classics/[A-Za-z0-9_-]+)/index\\.html|/jp/xenoblade/index\\.html|/jp/topics/article/[0-9a-f-]{36})", - "stable_text_anchors": [], - "name_validation": "nonempty", - "entry_scope": { - "html_selector": ".nc3-c-softCard", - "title_selector": ".nc3-c-softCard__name", - "link_selector": "a[href]:has(.nc3-c-softCard__name)", - "start_pattern": "^[ \\t#*·-]*ピックアップ[ \\t*]*$", - "end_pattern": "^[ \\t#*]*ゲームを楽しむ" - } - } - }, - "artifact": "## ピックアップ\n[Future Game Alpha](https://store-jp.nintendo.com/item/software/D70010000999991)\n[Future Game Beta](https://www.nintendo.com/jp/topics/article/01234567-1234-1234-1234-123456789abc)\n### ゲームを楽しむ", - "ok": true, - "expected_task_pass": true - }, - { - "id": "nintendo-markdown-section-navigation-news-only", - "category": "structured-extraction", - "root_cause": "game_card_announcements_not_navigation_news", - "failure_mode": "false-positive", - "case": { - "url": "https://www.nintendo.com/", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 2, - "hosts": [ - "www.nintendo.com", - "store-jp.nintendo.com" - ], - "entry_path": "(?:/item/software/[A-Za-z0-9]+|/jp/games/(?:switch2/[A-Za-z0-9-]+|zelda/(?:botw|totk)/edition|feature/nintendo-classics/[A-Za-z0-9_-]+)/index\\.html|/jp/xenoblade/index\\.html|/jp/topics/article/[0-9a-f-]{36})", - "stable_text_anchors": [], - "name_validation": "nonempty", - "entry_scope": { - "html_selector": ".nc3-c-softCard", - "title_selector": ".nc3-c-softCard__name", - "link_selector": "a[href]:has(.nc3-c-softCard__name)", - "start_pattern": "^[ \\t#*·-]*ピックアップ[ \\t*]*$", - "end_pattern": "^[ \\t#*]*ゲームを楽しむ" - } - } - }, - "artifact": "[Future Game Alpha](https://store-jp.nintendo.com/item/software/D70010000999991)\n[Future Game Beta](https://www.nintendo.com/jp/topics/article/01234567-1234-1234-1234-123456789abc)\n## ピックアップ\nNo game records\n### ゲームを楽しむ", - "ok": true, - "expected_task_pass": false - }, - { - "id": "nintendo-markdown-section-links-after-game-section", - "category": "structured-extraction", - "root_cause": "game_card_announcements_not_navigation_news", - "failure_mode": "false-positive", - "case": { - "url": "https://www.nintendo.com/", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 2, - "hosts": [ - "www.nintendo.com", - "store-jp.nintendo.com" - ], - "entry_path": "(?:/item/software/[A-Za-z0-9]+|/jp/games/(?:switch2/[A-Za-z0-9-]+|zelda/(?:botw|totk)/edition|feature/nintendo-classics/[A-Za-z0-9_-]+)/index\\.html|/jp/xenoblade/index\\.html|/jp/topics/article/[0-9a-f-]{36})", - "stable_text_anchors": [], - "name_validation": "nonempty", - "entry_scope": { - "html_selector": ".nc3-c-softCard", - "title_selector": ".nc3-c-softCard__name", - "link_selector": "a[href]:has(.nc3-c-softCard__name)", - "start_pattern": "^[ \\t#*·-]*ピックアップ[ \\t*]*$", - "end_pattern": "^[ \\t#*]*ゲームを楽しむ" - } - } - }, - "artifact": "## ピックアップ\nNo game records\n### ゲームを楽しむ\n[Future Game Alpha](https://store-jp.nintendo.com/item/software/D70010000999991)\n[Future Game Beta](https://www.nintendo.com/jp/topics/article/01234567-1234-1234-1234-123456789abc)", - "ok": true, - "expected_task_pass": false - }, - { - "id": "nintendo-markdown-section-bullet-heading", - "category": "structured-extraction", - "root_cause": "game_card_announcements_not_navigation_news", - "failure_mode": "false-negative", - "case": { - "url": "https://www.nintendo.com/", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 2, - "hosts": [ - "www.nintendo.com", - "store-jp.nintendo.com" - ], - "entry_path": "(?:/item/software/[A-Za-z0-9]+|/jp/games/(?:switch2/[A-Za-z0-9-]+|zelda/(?:botw|totk)/edition|feature/nintendo-classics/[A-Za-z0-9_-]+)/index\\.html|/jp/xenoblade/index\\.html|/jp/topics/article/[0-9a-f-]{36})", - "stable_text_anchors": [], - "name_validation": "nonempty", - "entry_scope": { - "html_selector": ".nc3-c-softCard", - "title_selector": ".nc3-c-softCard__name", - "link_selector": "a[href]:has(.nc3-c-softCard__name)", - "start_pattern": "^[ \\t#*·-]*ピックアップ[ \\t*]*$", - "end_pattern": "^[ \\t#*]*ゲームを楽しむ" - } - } - }, - "artifact": "* ピックアップ\n[Future Game Alpha](https://store-jp.nintendo.com/item/software/D70010000999991)\n[Future Game Beta](https://www.nintendo.com/jp/topics/article/01234567-1234-1234-1234-123456789abc)", - "ok": true, - "expected_task_pass": true - }, - { - "id": "dynamic-026-bilibili-rank-markdown-scope-inside", - "category": "structured-extraction", - "root_cause": "rank_markdown_scope_excludes_navigation", - "failure_mode": "false-negative", - "case": { - "url": "https://www.bilibili.com/", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 2, - "hosts": [ - "www.bilibili.com" - ], - "entry_path": "/video/(?:BV[A-Za-z0-9]+|av[0-9]+)", - "stable_text_anchors": [], - "entry_scope": { - "html_selector": ".rank-item a.title", - "start_pattern": "排行榜根据稿件内容质量，近期的数据综合展示，动态更新", - "end_pattern": "(?i)^[ \\t#*]*bilibili[ \\t*]*$" - }, - "name_validation": "nonempty" - } - }, - "artifact": "排行榜根据稿件内容质量，近期的数据综合展示，动态更新\n[Actual Name 0](https://www.bilibili.com/video/BV1Ab4111111)\n[Actual Name 1](https://www.bilibili.com/video/BV1Ab4222222)\nbilibili", - "ok": true, - "expected_task_pass": true - }, - { - "id": "dynamic-026-bilibili-rank-markdown-scope-navigation-only", - "category": "structured-extraction", - "root_cause": "rank_markdown_scope_excludes_navigation", - "failure_mode": "false-positive", - "case": { - "url": "https://www.bilibili.com/", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 2, - "hosts": [ - "www.bilibili.com" - ], - "entry_path": "/video/(?:BV[A-Za-z0-9]+|av[0-9]+)", - "stable_text_anchors": [], - "entry_scope": { - "html_selector": ".rank-item a.title", - "start_pattern": "排行榜根据稿件内容质量，近期的数据综合展示，动态更新", - "end_pattern": "(?i)^[ \\t#*]*bilibili[ \\t*]*$" - }, - "name_validation": "nonempty" - } - }, - "artifact": "[Actual Name 0](https://www.bilibili.com/video/BV1Ab4111111)\n[Actual Name 1](https://www.bilibili.com/video/BV1Ab4222222)\n排行榜根据稿件内容质量，近期的数据综合展示，动态更新\nNo target records\nbilibili", - "ok": true, - "expected_task_pass": false - }, - { - "id": "dynamic-026-bilibili-rank-markdown-scope-footer-only", - "category": "structured-extraction", - "root_cause": "rank_markdown_scope_excludes_navigation", - "failure_mode": "false-positive", - "case": { - "url": "https://www.bilibili.com/", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 2, - "hosts": [ - "www.bilibili.com" - ], - "entry_path": "/video/(?:BV[A-Za-z0-9]+|av[0-9]+)", - "stable_text_anchors": [], - "entry_scope": { - "html_selector": ".rank-item a.title", - "start_pattern": "排行榜根据稿件内容质量，近期的数据综合展示，动态更新", - "end_pattern": "(?i)^[ \\t#*]*bilibili[ \\t*]*$" - }, - "name_validation": "nonempty" - } - }, - "artifact": "排行榜根据稿件内容质量，近期的数据综合展示，动态更新\nNo target records\nbilibili\n[Actual Name 0](https://www.bilibili.com/video/BV1Ab4111111)\n[Actual Name 1](https://www.bilibili.com/video/BV1Ab4222222)", - "ok": true, - "expected_task_pass": false - }, - { - "id": "dynamic-048-en-github-trending-developers-markdown-scope-inside", - "category": "structured-extraction", - "root_cause": "rank_markdown_scope_excludes_navigation", - "failure_mode": "false-negative", - "case": { - "url": "https://github.com/", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 2, - "hosts": [ - "github.com" - ], - "entry_path": "/(?!login$|signup$|features$|topics$|trending$)[A-Za-z0-9_.-]+", - "stable_text_anchors": [], - "entry_scope": { - "html_selector": "article.Box-row h1", - "start_pattern": "These are the developers building the hot tools today\\.", - "end_pattern": "^[ \\t#*]*Footer[ \\t*]*$" - }, - "name_validation": "nonempty" - } - }, - "artifact": "These are the developers building the hot tools today.\n[Actual Name 0](https://github.com/newalpha)\n[Actual Name 1](https://github.com/newbeta)\nFooter", - "ok": true, - "expected_task_pass": true - }, - { - "id": "dynamic-048-en-github-trending-developers-markdown-scope-navigation-only", - "category": "structured-extraction", - "root_cause": "rank_markdown_scope_excludes_navigation", - "failure_mode": "false-positive", - "case": { - "url": "https://github.com/", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 2, - "hosts": [ - "github.com" - ], - "entry_path": "/(?!login$|signup$|features$|topics$|trending$)[A-Za-z0-9_.-]+", - "stable_text_anchors": [], - "entry_scope": { - "html_selector": "article.Box-row h1", - "start_pattern": "These are the developers building the hot tools today\\.", - "end_pattern": "^[ \\t#*]*Footer[ \\t*]*$" - }, - "name_validation": "nonempty" - } - }, - "artifact": "[Actual Name 0](https://github.com/newalpha)\n[Actual Name 1](https://github.com/newbeta)\nThese are the developers building the hot tools today.\nNo target records\nFooter", - "ok": true, - "expected_task_pass": false - }, - { - "id": "dynamic-048-en-github-trending-developers-markdown-scope-footer-only", - "category": "structured-extraction", - "root_cause": "rank_markdown_scope_excludes_navigation", - "failure_mode": "false-positive", - "case": { - "url": "https://github.com/", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 2, - "hosts": [ - "github.com" - ], - "entry_path": "/(?!login$|signup$|features$|topics$|trending$)[A-Za-z0-9_.-]+", - "stable_text_anchors": [], - "entry_scope": { - "html_selector": "article.Box-row h1", - "start_pattern": "These are the developers building the hot tools today\\.", - "end_pattern": "^[ \\t#*]*Footer[ \\t*]*$" - }, - "name_validation": "nonempty" - } - }, - "artifact": "These are the developers building the hot tools today.\nNo target records\nFooter\n[Actual Name 0](https://github.com/newalpha)\n[Actual Name 1](https://github.com/newbeta)", - "ok": true, - "expected_task_pass": false - }, - { - "id": "dynamic-044-en-real-github-trending-markdown-scope-inside", - "category": "structured-extraction", - "root_cause": "rank_markdown_scope_excludes_navigation", - "failure_mode": "false-negative", - "case": { - "url": "https://github.com/", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 2, - "hosts": [ - "github.com" - ], - "entry_path": "/(?!trending(?:/|$)|apps/|sponsors/|topics/|collections/|settings/|features/|marketplace/|orgs/|users/|login/|signup/)[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+", - "stable_text_anchors": [], - "entry_scope": { - "html_selector": "article.Box-row h2", - "start_pattern": "See what the GitHub community is most excited about today\\.", - "end_pattern": "^[ \\t#*]*Footer[ \\t*]*$" - }, - "name_validation": "nonempty" - } - }, - "artifact": "See what the GitHub community is most excited about today.\n[Actual Name 0](https://github.com/newalpha/project)\n[Actual Name 1](https://github.com/newbeta/project)\nFooter", - "ok": true, - "expected_task_pass": true - }, - { - "id": "dynamic-044-en-real-github-trending-markdown-scope-navigation-only", - "category": "structured-extraction", - "root_cause": "rank_markdown_scope_excludes_navigation", - "failure_mode": "false-positive", - "case": { - "url": "https://github.com/", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 2, - "hosts": [ - "github.com" - ], - "entry_path": "/(?!trending(?:/|$)|apps/|sponsors/|topics/|collections/|settings/|features/|marketplace/|orgs/|users/|login/|signup/)[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+", - "stable_text_anchors": [], - "entry_scope": { - "html_selector": "article.Box-row h2", - "start_pattern": "See what the GitHub community is most excited about today\\.", - "end_pattern": "^[ \\t#*]*Footer[ \\t*]*$" - }, - "name_validation": "nonempty" - } - }, - "artifact": "[Actual Name 0](https://github.com/newalpha/project)\n[Actual Name 1](https://github.com/newbeta/project)\nSee what the GitHub community is most excited about today.\nNo target records\nFooter", - "ok": true, - "expected_task_pass": false - }, - { - "id": "dynamic-044-en-real-github-trending-markdown-scope-footer-only", - "category": "structured-extraction", - "root_cause": "rank_markdown_scope_excludes_navigation", - "failure_mode": "false-positive", - "case": { - "url": "https://github.com/", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 2, - "hosts": [ - "github.com" - ], - "entry_path": "/(?!trending(?:/|$)|apps/|sponsors/|topics/|collections/|settings/|features/|marketplace/|orgs/|users/|login/|signup/)[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+", - "stable_text_anchors": [], - "entry_scope": { - "html_selector": "article.Box-row h2", - "start_pattern": "See what the GitHub community is most excited about today\\.", - "end_pattern": "^[ \\t#*]*Footer[ \\t*]*$" - }, - "name_validation": "nonempty" - } - }, - "artifact": "See what the GitHub community is most excited about today.\nNo target records\nFooter\n[Actual Name 0](https://github.com/newalpha/project)\n[Actual Name 1](https://github.com/newbeta/project)", - "ok": true, - "expected_task_pass": false - }, - { - "id": "auxiliary-video-links-two-main-plus-popup", - "category": "structured-extraction", - "root_cause": "hidden_related_video_popup_not_main_rank", - "failure_mode": "false-negative", - "case": { - "url": "https://www.bilibili.com/", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 2, - "hosts": [ - "www.bilibili.com" - ], - "entry_path": "/video/(?:BV[A-Za-z0-9]+|av[0-9]+)", - "stable_text_anchors": [], - "entry_scope": { - "html_selector": ".rank-item a.title", - "start_pattern": "排行榜根据稿件内容质量，近期的数据综合展示，动态更新", - "end_pattern": "(?i)^[ \\t#*]*bilibili[ \\t*]*$", - "auxiliary_links_end_pattern": "全部上榜视频" - }, - "name_validation": "nonempty" - } - }, - "artifact": "排行榜根据稿件内容质量，近期的数据综合展示，动态更新\n[Main Alpha](https://www.bilibili.com/video/BV1Ab4111111)\n123 views\n[Other One](https://www.bilibili.com/video/BV1Ab4333333)\n[Other Two](https://www.bilibili.com/video/BV1Ab4444444)\n全部上榜视频\n[Main Beta](https://www.bilibili.com/video/BV1Ab4222222)", - "ok": true, - "expected_task_pass": true - }, - { - "id": "auxiliary-video-links-popup-cannot-fill", - "category": "structured-extraction", - "root_cause": "hidden_related_video_popup_not_main_rank", - "failure_mode": "false-positive", - "case": { - "url": "https://www.bilibili.com/", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 2, - "hosts": [ - "www.bilibili.com" - ], - "entry_path": "/video/(?:BV[A-Za-z0-9]+|av[0-9]+)", - "stable_text_anchors": [], - "entry_scope": { - "html_selector": ".rank-item a.title", - "start_pattern": "排行榜根据稿件内容质量，近期的数据综合展示，动态更新", - "end_pattern": "(?i)^[ \\t#*]*bilibili[ \\t*]*$", - "auxiliary_links_end_pattern": "全部上榜视频" - }, - "name_validation": "nonempty" - } - }, - "artifact": "排行榜根据稿件内容质量，近期的数据综合展示，动态更新\n[Main Alpha](https://www.bilibili.com/video/BV1Ab4111111)\n123 views\n[Other One](https://www.bilibili.com/video/BV1Ab4333333)\n[Other Two](https://www.bilibili.com/video/BV1Ab4444444)\n全部上榜视频", - "ok": true, - "expected_task_pass": false - }, - { - "id": "auxiliary-video-links-normal-two-main", - "category": "structured-extraction", - "root_cause": "hidden_related_video_popup_not_main_rank", - "failure_mode": "false-negative", - "case": { - "url": "https://www.bilibili.com/", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 2, - "hosts": [ - "www.bilibili.com" - ], - "entry_path": "/video/(?:BV[A-Za-z0-9]+|av[0-9]+)", - "stable_text_anchors": [], - "entry_scope": { - "html_selector": ".rank-item a.title", - "start_pattern": "排行榜根据稿件内容质量，近期的数据综合展示，动态更新", - "end_pattern": "(?i)^[ \\t#*]*bilibili[ \\t*]*$", - "auxiliary_links_end_pattern": "全部上榜视频" - }, - "name_validation": "nonempty" - } - }, - "artifact": "排行榜根据稿件内容质量，近期的数据综合展示，动态更新\n[Main Alpha](https://www.bilibili.com/video/BV1Ab4111111)\n[Main Beta](https://www.bilibili.com/video/BV1Ab4222222)", - "ok": true, - "expected_task_pass": true - }, - { - "id": "markdown-title-metadata-inner-double", - "category": "format-normalization", - "root_cause": "quoted_title_metadata_not_link_destination", - "failure_mode": "false-negative", - "case": { - "url": "https://www.bilibili.com/", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 1, - "hosts": [ - "www.bilibili.com" - ], - "entry_path": "/video/(?:BV[A-Za-z0-9]+|av[0-9]+)", - "stable_text_anchors": [], - "entry_scope": { - "html_selector": ".rank-item a.title", - "start_pattern": "排行榜根据稿件内容质量，近期的数据综合展示，动态更新", - "end_pattern": "(?i)^[ \\t#*]*bilibili[ \\t*]*$", - "auxiliary_links_end_pattern": "全部上榜视频" - }, - "name_validation": "nonempty" - } - }, - "artifact": "排行榜根据稿件内容质量，近期的数据综合展示，动态更新\n[Actual \"Name\"](https://www.bilibili.com/video/BV1Ab4111111 \"Actual \"Name\"\")", - "ok": true, - "expected_task_pass": true - }, - { - "id": "markdown-title-metadata-inner-apostrophe", - "category": "format-normalization", - "root_cause": "quoted_title_metadata_not_link_destination", - "failure_mode": "false-negative", - "case": { - "url": "https://www.bilibili.com/", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 1, - "hosts": [ - "www.bilibili.com" - ], - "entry_path": "/video/(?:BV[A-Za-z0-9]+|av[0-9]+)", - "stable_text_anchors": [], - "entry_scope": { - "html_selector": ".rank-item a.title", - "start_pattern": "排行榜根据稿件内容质量，近期的数据综合展示，动态更新", - "end_pattern": "(?i)^[ \\t#*]*bilibili[ \\t*]*$", - "auxiliary_links_end_pattern": "全部上榜视频" - }, - "name_validation": "nonempty" - } - }, - "artifact": "排行榜根据稿件内容质量，近期的数据综合展示，动态更新\n[Actual Name](https://www.bilibili.com/video/BV1Ab4111111 \"User's title\")", - "ok": true, - "expected_task_pass": true - }, - { - "id": "markdown-title-metadata-wrong-path", - "category": "format-normalization", - "root_cause": "quoted_title_metadata_not_link_destination", - "failure_mode": "false-positive", - "case": { - "url": "https://www.bilibili.com/", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 1, - "hosts": [ - "www.bilibili.com" - ], - "entry_path": "/video/(?:BV[A-Za-z0-9]+|av[0-9]+)", - "stable_text_anchors": [], - "entry_scope": { - "html_selector": ".rank-item a.title", - "start_pattern": "排行榜根据稿件内容质量，近期的数据综合展示，动态更新", - "end_pattern": "(?i)^[ \\t#*]*bilibili[ \\t*]*$", - "auxiliary_links_end_pattern": "全部上榜视频" - }, - "name_validation": "nonempty" - } - }, - "artifact": "排行榜根据稿件内容质量，近期的数据综合展示，动态更新\n[Actual Name](https://www.bilibili.com/help \"Actual \"Name\"\")", - "ok": true, - "expected_task_pass": false - }, - { - "id": "markdown-title-metadata-bare-url", - "category": "format-normalization", - "root_cause": "quoted_title_metadata_not_link_destination", - "failure_mode": "false-positive", - "case": { - "url": "https://www.bilibili.com/", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 1, - "hosts": [ - "www.bilibili.com" - ], - "entry_path": "/video/(?:BV[A-Za-z0-9]+|av[0-9]+)", - "stable_text_anchors": [], - "entry_scope": { - "html_selector": ".rank-item a.title", - "start_pattern": "排行榜根据稿件内容质量，近期的数据综合展示，动态更新", - "end_pattern": "(?i)^[ \\t#*]*bilibili[ \\t*]*$", - "auxiliary_links_end_pattern": "全部上榜视频" - }, - "name_validation": "nonempty" + { + "name": "gender", + "label": "Gender", + "type": "text", + "pattern": "(?i)female|male" + }, + { + "name": "age", + "label": "Age", + "type": "integer", + "minimum": 0, + "maximum": 150 + }, + { + "name": "postcode", + "label": "Postcode", + "type": "token", + "pattern": "\\d{5}(?:-\\d{4})?" + }, + { + "name": "email", + "label": "Email", + "type": "text", + "pattern": "[^\\s@]+@[^\\s@]+\\.[^\\s@]+" } + ], + "record_identity_fields": [ + "id" + ], + "table_scope": { + "root_selector": ".rs-table", + "root_index": 0, + "row_selector": "[role=row]:not(.rs-table-row-header)", + "cell_selector": "[role=gridcell]:not([aria-colindex=\"8\"]) .rs-table-cell-content", + "start_pattern": "(?im)^(?=[ \\t|]*Id(?:[ \\t|]+|\\n)First Name)", + "end_pattern": "(?im)^[ \\t#]*(?:Appearance|Loading\\.\\.\\.)[ \\t]*$", + "plain_row_pattern": "(?P\\d+)\\s+(?P[^\\d\\n]+?)\\s+(?P[^\\d\\n]+?)\\s+(?P(?i:female|male))\\s+(?P\\d+)\\s+(?P\\d{5}(?:-\\d{4})?)\\s+(?P[^\\s@]+@[^\\s@]+\\.[^\\s@]+)(?:\\s+Edit)?", + "plain_multiline_row_pattern": "(?m)^(?P\\d+)\\n(?P[^\\d\\n]+)\\n(?P[^\\d\\n]+)\\n(?P(?i:female|male))\\n(?P\\d+)\\n(?P\\d{5}(?:-\\d{4})?)\\n(?P[^\\s@]+@[^\\s@]+\\.[^\\s@]+)\\nEdit[ \\t]*(?=\\n|$)" + } }, - "artifact": "排行榜根据稿件内容质量，近期的数据综合展示，动态更新\nActual Name https://www.bilibili.com/video/BV1Ab4111111", - "ok": true, - "expected_task_pass": false - }, - { - "id": "adjacent-song-pair-escaped-markdown-complete", - "category": "structured-extraction", - "root_cause": "adjacent_markdown_links_same_song_identity", - "failure_mode": "false-negative", - "case": { - "url": "https://www.melon.com/chart/", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 2, - "hosts": [ - "www.melon.com" - ], - "entry_path": "/song/detail\\.htm", - "stable_text_anchors": [], - "identity_query_keys": [ - "songId" - ], - "entry_scope": { - "html_selector": "tr[data-song-no]", - "markdown_title_identity_pattern": "javascript:melon\\.play\\.playSong\\([^,]*,\\s*([0-9]+)\\);?", - "markdown_detail_identity_pattern": "[?&]songId=([0-9]+)(?:&|$)" - }, - "name_validation": "nonempty", - "title_exclusion_patterns": [ - "곡정보" - ] + "body_requirements": {} + }, + "artifact": "Id\nFirst Name\nLast Name\nGender\nAge\nPostcode\nEmail\n...\n", + "ok": true, + "expected_task_pass": false + }, + { + "id": "table-ordinal-date-real-date", + "category": "structured-extraction", + "root_cause": "calendar_validity_for_table_dates", + "failure_mode": "false-negative", + "case": { + "url": "https://example.test/people", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 1, + "table_columns": [ + { + "name": "name", + "label": "Name", + "type": "text" + }, + { + "name": "birth", + "label": "Date of Birth", + "type": "date_ordinal_en" } - }, - "artifact": "[곡정보](/song/detail.htm?songId=101)\n\n[새 노래 Alpha](javascript:melon.play.playSong\\('1000002721',101\\);)\n\n[곡정보](/song/detail.htm?songId=102)\n\n[새 노래 Beta](javascript:melon.play.playSong\\('1000002721',102\\);)", - "ok": true, - "expected_task_pass": true - }, - { - "id": "adjacent-song-pair-mismatched-identity", - "category": "structured-extraction", - "root_cause": "adjacent_markdown_links_same_song_identity", - "failure_mode": "false-positive", - "case": { - "url": "https://www.melon.com/chart/", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 2, - "hosts": [ - "www.melon.com" - ], - "entry_path": "/song/detail\\.htm", - "stable_text_anchors": [], - "identity_query_keys": [ - "songId" - ], - "entry_scope": { - "html_selector": "tr[data-song-no]", - "markdown_title_identity_pattern": "javascript:melon\\.play\\.playSong\\([^,]*,\\s*([0-9]+)\\);?", - "markdown_detail_identity_pattern": "[?&]songId=([0-9]+)(?:&|$)" - }, - "name_validation": "nonempty", - "title_exclusion_patterns": [ - "곡정보" - ] + ], + "record_identity_fields": [ + "name" + ], + "table_scope": { + "row_selector": "tbody tr", + "cell_selector": "td", + "start_pattern": "People\n", + "end_pattern": "Next example", + "plain_row_pattern": "(?P[^\\d\\n]+?)\\s+(?P\\d{1,2}(?:st|nd|rd|th) [A-Za-z]{3} \\d{2,4})" + } + } + }, + "artifact": "People\nNew Person 23rd Feb 26", + "ok": true, + "expected_task_pass": true + }, + { + "id": "table-ordinal-date-leap-date", + "category": "structured-extraction", + "root_cause": "calendar_validity_for_table_dates", + "failure_mode": "false-negative", + "case": { + "url": "https://example.test/people", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 1, + "table_columns": [ + { + "name": "name", + "label": "Name", + "type": "text" + }, + { + "name": "birth", + "label": "Date of Birth", + "type": "date_ordinal_en" } - }, - "artifact": "[곡정보](/song/detail.htm?songId=101)\n\n[새 노래 Alpha](javascript:melon.play.playSong\\('1000002721',101\\);)\n\n[곡정보](/song/detail.htm?songId=999)\n\n[새 노래 Beta](javascript:melon.play.playSong\\('1000002721',102\\);)", - "ok": true, - "expected_task_pass": false - }, - { - "id": "same-row-song-pair-complete", - "category": "structured-extraction", - "root_cause": "visible_song_name_and_detail_link_same_row_identity", - "failure_mode": "false-negative", - "case": { - "url": "https://www.melon.com/chart/", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 2, - "hosts": [ - "www.melon.com" - ], - "entry_path": "/song/detail\\.htm", - "stable_text_anchors": [], - "identity_query_keys": [ - "songId" - ], - "entry_scope": { - "html_selector": "tr[data-song-no]", - "title_selector": ".rank01 a", - "link_selector": "a.song_info[href]", - "markdown_title_identity_pattern": "javascript:melon\\.play\\.playSong\\([^,]*,\\s*([0-9]+)\\);?", - "markdown_detail_identity_pattern": "[?&]songId=([0-9]+)(?:&|$)" - }, - "name_validation": "nonempty", - "title_exclusion_patterns": [ - "곡정보" - ] + ], + "record_identity_fields": [ + "name" + ], + "table_scope": { + "row_selector": "tbody tr", + "cell_selector": "td", + "start_pattern": "People\n", + "end_pattern": "Next example", + "plain_row_pattern": "(?P[^\\d\\n]+?)\\s+(?P\\d{1,2}(?:st|nd|rd|th) [A-Za-z]{3} \\d{2,4})" + } + } + }, + "artifact": "People\nNew Person 29th Feb 2024", + "ok": true, + "expected_task_pass": true + }, + { + "id": "table-ordinal-date-impossible-day", + "category": "structured-extraction", + "root_cause": "calendar_validity_for_table_dates", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/people", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 1, + "table_columns": [ + { + "name": "name", + "label": "Name", + "type": "text" + }, + { + "name": "birth", + "label": "Date of Birth", + "type": "date_ordinal_en" } - }, - "artifact": "| [곡정보](/song/detail.htm?songId=101) | [새 노래 Alpha](javascript:melon.play.playSong%28'1000002721',101%29;) |\n| [곡정보](/song/detail.htm?songId=102) | [새 노래 Beta](javascript:melon.play.playSong%28'1000002721',102%29;) |", - "ok": true, - "expected_task_pass": true - }, - { - "id": "same-row-song-pair-mismatched-id", - "category": "structured-extraction", - "root_cause": "visible_song_name_and_detail_link_same_row_identity", - "failure_mode": "false-positive", - "case": { - "url": "https://www.melon.com/chart/", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 2, - "hosts": [ - "www.melon.com" - ], - "entry_path": "/song/detail\\.htm", - "stable_text_anchors": [], - "identity_query_keys": [ - "songId" - ], - "entry_scope": { - "html_selector": "tr[data-song-no]", - "title_selector": ".rank01 a", - "link_selector": "a.song_info[href]", - "markdown_title_identity_pattern": "javascript:melon\\.play\\.playSong\\([^,]*,\\s*([0-9]+)\\);?", - "markdown_detail_identity_pattern": "[?&]songId=([0-9]+)(?:&|$)" - }, - "name_validation": "nonempty", - "title_exclusion_patterns": [ - "곡정보" - ] + ], + "record_identity_fields": [ + "name" + ], + "table_scope": { + "row_selector": "tbody tr", + "cell_selector": "td", + "start_pattern": "People\n", + "end_pattern": "Next example", + "plain_row_pattern": "(?P[^\\d\\n]+?)\\s+(?P\\d{1,2}(?:st|nd|rd|th) [A-Za-z]{3} \\d{2,4})" + } + } + }, + "artifact": "People\nNew Person 31st Feb 26", + "ok": true, + "expected_task_pass": false + }, + { + "id": "table-ordinal-date-non-leap-date", + "category": "structured-extraction", + "root_cause": "calendar_validity_for_table_dates", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/people", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 1, + "table_columns": [ + { + "name": "name", + "label": "Name", + "type": "text" + }, + { + "name": "birth", + "label": "Date of Birth", + "type": "date_ordinal_en" } - }, - "artifact": "| [곡정보](/song/detail.htm?songId=101) | [새 노래 Alpha](javascript:melon.play.playSong%28'1000002721',101%29;) |\n| [곡정보](/song/detail.htm?songId=999) | [새 노래 Beta](javascript:melon.play.playSong%28'1000002721',102%29;) |", - "ok": true, - "expected_task_pass": false - }, - { - "id": "same-row-song-pair-cross-row", - "category": "structured-extraction", - "root_cause": "visible_song_name_and_detail_link_same_row_identity", - "failure_mode": "false-positive", - "case": { - "url": "https://www.melon.com/chart/", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 2, - "hosts": [ - "www.melon.com" - ], - "entry_path": "/song/detail\\.htm", - "stable_text_anchors": [], - "identity_query_keys": [ - "songId" - ], - "entry_scope": { - "html_selector": "tr[data-song-no]", - "title_selector": ".rank01 a", - "link_selector": "a.song_info[href]", - "markdown_title_identity_pattern": "javascript:melon\\.play\\.playSong\\([^,]*,\\s*([0-9]+)\\);?", - "markdown_detail_identity_pattern": "[?&]songId=([0-9]+)(?:&|$)" - }, - "name_validation": "nonempty", - "title_exclusion_patterns": [ - "곡정보" - ] + ], + "record_identity_fields": [ + "name" + ], + "table_scope": { + "row_selector": "tbody tr", + "cell_selector": "td", + "start_pattern": "People\n", + "end_pattern": "Next example", + "plain_row_pattern": "(?P[^\\d\\n]+?)\\s+(?P\\d{1,2}(?:st|nd|rd|th) [A-Za-z]{3} \\d{2,4})" + } + } + }, + "artifact": "People\nNew Person 29th Feb 2025", + "ok": true, + "expected_task_pass": false + }, + { + "id": "table-ordinal-date-wrong-ordinal", + "category": "structured-extraction", + "root_cause": "calendar_validity_for_table_dates", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/people", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 1, + "table_columns": [ + { + "name": "name", + "label": "Name", + "type": "text" + }, + { + "name": "birth", + "label": "Date of Birth", + "type": "date_ordinal_en" } - }, - "artifact": "| [곡정보](/song/detail.htm?songId=101) | [새 노래 Alpha](javascript:melon.play.playSong%28'1000002721',101%29;) |\n| [곡정보](/song/detail.htm?songId=102) |\n| [새 노래 Beta](javascript:melon.play.playSong%28'1000002721',102%29;) |", - "ok": true, - "expected_task_pass": false - }, - { - "id": "same-row-song-pair-duplicates", - "category": "structured-extraction", - "root_cause": "visible_song_name_and_detail_link_same_row_identity", - "failure_mode": "false-positive", - "case": { - "url": "https://www.melon.com/chart/", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 2, - "hosts": [ - "www.melon.com" - ], - "entry_path": "/song/detail\\.htm", - "stable_text_anchors": [], - "identity_query_keys": [ - "songId" - ], - "entry_scope": { - "html_selector": "tr[data-song-no]", - "title_selector": ".rank01 a", - "link_selector": "a.song_info[href]", - "markdown_title_identity_pattern": "javascript:melon\\.play\\.playSong\\([^,]*,\\s*([0-9]+)\\);?", - "markdown_detail_identity_pattern": "[?&]songId=([0-9]+)(?:&|$)" - }, - "name_validation": "nonempty", - "title_exclusion_patterns": [ - "곡정보" - ] + ], + "record_identity_fields": [ + "name" + ], + "table_scope": { + "row_selector": "tbody tr", + "cell_selector": "td", + "start_pattern": "People\n", + "end_pattern": "Next example", + "plain_row_pattern": "(?P[^\\d\\n]+?)\\s+(?P\\d{1,2}(?:st|nd|rd|th) [A-Za-z]{3} \\d{2,4})" + } + } + }, + "artifact": "People\nNew Person 11st Jan 26", + "ok": true, + "expected_task_pass": false + }, + { + "id": "eui-complete-record-eight-complete", + "category": "acceptance-contract", + "root_cause": "volatile_eui_people_and_invisible_reference", + "failure_mode": "false-negative", + "case": { + "url": "https://example.test/people", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 8, + "table_columns": [ + { + "name": "first_name", + "label": "First Name", + "type": "text" + }, + { + "name": "last_name", + "label": "Last Name", + "type": "text" + }, + { + "name": "github", + "label": "Github", + "type": "text" + }, + { + "name": "birth", + "label": "Date of Birth", + "type": "date_ordinal_en" + }, + { + "name": "online", + "label": "Online", + "type": "text", + "pattern": "(?i)Online|Offline" } + ], + "record_identity_fields": [ + "first_name", + "last_name", + "github" + ], + "table_scope": { + "root_selector": "table.euiTable", + "root_index": 0, + "row_selector": "tbody tr", + "cell_selector": ":scope > th, :scope > td", + "exclude_selector": "[class*=euiScreenReaderOnly],svg", + "start_pattern": "(?im)^(?=[ \\t|]*First Name(?:[ \\t|]+|\\n))", + "end_pattern": "(?im)^[ \\t#]*Show source[ \\t]*$", + "plain_row_pattern": "(?P(?!(?i:online|offline)(?:\\s|$))[^\\d\\s↦↵]+)\\s+(?P(?!(?i:online|offline)(?:\\s|$))[^\\d\\s↦↵]+)\\s+(?P[^\\s↦↵]+)\\s+(?P\\d{1,2}(?:st|nd|rd|th) [A-Za-z]{3} (?:\\d{4}|\\d{2}))\\s+(?P(?i:Online|Offline))", + "plain_multiline_row_pattern": "(?m)^(?P(?!(?i:online|offline)(?:\\s|$))[^\\d\\s↦↵]+)[ \\t]*(?:↦[ \\t]*)?\\n(?:[ \\t]*(?:↦)?[ \\t]*\\n)*[ \\t]*(?P(?!(?i:online|offline)(?:\\s|$))[^\\d\\s↦↵]+)[ \\t]*(?:↦[ \\t]*)?\\n(?:[ \\t]*(?:↦)?[ \\t]*\\n)*[ \\t]*(?P[^\\s↦↵]+)(?:\\n\\(external, opens in a new tab or window\\))?[ \\t]*(?:↦[ \\t]*)?\\n(?:[ \\t]*(?:↦)?[ \\t]*\\n)*[ \\t]*(?P\\d{1,2}(?:st|nd|rd|th) [A-Za-z]{3} (?:\\d{4}|\\d{2}))[ \\t]*(?:↦[ \\t]*)?\\n(?:[ \\t]*(?:↦)?[ \\t]*\\n)*[ \\t]*(?P(?i:Online|Offline))[ \\t]*(?:\\n↵)?(?=\\n|$)" + } }, - "artifact": "| [곡정보](/song/detail.htm?songId=101) | [새 노래 Alpha](javascript:melon.play.playSong%28'1000002721',101%29;) |\n| [곡정보](/song/detail.htm?songId=101) | [새 노래 Alpha](javascript:melon.play.playSong%28'1000002721',101%29;) |", - "ok": true, - "expected_task_pass": false - }, - { - "id": "same-row-song-pair-empty-name", - "category": "structured-extraction", - "root_cause": "visible_song_name_and_detail_link_same_row_identity", - "failure_mode": "false-positive", - "case": { - "url": "https://www.melon.com/chart/", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 2, - "hosts": [ - "www.melon.com" - ], - "entry_path": "/song/detail\\.htm", - "stable_text_anchors": [], - "identity_query_keys": [ - "songId" - ], - "entry_scope": { - "html_selector": "tr[data-song-no]", - "title_selector": ".rank01 a", - "link_selector": "a.song_info[href]", - "markdown_title_identity_pattern": "javascript:melon\\.play\\.playSong\\([^,]*,\\s*([0-9]+)\\);?", - "markdown_detail_identity_pattern": "[?&]songId=([0-9]+)(?:&|$)" - }, - "name_validation": "nonempty", - "title_exclusion_patterns": [ - "곡정보" - ] + "body_requirements": {} + }, + "artifact": "First Name\nLast Name\nGithub\nDate of Birth\nOnline\nAda\nExample\nAda_account\n23rd Feb 26\nOnline\nBob\nExample\nBob_account\n23rd Feb 26\nOnline\nCarol\nExample\nCarol_account\n23rd Feb 26\nOnline\nDana\nExample\nDana_account\n23rd Feb 26\nOnline\nEmma\nExample\nEmma_account\n23rd Feb 26\nOnline\nFrank\nExample\nFrank_account\n23rd Feb 26\nOnline\nGrace\nExample\nGrace_account\n23rd Feb 26\nOnline\nHelen\nExample\nHelen_account\n23rd Feb 26\nOnline\nShow source", + "ok": true, + "expected_task_pass": true + }, + { + "id": "eui-complete-record-seven-below80", + "category": "acceptance-contract", + "root_cause": "volatile_eui_people_and_invisible_reference", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/people", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 8, + "table_columns": [ + { + "name": "first_name", + "label": "First Name", + "type": "text" + }, + { + "name": "last_name", + "label": "Last Name", + "type": "text" + }, + { + "name": "github", + "label": "Github", + "type": "text" + }, + { + "name": "birth", + "label": "Date of Birth", + "type": "date_ordinal_en" + }, + { + "name": "online", + "label": "Online", + "type": "text", + "pattern": "(?i)Online|Offline" } + ], + "record_identity_fields": [ + "first_name", + "last_name", + "github" + ], + "table_scope": { + "root_selector": "table.euiTable", + "root_index": 0, + "row_selector": "tbody tr", + "cell_selector": ":scope > th, :scope > td", + "exclude_selector": "[class*=euiScreenReaderOnly],svg", + "start_pattern": "(?im)^(?=[ \\t|]*First Name(?:[ \\t|]+|\\n))", + "end_pattern": "(?im)^[ \\t#]*Show source[ \\t]*$", + "plain_row_pattern": "(?P(?!(?i:online|offline)(?:\\s|$))[^\\d\\s↦↵]+)\\s+(?P(?!(?i:online|offline)(?:\\s|$))[^\\d\\s↦↵]+)\\s+(?P[^\\s↦↵]+)\\s+(?P\\d{1,2}(?:st|nd|rd|th) [A-Za-z]{3} (?:\\d{4}|\\d{2}))\\s+(?P(?i:Online|Offline))", + "plain_multiline_row_pattern": "(?m)^(?P(?!(?i:online|offline)(?:\\s|$))[^\\d\\s↦↵]+)[ \\t]*(?:↦[ \\t]*)?\\n(?:[ \\t]*(?:↦)?[ \\t]*\\n)*[ \\t]*(?P(?!(?i:online|offline)(?:\\s|$))[^\\d\\s↦↵]+)[ \\t]*(?:↦[ \\t]*)?\\n(?:[ \\t]*(?:↦)?[ \\t]*\\n)*[ \\t]*(?P[^\\s↦↵]+)(?:\\n\\(external, opens in a new tab or window\\))?[ \\t]*(?:↦[ \\t]*)?\\n(?:[ \\t]*(?:↦)?[ \\t]*\\n)*[ \\t]*(?P\\d{1,2}(?:st|nd|rd|th) [A-Za-z]{3} (?:\\d{4}|\\d{2}))[ \\t]*(?:↦[ \\t]*)?\\n(?:[ \\t]*(?:↦)?[ \\t]*\\n)*[ \\t]*(?P(?i:Online|Offline))[ \\t]*(?:\\n↵)?(?=\\n|$)" + } }, - "artifact": "| [곡정보](/song/detail.htm?songId=101) | [새 노래 Alpha](javascript:melon.play.playSong%28'1000002721',101%29;) |\n| [곡정보](/song/detail.htm?songId=102) | [](javascript:melon.play.playSong%28'1000002721',102%29;) |", - "ok": true, - "expected_task_pass": false - }, - { - "id": "same-row-song-pair-direct-named-links", - "category": "structured-extraction", - "root_cause": "visible_song_name_and_detail_link_same_row_identity", - "failure_mode": "false-negative", - "case": { - "url": "https://www.melon.com/chart/", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 2, - "hosts": [ - "www.melon.com" - ], - "entry_path": "/song/detail\\.htm", - "stable_text_anchors": [], - "identity_query_keys": [ - "songId" - ], - "entry_scope": { - "html_selector": "tr[data-song-no]", - "title_selector": ".rank01 a", - "link_selector": "a.song_info[href]", - "markdown_title_identity_pattern": "javascript:melon\\.play\\.playSong\\([^,]*,\\s*([0-9]+)\\);?", - "markdown_detail_identity_pattern": "[?&]songId=([0-9]+)(?:&|$)" - }, - "name_validation": "nonempty", - "title_exclusion_patterns": [ - "곡정보" - ] + "body_requirements": {} + }, + "artifact": "First Name\nLast Name\nGithub\nDate of Birth\nOnline\nAda\nExample\nAda_account\n23rd Feb 26\nOnline\nBob\nExample\nBob_account\n23rd Feb 26\nOnline\nCarol\nExample\nCarol_account\n23rd Feb 26\nOnline\nDana\nExample\nDana_account\n23rd Feb 26\nOnline\nEmma\nExample\nEmma_account\n23rd Feb 26\nOnline\nFrank\nExample\nFrank_account\n23rd Feb 26\nOnline\nGrace\nExample\nGrace_account\n23rd Feb 26\nOnline\nShow source", + "ok": true, + "expected_task_pass": false + }, + { + "id": "eui-complete-record-impossible-date", + "category": "acceptance-contract", + "root_cause": "volatile_eui_people_and_invisible_reference", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/people", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 8, + "table_columns": [ + { + "name": "first_name", + "label": "First Name", + "type": "text" + }, + { + "name": "last_name", + "label": "Last Name", + "type": "text" + }, + { + "name": "github", + "label": "Github", + "type": "text" + }, + { + "name": "birth", + "label": "Date of Birth", + "type": "date_ordinal_en" + }, + { + "name": "online", + "label": "Online", + "type": "text", + "pattern": "(?i)Online|Offline" } + ], + "record_identity_fields": [ + "first_name", + "last_name", + "github" + ], + "table_scope": { + "root_selector": "table.euiTable", + "root_index": 0, + "row_selector": "tbody tr", + "cell_selector": ":scope > th, :scope > td", + "exclude_selector": "[class*=euiScreenReaderOnly],svg", + "start_pattern": "(?im)^(?=[ \\t|]*First Name(?:[ \\t|]+|\\n))", + "end_pattern": "(?im)^[ \\t#]*Show source[ \\t]*$", + "plain_row_pattern": "(?P(?!(?i:online|offline)(?:\\s|$))[^\\d\\s↦↵]+)\\s+(?P(?!(?i:online|offline)(?:\\s|$))[^\\d\\s↦↵]+)\\s+(?P[^\\s↦↵]+)\\s+(?P\\d{1,2}(?:st|nd|rd|th) [A-Za-z]{3} (?:\\d{4}|\\d{2}))\\s+(?P(?i:Online|Offline))", + "plain_multiline_row_pattern": "(?m)^(?P(?!(?i:online|offline)(?:\\s|$))[^\\d\\s↦↵]+)[ \\t]*(?:↦[ \\t]*)?\\n(?:[ \\t]*(?:↦)?[ \\t]*\\n)*[ \\t]*(?P(?!(?i:online|offline)(?:\\s|$))[^\\d\\s↦↵]+)[ \\t]*(?:↦[ \\t]*)?\\n(?:[ \\t]*(?:↦)?[ \\t]*\\n)*[ \\t]*(?P[^\\s↦↵]+)(?:\\n\\(external, opens in a new tab or window\\))?[ \\t]*(?:↦[ \\t]*)?\\n(?:[ \\t]*(?:↦)?[ \\t]*\\n)*[ \\t]*(?P\\d{1,2}(?:st|nd|rd|th) [A-Za-z]{3} (?:\\d{4}|\\d{2}))[ \\t]*(?:↦[ \\t]*)?\\n(?:[ \\t]*(?:↦)?[ \\t]*\\n)*[ \\t]*(?P(?i:Online|Offline))[ \\t]*(?:\\n↵)?(?=\\n|$)" + } }, - "artifact": "[새 노래 Alpha](/song/detail.htm?songId=101)\n[새 노래 Beta](/song/detail.htm?songId=102)", - "ok": true, - "expected_task_pass": true - }, - { - "id": "dynamic-052-bilibili-com-v-footer-scope-inside", - "category": "structured-extraction", - "root_cause": "footer_links_not_dynamic_list_records", - "failure_mode": "false-negative", - "case": { - "url": "https://www.bilibili.com/v/popular/all/", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 2, - "hosts": [ - "www.bilibili.com" - ], - "entry_path": "/video/(?:BV[A-Za-z0-9]+|av[0-9]+)", - "stable_text_anchors": [], - "entry_scope": { - "html_selector": ".video-card", - "title_selector": "p.video-name", - "link_selector": "a[href*=\"/video/\"]", - "start_pattern": "各个领域中新奇好玩的优质内容都在这里", - "end_pattern": "(?i)^[ \\t#*]*bilibili[ \\t*]*$" - }, - "name_validation": "nonempty" + "body_requirements": {} + }, + "artifact": "First Name\nLast Name\nGithub\nDate of Birth\nOnline\nAda\nExample\nAda_account\n23rd Feb 26\nOnline\nBob\nExample\nBob_account\n23rd Feb 26\nOnline\nCarol\nExample\nCarol_account\n23rd Feb 26\nOnline\nDana\nExample\nDana_account\n23rd Feb 26\nOnline\nEmma\nExample\nEmma_account\n23rd Feb 26\nOnline\nFrank\nExample\nFrank_account\n23rd Feb 26\nOnline\nGrace\nExample\nGrace_account\n23rd Feb 26\nOnline\nHelen\nExample\nHelen_account\n31st Feb 26\nOnline\nShow source", + "ok": true, + "expected_task_pass": false + }, + { + "id": "eui-complete-record-missing-status", + "category": "acceptance-contract", + "root_cause": "volatile_eui_people_and_invisible_reference", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/people", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 8, + "table_columns": [ + { + "name": "first_name", + "label": "First Name", + "type": "text" + }, + { + "name": "last_name", + "label": "Last Name", + "type": "text" + }, + { + "name": "github", + "label": "Github", + "type": "text" + }, + { + "name": "birth", + "label": "Date of Birth", + "type": "date_ordinal_en" + }, + { + "name": "online", + "label": "Online", + "type": "text", + "pattern": "(?i)Online|Offline" } + ], + "record_identity_fields": [ + "first_name", + "last_name", + "github" + ], + "table_scope": { + "root_selector": "table.euiTable", + "root_index": 0, + "row_selector": "tbody tr", + "cell_selector": ":scope > th, :scope > td", + "exclude_selector": "[class*=euiScreenReaderOnly],svg", + "start_pattern": "(?im)^(?=[ \\t|]*First Name(?:[ \\t|]+|\\n))", + "end_pattern": "(?im)^[ \\t#]*Show source[ \\t]*$", + "plain_row_pattern": "(?P(?!(?i:online|offline)(?:\\s|$))[^\\d\\s↦↵]+)\\s+(?P(?!(?i:online|offline)(?:\\s|$))[^\\d\\s↦↵]+)\\s+(?P[^\\s↦↵]+)\\s+(?P\\d{1,2}(?:st|nd|rd|th) [A-Za-z]{3} (?:\\d{4}|\\d{2}))\\s+(?P(?i:Online|Offline))", + "plain_multiline_row_pattern": "(?m)^(?P(?!(?i:online|offline)(?:\\s|$))[^\\d\\s↦↵]+)[ \\t]*(?:↦[ \\t]*)?\\n(?:[ \\t]*(?:↦)?[ \\t]*\\n)*[ \\t]*(?P(?!(?i:online|offline)(?:\\s|$))[^\\d\\s↦↵]+)[ \\t]*(?:↦[ \\t]*)?\\n(?:[ \\t]*(?:↦)?[ \\t]*\\n)*[ \\t]*(?P[^\\s↦↵]+)(?:\\n\\(external, opens in a new tab or window\\))?[ \\t]*(?:↦[ \\t]*)?\\n(?:[ \\t]*(?:↦)?[ \\t]*\\n)*[ \\t]*(?P\\d{1,2}(?:st|nd|rd|th) [A-Za-z]{3} (?:\\d{4}|\\d{2}))[ \\t]*(?:↦[ \\t]*)?\\n(?:[ \\t]*(?:↦)?[ \\t]*\\n)*[ \\t]*(?P(?i:Online|Offline))[ \\t]*(?:\\n↵)?(?=\\n|$)" + } }, - "artifact": "各个领域中新奇好玩的优质内容都在这里~\n[条目0](//www.bilibili.com/video/BV1Ab4111111)\n[条目1](//www.bilibili.com/video/BV1Ab4222222)\nbilibili", - "ok": true, - "expected_task_pass": true - }, - { - "id": "dynamic-052-bilibili-com-v-footer-scope-footer-only", - "category": "structured-extraction", - "root_cause": "footer_links_not_dynamic_list_records", - "failure_mode": "false-positive", - "case": { - "url": "https://www.bilibili.com/v/popular/all/", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 2, - "hosts": [ - "www.bilibili.com" - ], - "entry_path": "/video/(?:BV[A-Za-z0-9]+|av[0-9]+)", - "stable_text_anchors": [], - "entry_scope": { - "html_selector": ".video-card", - "title_selector": "p.video-name", - "link_selector": "a[href*=\"/video/\"]", - "start_pattern": "各个领域中新奇好玩的优质内容都在这里", - "end_pattern": "(?i)^[ \\t#*]*bilibili[ \\t*]*$" - }, - "name_validation": "nonempty" + "body_requirements": {} + }, + "artifact": "First Name\nLast Name\nGithub\nDate of Birth\nOnline\nAda\nExample\nAda_account\n23rd Feb 26\nOnline\nBob\nExample\nBob_account\n23rd Feb 26\nOnline\nCarol\nExample\nCarol_account\n23rd Feb 26\nOnline\nDana\nExample\nDana_account\n23rd Feb 26\nOnline\nEmma\nExample\nEmma_account\n23rd Feb 26\nOnline\nFrank\nExample\nFrank_account\n23rd Feb 26\nOnline\nGrace\nExample\nGrace_account\n23rd Feb 26\nOnline\nHelen\nExample\nHelen_account\n23rd Feb 26\nShow source", + "ok": true, + "expected_task_pass": false + }, + { + "id": "eui-complete-record-duplicates", + "category": "acceptance-contract", + "root_cause": "volatile_eui_people_and_invisible_reference", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/people", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 8, + "table_columns": [ + { + "name": "first_name", + "label": "First Name", + "type": "text" + }, + { + "name": "last_name", + "label": "Last Name", + "type": "text" + }, + { + "name": "github", + "label": "Github", + "type": "text" + }, + { + "name": "birth", + "label": "Date of Birth", + "type": "date_ordinal_en" + }, + { + "name": "online", + "label": "Online", + "type": "text", + "pattern": "(?i)Online|Offline" } + ], + "record_identity_fields": [ + "first_name", + "last_name", + "github" + ], + "table_scope": { + "root_selector": "table.euiTable", + "root_index": 0, + "row_selector": "tbody tr", + "cell_selector": ":scope > th, :scope > td", + "exclude_selector": "[class*=euiScreenReaderOnly],svg", + "start_pattern": "(?im)^(?=[ \\t|]*First Name(?:[ \\t|]+|\\n))", + "end_pattern": "(?im)^[ \\t#]*Show source[ \\t]*$", + "plain_row_pattern": "(?P(?!(?i:online|offline)(?:\\s|$))[^\\d\\s↦↵]+)\\s+(?P(?!(?i:online|offline)(?:\\s|$))[^\\d\\s↦↵]+)\\s+(?P[^\\s↦↵]+)\\s+(?P\\d{1,2}(?:st|nd|rd|th) [A-Za-z]{3} (?:\\d{4}|\\d{2}))\\s+(?P(?i:Online|Offline))", + "plain_multiline_row_pattern": "(?m)^(?P(?!(?i:online|offline)(?:\\s|$))[^\\d\\s↦↵]+)[ \\t]*(?:↦[ \\t]*)?\\n(?:[ \\t]*(?:↦)?[ \\t]*\\n)*[ \\t]*(?P(?!(?i:online|offline)(?:\\s|$))[^\\d\\s↦↵]+)[ \\t]*(?:↦[ \\t]*)?\\n(?:[ \\t]*(?:↦)?[ \\t]*\\n)*[ \\t]*(?P[^\\s↦↵]+)(?:\\n\\(external, opens in a new tab or window\\))?[ \\t]*(?:↦[ \\t]*)?\\n(?:[ \\t]*(?:↦)?[ \\t]*\\n)*[ \\t]*(?P\\d{1,2}(?:st|nd|rd|th) [A-Za-z]{3} (?:\\d{4}|\\d{2}))[ \\t]*(?:↦[ \\t]*)?\\n(?:[ \\t]*(?:↦)?[ \\t]*\\n)*[ \\t]*(?P(?i:Online|Offline))[ \\t]*(?:\\n↵)?(?=\\n|$)" + } }, - "artifact": "各个领域中新奇好玩的优质内容都在这里~\nbilibili\n[条目0](//www.bilibili.com/video/BV1Ab4111111)\n[条目1](//www.bilibili.com/video/BV1Ab4222222)", - "ok": true, - "expected_task_pass": false - }, - { - "id": "dynamic-052-bilibili-com-v-footer-scope-navigation-only", - "category": "structured-extraction", - "root_cause": "footer_links_not_dynamic_list_records", - "failure_mode": "false-positive", - "case": { - "url": "https://www.bilibili.com/v/popular/all/", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 2, - "hosts": [ - "www.bilibili.com" - ], - "entry_path": "/video/(?:BV[A-Za-z0-9]+|av[0-9]+)", - "stable_text_anchors": [], - "entry_scope": { - "html_selector": ".video-card", - "title_selector": "p.video-name", - "link_selector": "a[href*=\"/video/\"]", - "start_pattern": "各个领域中新奇好玩的优质内容都在这里", - "end_pattern": "(?i)^[ \\t#*]*bilibili[ \\t*]*$" - }, - "name_validation": "nonempty" + "body_requirements": {} + }, + "artifact": "First Name\nLast Name\nGithub\nDate of Birth\nOnline\nAda\nExample\nAda_account\n23rd Feb 26\nOnline\nBob\nExample\nBob_account\n23rd Feb 26\nOnline\nCarol\nExample\nCarol_account\n23rd Feb 26\nOnline\nDana\nExample\nDana_account\n23rd Feb 26\nOnline\nEmma\nExample\nEmma_account\n23rd Feb 26\nOnline\nFrank\nExample\nFrank_account\n23rd Feb 26\nOnline\nGrace\nExample\nGrace_account\n23rd Feb 26\nOnline\nAda\nExample\nAda_account\n23rd Feb 26\nOnline\nShow source", + "ok": true, + "expected_task_pass": false + }, + { + "id": "eui-complete-record-cannot-borrow-neighbor", + "category": "acceptance-contract", + "root_cause": "volatile_eui_people_and_invisible_reference", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/people", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 8, + "table_columns": [ + { + "name": "first_name", + "label": "First Name", + "type": "text" + }, + { + "name": "last_name", + "label": "Last Name", + "type": "text" + }, + { + "name": "github", + "label": "Github", + "type": "text" + }, + { + "name": "birth", + "label": "Date of Birth", + "type": "date_ordinal_en" + }, + { + "name": "online", + "label": "Online", + "type": "text", + "pattern": "(?i)Online|Offline" } + ], + "record_identity_fields": [ + "first_name", + "last_name", + "github" + ], + "table_scope": { + "root_selector": "table.euiTable", + "root_index": 0, + "row_selector": "tbody tr", + "cell_selector": ":scope > th, :scope > td", + "exclude_selector": "[class*=euiScreenReaderOnly],svg", + "start_pattern": "(?im)^(?=[ \\t|]*First Name(?:[ \\t|]+|\\n))", + "end_pattern": "(?im)^[ \\t#]*Show source[ \\t]*$", + "plain_row_pattern": "(?P(?!(?i:online|offline)(?:\\s|$))[^\\d\\s↦↵]+)\\s+(?P(?!(?i:online|offline)(?:\\s|$))[^\\d\\s↦↵]+)\\s+(?P[^\\s↦↵]+)\\s+(?P\\d{1,2}(?:st|nd|rd|th) [A-Za-z]{3} (?:\\d{4}|\\d{2}))\\s+(?P(?i:Online|Offline))", + "plain_multiline_row_pattern": "(?m)^(?P(?!(?i:online|offline)(?:\\s|$))[^\\d\\s↦↵]+)[ \\t]*(?:↦[ \\t]*)?\\n(?:[ \\t]*(?:↦)?[ \\t]*\\n)*[ \\t]*(?P(?!(?i:online|offline)(?:\\s|$))[^\\d\\s↦↵]+)[ \\t]*(?:↦[ \\t]*)?\\n(?:[ \\t]*(?:↦)?[ \\t]*\\n)*[ \\t]*(?P[^\\s↦↵]+)(?:\\n\\(external, opens in a new tab or window\\))?[ \\t]*(?:↦[ \\t]*)?\\n(?:[ \\t]*(?:↦)?[ \\t]*\\n)*[ \\t]*(?P\\d{1,2}(?:st|nd|rd|th) [A-Za-z]{3} (?:\\d{4}|\\d{2}))[ \\t]*(?:↦[ \\t]*)?\\n(?:[ \\t]*(?:↦)?[ \\t]*\\n)*[ \\t]*(?P(?i:Online|Offline))[ \\t]*(?:\\n↵)?(?=\\n|$)" + } }, - "artifact": "[条目0](//www.bilibili.com/video/BV1Ab4111111)\n[条目1](//www.bilibili.com/video/BV1Ab4222222)\n各个领域中新奇好玩的优质内容都在这里~\nbilibili", - "ok": true, - "expected_task_pass": false - }, - { - "id": "dynamic-052-bilibili-com-v-footer-scope-no-footer", - "category": "structured-extraction", - "root_cause": "footer_links_not_dynamic_list_records", - "failure_mode": "false-negative", - "case": { - "url": "https://www.bilibili.com/v/popular/all/", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 2, - "hosts": [ - "www.bilibili.com" - ], - "entry_path": "/video/(?:BV[A-Za-z0-9]+|av[0-9]+)", - "stable_text_anchors": [], - "entry_scope": { - "html_selector": ".video-card", - "title_selector": "p.video-name", - "link_selector": "a[href*=\"/video/\"]", - "start_pattern": "各个领域中新奇好玩的优质内容都在这里", - "end_pattern": "(?i)^[ \\t#*]*bilibili[ \\t*]*$" - }, - "name_validation": "nonempty" + "body_requirements": {} + }, + "artifact": "First Name\nLast Name\nGithub\nDate of Birth\nOnline\nAda\nExample\n23rd Feb 26\nOnline\nspare_account\nBob\nExample\nBob_account\n23rd Feb 26\nOnline\nCarol\nExample\nCarol_account\n23rd Feb 26\nOnline\nDana\nExample\nDana_account\n23rd Feb 26\nOnline\nEmma\nExample\nEmma_account\n23rd Feb 26\nOnline\nFrank\nExample\nFrank_account\n23rd Feb 26\nOnline\nGrace\nExample\nGrace_account\n23rd Feb 26\nOnline\nHelen\nExample\nHelen_account\n23rd Feb 26\nOnline\nShow source", + "ok": true, + "expected_task_pass": false + }, + { + "id": "eui-complete-record-markdown", + "category": "acceptance-contract", + "root_cause": "volatile_eui_people_and_invisible_reference", + "failure_mode": "false-negative", + "case": { + "url": "https://example.test/people", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 8, + "table_columns": [ + { + "name": "first_name", + "label": "First Name", + "type": "text" + }, + { + "name": "last_name", + "label": "Last Name", + "type": "text" + }, + { + "name": "github", + "label": "Github", + "type": "text" + }, + { + "name": "birth", + "label": "Date of Birth", + "type": "date_ordinal_en" + }, + { + "name": "online", + "label": "Online", + "type": "text", + "pattern": "(?i)Online|Offline" } + ], + "record_identity_fields": [ + "first_name", + "last_name", + "github" + ], + "table_scope": { + "root_selector": "table.euiTable", + "root_index": 0, + "row_selector": "tbody tr", + "cell_selector": ":scope > th, :scope > td", + "exclude_selector": "[class*=euiScreenReaderOnly],svg", + "start_pattern": "(?im)^(?=[ \\t|]*First Name(?:[ \\t|]+|\\n))", + "end_pattern": "(?im)^[ \\t#]*Show source[ \\t]*$", + "plain_row_pattern": "(?P(?!(?i:online|offline)(?:\\s|$))[^\\d\\s↦↵]+)\\s+(?P(?!(?i:online|offline)(?:\\s|$))[^\\d\\s↦↵]+)\\s+(?P[^\\s↦↵]+)\\s+(?P\\d{1,2}(?:st|nd|rd|th) [A-Za-z]{3} (?:\\d{4}|\\d{2}))\\s+(?P(?i:Online|Offline))", + "plain_multiline_row_pattern": "(?m)^(?P(?!(?i:online|offline)(?:\\s|$))[^\\d\\s↦↵]+)[ \\t]*(?:↦[ \\t]*)?\\n(?:[ \\t]*(?:↦)?[ \\t]*\\n)*[ \\t]*(?P(?!(?i:online|offline)(?:\\s|$))[^\\d\\s↦↵]+)[ \\t]*(?:↦[ \\t]*)?\\n(?:[ \\t]*(?:↦)?[ \\t]*\\n)*[ \\t]*(?P[^\\s↦↵]+)(?:\\n\\(external, opens in a new tab or window\\))?[ \\t]*(?:↦[ \\t]*)?\\n(?:[ \\t]*(?:↦)?[ \\t]*\\n)*[ \\t]*(?P\\d{1,2}(?:st|nd|rd|th) [A-Za-z]{3} (?:\\d{4}|\\d{2}))[ \\t]*(?:↦[ \\t]*)?\\n(?:[ \\t]*(?:↦)?[ \\t]*\\n)*[ \\t]*(?P(?i:Online|Offline))[ \\t]*(?:\\n↵)?(?=\\n|$)" + } }, - "artifact": "各个领域中新奇好玩的优质内容都在这里~\n[条目0](//www.bilibili.com/video/BV1Ab4111111)\n[条目1](//www.bilibili.com/video/BV1Ab4222222)", - "ok": true, - "expected_task_pass": true - }, - { - "id": "dynamic-063-s-weibo-com-top-footer-scope-inside", - "category": "structured-extraction", - "root_cause": "footer_links_not_dynamic_list_records", - "failure_mode": "false-negative", - "case": { - "url": "https://s.weibo.com/top/summary", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 2, - "hosts": [ - "s.weibo.com" - ], - "entry_path": "/weibo", - "stable_text_anchors": [], - "identity_query_keys": [ - "q" - ], - "entry_scope": { - "html_selector": "#pl_top_realtimehot tbody tr td.td-02 a[href]", - "start_pattern": "序号[^\\n]*关键词", - "end_pattern": "^[ \\t]*(?:\\[关于微博\\]\\(|关于微博[ \\t]+微博帮助)" - }, - "name_validation": "nonempty" + "body_requirements": {} + }, + "artifact": "| First Name | Last Name | Github | Date of Birth | Online |\n| --- | --- | --- | --- | --- |\n| Ada | Example | Ada_account | 23rd Feb 26 | Online |\n| Bob | Example | Bob_account | 23rd Feb 26 | Online |\n| Carol | Example | Carol_account | 23rd Feb 26 | Online |\n| Dana | Example | Dana_account | 23rd Feb 26 | Online |\n| Emma | Example | Emma_account | 23rd Feb 26 | Online |\n| Frank | Example | Frank_account | 23rd Feb 26 | Online |\n| Grace | Example | Grace_account | 23rd Feb 26 | Online |\n| Helen | Example | Helen_account | 23rd Feb 26 | Online |", + "ok": true, + "expected_task_pass": true + }, + { + "id": "eui-complete-record-html", + "category": "acceptance-contract", + "root_cause": "volatile_eui_people_and_invisible_reference", + "failure_mode": "false-negative", + "case": { + "url": "https://example.test/people", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 8, + "table_columns": [ + { + "name": "first_name", + "label": "First Name", + "type": "text" + }, + { + "name": "last_name", + "label": "Last Name", + "type": "text" + }, + { + "name": "github", + "label": "Github", + "type": "text" + }, + { + "name": "birth", + "label": "Date of Birth", + "type": "date_ordinal_en" + }, + { + "name": "online", + "label": "Online", + "type": "text", + "pattern": "(?i)Online|Offline" } + ], + "record_identity_fields": [ + "first_name", + "last_name", + "github" + ], + "table_scope": { + "root_selector": "table.euiTable", + "root_index": 0, + "row_selector": "tbody tr", + "cell_selector": ":scope > th, :scope > td", + "exclude_selector": "[class*=euiScreenReaderOnly],svg", + "start_pattern": "(?im)^(?=[ \\t|]*First Name(?:[ \\t|]+|\\n))", + "end_pattern": "(?im)^[ \\t#]*Show source[ \\t]*$", + "plain_row_pattern": "(?P(?!(?i:online|offline)(?:\\s|$))[^\\d\\s↦↵]+)\\s+(?P(?!(?i:online|offline)(?:\\s|$))[^\\d\\s↦↵]+)\\s+(?P[^\\s↦↵]+)\\s+(?P\\d{1,2}(?:st|nd|rd|th) [A-Za-z]{3} (?:\\d{4}|\\d{2}))\\s+(?P(?i:Online|Offline))", + "plain_multiline_row_pattern": "(?m)^(?P(?!(?i:online|offline)(?:\\s|$))[^\\d\\s↦↵]+)[ \\t]*(?:↦[ \\t]*)?\\n(?:[ \\t]*(?:↦)?[ \\t]*\\n)*[ \\t]*(?P(?!(?i:online|offline)(?:\\s|$))[^\\d\\s↦↵]+)[ \\t]*(?:↦[ \\t]*)?\\n(?:[ \\t]*(?:↦)?[ \\t]*\\n)*[ \\t]*(?P[^\\s↦↵]+)(?:\\n\\(external, opens in a new tab or window\\))?[ \\t]*(?:↦[ \\t]*)?\\n(?:[ \\t]*(?:↦)?[ \\t]*\\n)*[ \\t]*(?P\\d{1,2}(?:st|nd|rd|th) [A-Za-z]{3} (?:\\d{4}|\\d{2}))[ \\t]*(?:↦[ \\t]*)?\\n(?:[ \\t]*(?:↦)?[ \\t]*\\n)*[ \\t]*(?P(?i:Online|Offline))[ \\t]*(?:\\n↵)?(?=\\n|$)" + } }, - "artifact": "序号 关键词\n[条目0](https://s.weibo.com/weibo?q=newone)\n[条目1](https://s.weibo.com/weibo?q=newtwo)\n关于微博 微博帮助", - "ok": true, - "expected_task_pass": true - }, - { - "id": "dynamic-063-s-weibo-com-top-footer-scope-footer-only", - "category": "structured-extraction", - "root_cause": "footer_links_not_dynamic_list_records", - "failure_mode": "false-positive", - "case": { - "url": "https://s.weibo.com/top/summary", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 2, - "hosts": [ - "s.weibo.com" - ], - "entry_path": "/weibo", - "stable_text_anchors": [], - "identity_query_keys": [ - "q" - ], - "entry_scope": { - "html_selector": "#pl_top_realtimehot tbody tr td.td-02 a[href]", - "start_pattern": "序号[^\\n]*关键词", - "end_pattern": "^[ \\t]*(?:\\[关于微博\\]\\(|关于微博[ \\t]+微博帮助)" - }, - "name_validation": "nonempty" + "body_requirements": {} + }, + "artifact": "
AdaExampleAda_account23rd Feb 26Online
BobExampleBob_account23rd Feb 26Online
CarolExampleCarol_account23rd Feb 26Online
DanaExampleDana_account23rd Feb 26Online
EmmaExampleEmma_account23rd Feb 26Online
FrankExampleFrank_account23rd Feb 26Online
GraceExampleGrace_account23rd Feb 26Online
HelenExampleHelen_account23rd Feb 26Online
", + "ok": true, + "expected_task_pass": true + }, + { + "id": "eui-complete-record-later-table", + "category": "acceptance-contract", + "root_cause": "volatile_eui_people_and_invisible_reference", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/people", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 8, + "table_columns": [ + { + "name": "first_name", + "label": "First Name", + "type": "text" + }, + { + "name": "last_name", + "label": "Last Name", + "type": "text" + }, + { + "name": "github", + "label": "Github", + "type": "text" + }, + { + "name": "birth", + "label": "Date of Birth", + "type": "date_ordinal_en" + }, + { + "name": "online", + "label": "Online", + "type": "text", + "pattern": "(?i)Online|Offline" } + ], + "record_identity_fields": [ + "first_name", + "last_name", + "github" + ], + "table_scope": { + "root_selector": "table.euiTable", + "root_index": 0, + "row_selector": "tbody tr", + "cell_selector": ":scope > th, :scope > td", + "exclude_selector": "[class*=euiScreenReaderOnly],svg", + "start_pattern": "(?im)^(?=[ \\t|]*First Name(?:[ \\t|]+|\\n))", + "end_pattern": "(?im)^[ \\t#]*Show source[ \\t]*$", + "plain_row_pattern": "(?P(?!(?i:online|offline)(?:\\s|$))[^\\d\\s↦↵]+)\\s+(?P(?!(?i:online|offline)(?:\\s|$))[^\\d\\s↦↵]+)\\s+(?P[^\\s↦↵]+)\\s+(?P\\d{1,2}(?:st|nd|rd|th) [A-Za-z]{3} (?:\\d{4}|\\d{2}))\\s+(?P(?i:Online|Offline))", + "plain_multiline_row_pattern": "(?m)^(?P(?!(?i:online|offline)(?:\\s|$))[^\\d\\s↦↵]+)[ \\t]*(?:↦[ \\t]*)?\\n(?:[ \\t]*(?:↦)?[ \\t]*\\n)*[ \\t]*(?P(?!(?i:online|offline)(?:\\s|$))[^\\d\\s↦↵]+)[ \\t]*(?:↦[ \\t]*)?\\n(?:[ \\t]*(?:↦)?[ \\t]*\\n)*[ \\t]*(?P[^\\s↦↵]+)(?:\\n\\(external, opens in a new tab or window\\))?[ \\t]*(?:↦[ \\t]*)?\\n(?:[ \\t]*(?:↦)?[ \\t]*\\n)*[ \\t]*(?P\\d{1,2}(?:st|nd|rd|th) [A-Za-z]{3} (?:\\d{4}|\\d{2}))[ \\t]*(?:↦[ \\t]*)?\\n(?:[ \\t]*(?:↦)?[ \\t]*\\n)*[ \\t]*(?P(?i:Online|Offline))[ \\t]*(?:\\n↵)?(?=\\n|$)" + } }, - "artifact": "序号 关键词\n关于微博 微博帮助\n[条目0](https://s.weibo.com/weibo?q=newone)\n[条目1](https://s.weibo.com/weibo?q=newtwo)", - "ok": true, - "expected_task_pass": false - }, - { - "id": "dynamic-063-s-weibo-com-top-footer-scope-navigation-only", - "category": "structured-extraction", - "root_cause": "footer_links_not_dynamic_list_records", - "failure_mode": "false-positive", - "case": { - "url": "https://s.weibo.com/top/summary", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 2, - "hosts": [ - "s.weibo.com" - ], - "entry_path": "/weibo", - "stable_text_anchors": [], - "identity_query_keys": [ - "q" - ], - "entry_scope": { - "html_selector": "#pl_top_realtimehot tbody tr td.td-02 a[href]", - "start_pattern": "序号[^\\n]*关键词", - "end_pattern": "^[ \\t]*(?:\\[关于微博\\]\\(|关于微博[ \\t]+微博帮助)" - }, - "name_validation": "nonempty" + "body_requirements": {} + }, + "artifact": "
AdaExampleAda_account23rd Feb 26Online
BobExampleBob_account23rd Feb 26Online
CarolExampleCarol_account23rd Feb 26Online
DanaExampleDana_account23rd Feb 26Online
EmmaExampleEmma_account23rd Feb 26Online
FrankExampleFrank_account23rd Feb 26Online
GraceExampleGrace_account23rd Feb 26Online
HelenExampleHelen_account23rd Feb 26Online
IrisExampleIris_account23rd Feb 26Online
JamesExampleJames_account23rd Feb 26Online
", + "ok": true, + "expected_task_pass": false + }, + { + "id": "fixed-demo-product-complete", + "category": "acceptance-contract", + "root_cause": "fixed_demo_product_field_correspondence", + "failure_mode": "false-negative", + "case": { + "url": "https://example.test/fixed-products", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 7, + "table_columns": [ + { + "name": "product", + "label": "Product", + "type": "text" + }, + { + "name": "category", + "label": "Category", + "type": "text" + }, + { + "name": "price", + "label": "Price", + "type": "currency_usd" + }, + { + "name": "status", + "label": "Status", + "type": "text" } - }, - "artifact": "[条目0](https://s.weibo.com/weibo?q=newone)\n[条目1](https://s.weibo.com/weibo?q=newtwo)\n序号 关键词\n关于微博 微博帮助", - "ok": true, - "expected_task_pass": false - }, - { - "id": "dynamic-063-s-weibo-com-top-footer-scope-no-footer", - "category": "structured-extraction", - "root_cause": "footer_links_not_dynamic_list_records", - "failure_mode": "false-negative", - "case": { - "url": "https://s.weibo.com/top/summary", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 2, - "hosts": [ - "s.weibo.com" - ], - "entry_path": "/weibo", - "stable_text_anchors": [], - "identity_query_keys": [ - "q" - ], - "entry_scope": { - "html_selector": "#pl_top_realtimehot tbody tr td.td-02 a[href]", - "start_pattern": "序号[^\\n]*关键词", - "end_pattern": "^[ \\t]*(?:\\[关于微博\\]\\(|关于微博[ \\t]+微博帮助)" - }, - "name_validation": "nonempty" + ], + "record_identity_fields": [ + "product" + ], + "table_scope": { + "root_selector": ".doc-section-demo", + "root_index": 0, + "row_selector": "tbody tr", + "cell_selector": ":scope > td:not(:nth-child(4))", + "start_pattern": "(?im)^(?=[ \\t|]*Product[ \\t|]+Category)", + "end_pattern": "(?im)^[ \\t#]*(?:PREVIEW|Usage)[ \\t]*$", + "plain_row_pattern": "(?P[^\\n]+?\\s+[a-z0-9]{9})\\s+(?PAccessories|Fitness|Clothing)\\s+(?P\\$\\s*\\d+(?:\\.\\d{1,2})?)\\s+(?PIn Stock|Low Stock|Out of Stock)", + "plain_multiline_row_pattern": "(?m)^(?P[^\\n]+\\n[a-z0-9]{9})\\s*\\n[\\s]*(?PAccessories|Fitness|Clothing)[\\s]+(?P\\$\\s*\\d+(?:\\.\\d{1,2})?)[\\s]+(?PIn Stock|Low Stock|Out of Stock)(?=\\n|$)", + "reference_kind": "fixed_demo", + "reference_rows": [ + { + "product": "Bamboo Watch f230fh0g3", + "category": "Accessories", + "price": "$65", + "status": "In Stock" + }, + { + "product": "Black Watch nvklal433", + "category": "Accessories", + "price": "$72", + "status": "In Stock" + }, + { + "product": "Blue Band zz21cz3c1", + "category": "Fitness", + "price": "$79", + "status": "Low Stock" + }, + { + "product": "Blue T-Shirt 244wgerg2", + "category": "Clothing", + "price": "$29", + "status": "In Stock" + }, + { + "product": "Bracelet h456wer53", + "category": "Accessories", + "price": "$15", + "status": "In Stock" + }, + { + "product": "Brown Purse av2231fwg", + "category": "Accessories", + "price": "$120", + "status": "Out of Stock" + }, + { + "product": "Chakra Bracelet bib36pfvm", + "category": "Accessories", + "price": "$32", + "status": "Low Stock" + }, + { + "product": "Galaxy Earrings mbvjkgip5", + "category": "Accessories", + "price": "$34", + "status": "In Stock" + } + ], + "ignored_column_labels": [ + "Rating" + ] + } + } + }, + "artifact": "Product Category Price Rating Status\nBamboo Watch f230fh0g3 Accessories $65 In Stock\nBlack Watch nvklal433 Accessories $72 In Stock\nBlue Band zz21cz3c1 Fitness $79 Low Stock\nBlue T-Shirt 244wgerg2 Clothing $29 In Stock\nBracelet h456wer53 Accessories $15 In Stock\nBrown Purse av2231fwg Accessories $120 Out of Stock\nChakra Bracelet bib36pfvm Accessories $32 Low Stock\nGalaxy Earrings mbvjkgip5 Accessories $34 In Stock\nPREVIEW", + "ok": true, + "expected_task_pass": true + }, + { + "id": "fixed-demo-product-seven", + "category": "acceptance-contract", + "root_cause": "fixed_demo_product_field_correspondence", + "failure_mode": "false-negative", + "case": { + "url": "https://example.test/fixed-products", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 7, + "table_columns": [ + { + "name": "product", + "label": "Product", + "type": "text" + }, + { + "name": "category", + "label": "Category", + "type": "text" + }, + { + "name": "price", + "label": "Price", + "type": "currency_usd" + }, + { + "name": "status", + "label": "Status", + "type": "text" } - }, - "artifact": "序号 关键词\n[条目0](https://s.weibo.com/weibo?q=newone)\n[条目1](https://s.weibo.com/weibo?q=newtwo)", - "ok": true, - "expected_task_pass": true - }, - { - "id": "arabic-currency-visible-format-prefix", - "category": "format-normalization", - "root_cause": "arabic_currency_order_and_direction_marks", - "failure_mode": "false-negative", - "case": { - "url": "https://www.amazon.ae/s?k=%D9%87%D8%A7%D8%AA%D9%81&language=ar_AE", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 1, - "hosts": [ - "www.amazon.ae" - ], - "entry_path": "/(?:[^/]+/)*(?:dp|gp/aw/d)/[A-Z0-9]{10}(?:/ref=[^/?#]+)?", - "stable_text_anchors": [], - "page_patterns": [ - "(?s)(?:(?:درهم[\\s\\u061c\\u200e\\u200f\\u202a-\\u202e\\u2066-\\u2069]*[0-9]+(?:[.,][0-9]+)*|[0-9]+(?:[.,][0-9]+)*[\\s\\u061c\\u200e\\u200f\\u202a-\\u202e\\u2066-\\u2069]*درهم).*?){5}" - ], - "entry_identity_pattern": "/(?:[^/]+/)*(?:dp|gp/aw/d)/([A-Z0-9]{10})(?:/|$)", - "name_validation": "nonempty", - "entry_scope": { - "html_selector": "[data-component-type=\"s-search-result\"][data-asin]", - "title_selector": "a h2", - "link_selector": "a:has(h2)" - } + ], + "record_identity_fields": [ + "product" + ], + "table_scope": { + "root_selector": ".doc-section-demo", + "root_index": 0, + "row_selector": "tbody tr", + "cell_selector": ":scope > td:not(:nth-child(4))", + "start_pattern": "(?im)^(?=[ \\t|]*Product[ \\t|]+Category)", + "end_pattern": "(?im)^[ \\t#]*(?:PREVIEW|Usage)[ \\t]*$", + "plain_row_pattern": "(?P[^\\n]+?\\s+[a-z0-9]{9})\\s+(?PAccessories|Fitness|Clothing)\\s+(?P\\$\\s*\\d+(?:\\.\\d{1,2})?)\\s+(?PIn Stock|Low Stock|Out of Stock)", + "plain_multiline_row_pattern": "(?m)^(?P[^\\n]+\\n[a-z0-9]{9})\\s*\\n[\\s]*(?PAccessories|Fitness|Clothing)[\\s]+(?P\\$\\s*\\d+(?:\\.\\d{1,2})?)[\\s]+(?PIn Stock|Low Stock|Out of Stock)(?=\\n|$)", + "reference_kind": "fixed_demo", + "reference_rows": [ + { + "product": "Bamboo Watch f230fh0g3", + "category": "Accessories", + "price": "$65", + "status": "In Stock" + }, + { + "product": "Black Watch nvklal433", + "category": "Accessories", + "price": "$72", + "status": "In Stock" + }, + { + "product": "Blue Band zz21cz3c1", + "category": "Fitness", + "price": "$79", + "status": "Low Stock" + }, + { + "product": "Blue T-Shirt 244wgerg2", + "category": "Clothing", + "price": "$29", + "status": "In Stock" + }, + { + "product": "Bracelet h456wer53", + "category": "Accessories", + "price": "$15", + "status": "In Stock" + }, + { + "product": "Brown Purse av2231fwg", + "category": "Accessories", + "price": "$120", + "status": "Out of Stock" + }, + { + "product": "Chakra Bracelet bib36pfvm", + "category": "Accessories", + "price": "$32", + "status": "Low Stock" + }, + { + "product": "Galaxy Earrings mbvjkgip5", + "category": "Accessories", + "price": "$34", + "status": "In Stock" + } + ], + "ignored_column_labels": [ + "Rating" + ] + } + } + }, + "artifact": "Product Category Price Rating Status\nBamboo Watch f230fh0g3 Accessories $65 In Stock\nBlack Watch nvklal433 Accessories $72 In Stock\nBlue Band zz21cz3c1 Fitness $79 Low Stock\nBlue T-Shirt 244wgerg2 Clothing $29 In Stock\nBracelet h456wer53 Accessories $15 In Stock\nBrown Purse av2231fwg Accessories $120 Out of Stock\nChakra Bracelet bib36pfvm Accessories $32 Low Stock\nPREVIEW", + "ok": true, + "expected_task_pass": true + }, + { + "id": "fixed-demo-product-six", + "category": "acceptance-contract", + "root_cause": "fixed_demo_product_field_correspondence", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/fixed-products", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 7, + "table_columns": [ + { + "name": "product", + "label": "Product", + "type": "text" + }, + { + "name": "category", + "label": "Category", + "type": "text" + }, + { + "name": "price", + "label": "Price", + "type": "currency_usd" + }, + { + "name": "status", + "label": "Status", + "type": "text" } - }, - "artifact": "[Phone](https://www.amazon.ae/dp/B000000001)\nدرهم 659.99\nدرهم 659.99\nدرهم 659.99\nدرهم 659.99\nدرهم 659.99\n", - "ok": true, - "expected_task_pass": true - }, - { - "id": "arabic-currency-visible-format-suffix", - "category": "format-normalization", - "root_cause": "arabic_currency_order_and_direction_marks", - "failure_mode": "false-negative", - "case": { - "url": "https://www.amazon.ae/s?k=%D9%87%D8%A7%D8%AA%D9%81&language=ar_AE", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 1, - "hosts": [ - "www.amazon.ae" - ], - "entry_path": "/(?:[^/]+/)*(?:dp|gp/aw/d)/[A-Z0-9]{10}(?:/ref=[^/?#]+)?", - "stable_text_anchors": [], - "page_patterns": [ - "(?s)(?:(?:درهم[\\s\\u061c\\u200e\\u200f\\u202a-\\u202e\\u2066-\\u2069]*[0-9]+(?:[.,][0-9]+)*|[0-9]+(?:[.,][0-9]+)*[\\s\\u061c\\u200e\\u200f\\u202a-\\u202e\\u2066-\\u2069]*درهم).*?){5}" - ], - "entry_identity_pattern": "/(?:[^/]+/)*(?:dp|gp/aw/d)/([A-Z0-9]{10})(?:/|$)", - "name_validation": "nonempty", - "entry_scope": { - "html_selector": "[data-component-type=\"s-search-result\"][data-asin]", - "title_selector": "a h2", - "link_selector": "a:has(h2)" - } + ], + "record_identity_fields": [ + "product" + ], + "table_scope": { + "root_selector": ".doc-section-demo", + "root_index": 0, + "row_selector": "tbody tr", + "cell_selector": ":scope > td:not(:nth-child(4))", + "start_pattern": "(?im)^(?=[ \\t|]*Product[ \\t|]+Category)", + "end_pattern": "(?im)^[ \\t#]*(?:PREVIEW|Usage)[ \\t]*$", + "plain_row_pattern": "(?P[^\\n]+?\\s+[a-z0-9]{9})\\s+(?PAccessories|Fitness|Clothing)\\s+(?P\\$\\s*\\d+(?:\\.\\d{1,2})?)\\s+(?PIn Stock|Low Stock|Out of Stock)", + "plain_multiline_row_pattern": "(?m)^(?P[^\\n]+\\n[a-z0-9]{9})\\s*\\n[\\s]*(?PAccessories|Fitness|Clothing)[\\s]+(?P\\$\\s*\\d+(?:\\.\\d{1,2})?)[\\s]+(?PIn Stock|Low Stock|Out of Stock)(?=\\n|$)", + "reference_kind": "fixed_demo", + "reference_rows": [ + { + "product": "Bamboo Watch f230fh0g3", + "category": "Accessories", + "price": "$65", + "status": "In Stock" + }, + { + "product": "Black Watch nvklal433", + "category": "Accessories", + "price": "$72", + "status": "In Stock" + }, + { + "product": "Blue Band zz21cz3c1", + "category": "Fitness", + "price": "$79", + "status": "Low Stock" + }, + { + "product": "Blue T-Shirt 244wgerg2", + "category": "Clothing", + "price": "$29", + "status": "In Stock" + }, + { + "product": "Bracelet h456wer53", + "category": "Accessories", + "price": "$15", + "status": "In Stock" + }, + { + "product": "Brown Purse av2231fwg", + "category": "Accessories", + "price": "$120", + "status": "Out of Stock" + }, + { + "product": "Chakra Bracelet bib36pfvm", + "category": "Accessories", + "price": "$32", + "status": "Low Stock" + }, + { + "product": "Galaxy Earrings mbvjkgip5", + "category": "Accessories", + "price": "$34", + "status": "In Stock" + } + ], + "ignored_column_labels": [ + "Rating" + ] + } + } + }, + "artifact": "Product Category Price Rating Status\nBamboo Watch f230fh0g3 Accessories $65 In Stock\nBlack Watch nvklal433 Accessories $72 In Stock\nBlue Band zz21cz3c1 Fitness $79 Low Stock\nBlue T-Shirt 244wgerg2 Clothing $29 In Stock\nBracelet h456wer53 Accessories $15 In Stock\nBrown Purse av2231fwg Accessories $120 Out of Stock\nPREVIEW", + "ok": true, + "expected_task_pass": false + }, + { + "id": "fixed-demo-product-missing-prices", + "category": "acceptance-contract", + "root_cause": "fixed_demo_product_field_correspondence", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/fixed-products", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 7, + "table_columns": [ + { + "name": "product", + "label": "Product", + "type": "text" + }, + { + "name": "category", + "label": "Category", + "type": "text" + }, + { + "name": "price", + "label": "Price", + "type": "currency_usd" + }, + { + "name": "status", + "label": "Status", + "type": "text" } - }, - "artifact": "[Phone](https://www.amazon.ae/dp/B000000001)\n659.99 درهم\n659.99 درهم\n659.99 درهم\n659.99 درهم\n659.99 درهم\n", - "ok": true, - "expected_task_pass": true - }, - { - "id": "arabic-currency-visible-format-direction-marks", - "category": "format-normalization", - "root_cause": "arabic_currency_order_and_direction_marks", - "failure_mode": "false-negative", - "case": { - "url": "https://www.amazon.ae/s?k=%D9%87%D8%A7%D8%AA%D9%81&language=ar_AE", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 1, - "hosts": [ - "www.amazon.ae" - ], - "entry_path": "/(?:[^/]+/)*(?:dp|gp/aw/d)/[A-Z0-9]{10}(?:/ref=[^/?#]+)?", - "stable_text_anchors": [], - "page_patterns": [ - "(?s)(?:(?:درهم[\\s\\u061c\\u200e\\u200f\\u202a-\\u202e\\u2066-\\u2069]*[0-9]+(?:[.,][0-9]+)*|[0-9]+(?:[.,][0-9]+)*[\\s\\u061c\\u200e\\u200f\\u202a-\\u202e\\u2066-\\u2069]*درهم).*?){5}" - ], - "entry_identity_pattern": "/(?:[^/]+/)*(?:dp|gp/aw/d)/([A-Z0-9]{10})(?:/|$)", - "name_validation": "nonempty", - "entry_scope": { - "html_selector": "[data-component-type=\"s-search-result\"][data-asin]", - "title_selector": "a h2", - "link_selector": "a:has(h2)" - } + ], + "record_identity_fields": [ + "product" + ], + "table_scope": { + "root_selector": ".doc-section-demo", + "root_index": 0, + "row_selector": "tbody tr", + "cell_selector": ":scope > td:not(:nth-child(4))", + "start_pattern": "(?im)^(?=[ \\t|]*Product[ \\t|]+Category)", + "end_pattern": "(?im)^[ \\t#]*(?:PREVIEW|Usage)[ \\t]*$", + "plain_row_pattern": "(?P[^\\n]+?\\s+[a-z0-9]{9})\\s+(?PAccessories|Fitness|Clothing)\\s+(?P\\$\\s*\\d+(?:\\.\\d{1,2})?)\\s+(?PIn Stock|Low Stock|Out of Stock)", + "plain_multiline_row_pattern": "(?m)^(?P[^\\n]+\\n[a-z0-9]{9})\\s*\\n[\\s]*(?PAccessories|Fitness|Clothing)[\\s]+(?P\\$\\s*\\d+(?:\\.\\d{1,2})?)[\\s]+(?PIn Stock|Low Stock|Out of Stock)(?=\\n|$)", + "reference_kind": "fixed_demo", + "reference_rows": [ + { + "product": "Bamboo Watch f230fh0g3", + "category": "Accessories", + "price": "$65", + "status": "In Stock" + }, + { + "product": "Black Watch nvklal433", + "category": "Accessories", + "price": "$72", + "status": "In Stock" + }, + { + "product": "Blue Band zz21cz3c1", + "category": "Fitness", + "price": "$79", + "status": "Low Stock" + }, + { + "product": "Blue T-Shirt 244wgerg2", + "category": "Clothing", + "price": "$29", + "status": "In Stock" + }, + { + "product": "Bracelet h456wer53", + "category": "Accessories", + "price": "$15", + "status": "In Stock" + }, + { + "product": "Brown Purse av2231fwg", + "category": "Accessories", + "price": "$120", + "status": "Out of Stock" + }, + { + "product": "Chakra Bracelet bib36pfvm", + "category": "Accessories", + "price": "$32", + "status": "Low Stock" + }, + { + "product": "Galaxy Earrings mbvjkgip5", + "category": "Accessories", + "price": "$34", + "status": "In Stock" + } + ], + "ignored_column_labels": [ + "Rating" + ] + } + } + }, + "artifact": "Product Category Price Rating Status\nBamboo Watch f230fh0g3 Accessories In Stock\nBlack Watch nvklal433 Accessories In Stock\nBlue Band zz21cz3c1 Fitness Low Stock\nBlue T-Shirt 244wgerg2 Clothing In Stock\nBracelet h456wer53 Accessories In Stock\nBrown Purse av2231fwg Accessories Out of Stock\nChakra Bracelet bib36pfvm Accessories Low Stock\nGalaxy Earrings mbvjkgip5 Accessories In Stock\nPREVIEW", + "ok": true, + "expected_task_pass": false + }, + { + "id": "fixed-demo-product-wrong-prices", + "category": "acceptance-contract", + "root_cause": "fixed_demo_product_field_correspondence", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/fixed-products", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 7, + "table_columns": [ + { + "name": "product", + "label": "Product", + "type": "text" + }, + { + "name": "category", + "label": "Category", + "type": "text" + }, + { + "name": "price", + "label": "Price", + "type": "currency_usd" + }, + { + "name": "status", + "label": "Status", + "type": "text" } - }, - "artifact": "[Phone](https://www.amazon.ae/dp/B000000001)\n‏659.99 درهم‎\n‏659.99 درهم‎\n‏659.99 درهم‎\n‏659.99 درهم‎\n‏659.99 درهم‎\n", - "ok": true, - "expected_task_pass": true - }, - { - "id": "arabic-currency-visible-format-prefix-direction", - "category": "format-normalization", - "root_cause": "arabic_currency_order_and_direction_marks", - "failure_mode": "false-negative", - "case": { - "url": "https://www.amazon.ae/s?k=%D9%87%D8%A7%D8%AA%D9%81&language=ar_AE", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 1, - "hosts": [ - "www.amazon.ae" - ], - "entry_path": "/(?:[^/]+/)*(?:dp|gp/aw/d)/[A-Z0-9]{10}(?:/ref=[^/?#]+)?", - "stable_text_anchors": [], - "page_patterns": [ - "(?s)(?:(?:درهم[\\s\\u061c\\u200e\\u200f\\u202a-\\u202e\\u2066-\\u2069]*[0-9]+(?:[.,][0-9]+)*|[0-9]+(?:[.,][0-9]+)*[\\s\\u061c\\u200e\\u200f\\u202a-\\u202e\\u2066-\\u2069]*درهم).*?){5}" - ], - "entry_identity_pattern": "/(?:[^/]+/)*(?:dp|gp/aw/d)/([A-Z0-9]{10})(?:/|$)", - "name_validation": "nonempty", - "entry_scope": { - "html_selector": "[data-component-type=\"s-search-result\"][data-asin]", - "title_selector": "a h2", - "link_selector": "a:has(h2)" - } + ], + "record_identity_fields": [ + "product" + ], + "table_scope": { + "root_selector": ".doc-section-demo", + "root_index": 0, + "row_selector": "tbody tr", + "cell_selector": ":scope > td:not(:nth-child(4))", + "start_pattern": "(?im)^(?=[ \\t|]*Product[ \\t|]+Category)", + "end_pattern": "(?im)^[ \\t#]*(?:PREVIEW|Usage)[ \\t]*$", + "plain_row_pattern": "(?P[^\\n]+?\\s+[a-z0-9]{9})\\s+(?PAccessories|Fitness|Clothing)\\s+(?P\\$\\s*\\d+(?:\\.\\d{1,2})?)\\s+(?PIn Stock|Low Stock|Out of Stock)", + "plain_multiline_row_pattern": "(?m)^(?P[^\\n]+\\n[a-z0-9]{9})\\s*\\n[\\s]*(?PAccessories|Fitness|Clothing)[\\s]+(?P\\$\\s*\\d+(?:\\.\\d{1,2})?)[\\s]+(?PIn Stock|Low Stock|Out of Stock)(?=\\n|$)", + "reference_kind": "fixed_demo", + "reference_rows": [ + { + "product": "Bamboo Watch f230fh0g3", + "category": "Accessories", + "price": "$65", + "status": "In Stock" + }, + { + "product": "Black Watch nvklal433", + "category": "Accessories", + "price": "$72", + "status": "In Stock" + }, + { + "product": "Blue Band zz21cz3c1", + "category": "Fitness", + "price": "$79", + "status": "Low Stock" + }, + { + "product": "Blue T-Shirt 244wgerg2", + "category": "Clothing", + "price": "$29", + "status": "In Stock" + }, + { + "product": "Bracelet h456wer53", + "category": "Accessories", + "price": "$15", + "status": "In Stock" + }, + { + "product": "Brown Purse av2231fwg", + "category": "Accessories", + "price": "$120", + "status": "Out of Stock" + }, + { + "product": "Chakra Bracelet bib36pfvm", + "category": "Accessories", + "price": "$32", + "status": "Low Stock" + }, + { + "product": "Galaxy Earrings mbvjkgip5", + "category": "Accessories", + "price": "$34", + "status": "In Stock" + } + ], + "ignored_column_labels": [ + "Rating" + ] + } + } + }, + "artifact": "Product Category Price Rating Status\nBamboo Watch f230fh0g3 Accessories $99999 In Stock\nBlack Watch nvklal433 Accessories $99999 In Stock\nBlue Band zz21cz3c1 Fitness $99999 Low Stock\nBlue T-Shirt 244wgerg2 Clothing $99999 In Stock\nBracelet h456wer53 Accessories $99999 In Stock\nBrown Purse av2231fwg Accessories $99999 Out of Stock\nChakra Bracelet bib36pfvm Accessories $99999 Low Stock\nGalaxy Earrings mbvjkgip5 Accessories $99999 In Stock\nPREVIEW", + "ok": true, + "expected_task_pass": false + }, + { + "id": "fixed-demo-product-wrong-statuses", + "category": "acceptance-contract", + "root_cause": "fixed_demo_product_field_correspondence", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/fixed-products", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 7, + "table_columns": [ + { + "name": "product", + "label": "Product", + "type": "text" + }, + { + "name": "category", + "label": "Category", + "type": "text" + }, + { + "name": "price", + "label": "Price", + "type": "currency_usd" + }, + { + "name": "status", + "label": "Status", + "type": "text" } - }, - "artifact": "[Phone](https://www.amazon.ae/dp/B000000001)\nدرهم‎659.99‏\nدرهم‎659.99‏\nدرهم‎659.99‏\nدرهم‎659.99‏\nدرهم‎659.99‏\n", - "ok": true, - "expected_task_pass": true - }, - { - "id": "arabic-currency-visible-format-four-only", - "category": "format-normalization", - "root_cause": "arabic_currency_order_and_direction_marks", - "failure_mode": "false-positive", - "case": { - "url": "https://www.amazon.ae/s?k=%D9%87%D8%A7%D8%AA%D9%81&language=ar_AE", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 1, - "hosts": [ - "www.amazon.ae" - ], - "entry_path": "/(?:[^/]+/)*(?:dp|gp/aw/d)/[A-Z0-9]{10}(?:/ref=[^/?#]+)?", - "stable_text_anchors": [], - "page_patterns": [ - "(?s)(?:(?:درهم[\\s\\u061c\\u200e\\u200f\\u202a-\\u202e\\u2066-\\u2069]*[0-9]+(?:[.,][0-9]+)*|[0-9]+(?:[.,][0-9]+)*[\\s\\u061c\\u200e\\u200f\\u202a-\\u202e\\u2066-\\u2069]*درهم).*?){5}" - ], - "entry_identity_pattern": "/(?:[^/]+/)*(?:dp|gp/aw/d)/([A-Z0-9]{10})(?:/|$)", - "name_validation": "nonempty", - "entry_scope": { - "html_selector": "[data-component-type=\"s-search-result\"][data-asin]", - "title_selector": "a h2", - "link_selector": "a:has(h2)" - } + ], + "record_identity_fields": [ + "product" + ], + "table_scope": { + "root_selector": ".doc-section-demo", + "root_index": 0, + "row_selector": "tbody tr", + "cell_selector": ":scope > td:not(:nth-child(4))", + "start_pattern": "(?im)^(?=[ \\t|]*Product[ \\t|]+Category)", + "end_pattern": "(?im)^[ \\t#]*(?:PREVIEW|Usage)[ \\t]*$", + "plain_row_pattern": "(?P[^\\n]+?\\s+[a-z0-9]{9})\\s+(?PAccessories|Fitness|Clothing)\\s+(?P\\$\\s*\\d+(?:\\.\\d{1,2})?)\\s+(?PIn Stock|Low Stock|Out of Stock)", + "plain_multiline_row_pattern": "(?m)^(?P[^\\n]+\\n[a-z0-9]{9})\\s*\\n[\\s]*(?PAccessories|Fitness|Clothing)[\\s]+(?P\\$\\s*\\d+(?:\\.\\d{1,2})?)[\\s]+(?PIn Stock|Low Stock|Out of Stock)(?=\\n|$)", + "reference_kind": "fixed_demo", + "reference_rows": [ + { + "product": "Bamboo Watch f230fh0g3", + "category": "Accessories", + "price": "$65", + "status": "In Stock" + }, + { + "product": "Black Watch nvklal433", + "category": "Accessories", + "price": "$72", + "status": "In Stock" + }, + { + "product": "Blue Band zz21cz3c1", + "category": "Fitness", + "price": "$79", + "status": "Low Stock" + }, + { + "product": "Blue T-Shirt 244wgerg2", + "category": "Clothing", + "price": "$29", + "status": "In Stock" + }, + { + "product": "Bracelet h456wer53", + "category": "Accessories", + "price": "$15", + "status": "In Stock" + }, + { + "product": "Brown Purse av2231fwg", + "category": "Accessories", + "price": "$120", + "status": "Out of Stock" + }, + { + "product": "Chakra Bracelet bib36pfvm", + "category": "Accessories", + "price": "$32", + "status": "Low Stock" + }, + { + "product": "Galaxy Earrings mbvjkgip5", + "category": "Accessories", + "price": "$34", + "status": "In Stock" + } + ], + "ignored_column_labels": [ + "Rating" + ] + } + } + }, + "artifact": "Product Category Price Rating Status\nBamboo Watch f230fh0g3 Accessories $65 In Stock\nBlack Watch nvklal433 Accessories $72 In Stock\nBlue Band zz21cz3c1 Fitness $79 In Stock\nBlue T-Shirt 244wgerg2 Clothing $29 In Stock\nBracelet h456wer53 Accessories $15 In Stock\nBrown Purse av2231fwg Accessories $120 In Stock\nChakra Bracelet bib36pfvm Accessories $32 In Stock\nGalaxy Earrings mbvjkgip5 Accessories $34 In Stock\nPREVIEW", + "ok": true, + "expected_task_pass": false + }, + { + "id": "fixed-demo-product-equivalent-currency", + "category": "acceptance-contract", + "root_cause": "fixed_demo_product_field_correspondence", + "failure_mode": "false-negative", + "case": { + "url": "https://example.test/fixed-products", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 7, + "table_columns": [ + { + "name": "product", + "label": "Product", + "type": "text" + }, + { + "name": "category", + "label": "Category", + "type": "text" + }, + { + "name": "price", + "label": "Price", + "type": "currency_usd" + }, + { + "name": "status", + "label": "Status", + "type": "text" } - }, - "artifact": "[Phone](https://www.amazon.ae/dp/B000000001)\n659.99 درهم\n659.99 درهم\n659.99 درهم\n659.99 درهم\n", - "ok": true, - "expected_task_pass": false - }, - { - "id": "arabic-currency-visible-format-unrelated-digits", - "category": "format-normalization", - "root_cause": "arabic_currency_order_and_direction_marks", - "failure_mode": "false-positive", - "case": { - "url": "https://www.amazon.ae/s?k=%D9%87%D8%A7%D8%AA%D9%81&language=ar_AE", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 1, - "hosts": [ - "www.amazon.ae" - ], - "entry_path": "/(?:[^/]+/)*(?:dp|gp/aw/d)/[A-Z0-9]{10}(?:/ref=[^/?#]+)?", - "stable_text_anchors": [], - "page_patterns": [ - "(?s)(?:(?:درهم[\\s\\u061c\\u200e\\u200f\\u202a-\\u202e\\u2066-\\u2069]*[0-9]+(?:[.,][0-9]+)*|[0-9]+(?:[.,][0-9]+)*[\\s\\u061c\\u200e\\u200f\\u202a-\\u202e\\u2066-\\u2069]*درهم).*?){5}" - ], - "entry_identity_pattern": "/(?:[^/]+/)*(?:dp|gp/aw/d)/([A-Z0-9]{10})(?:/|$)", - "name_validation": "nonempty", - "entry_scope": { - "html_selector": "[data-component-type=\"s-search-result\"][data-asin]", - "title_selector": "a h2", - "link_selector": "a:has(h2)" - } + ], + "record_identity_fields": [ + "product" + ], + "table_scope": { + "root_selector": ".doc-section-demo", + "root_index": 0, + "row_selector": "tbody tr", + "cell_selector": ":scope > td:not(:nth-child(4))", + "start_pattern": "(?im)^(?=[ \\t|]*Product[ \\t|]+Category)", + "end_pattern": "(?im)^[ \\t#]*(?:PREVIEW|Usage)[ \\t]*$", + "plain_row_pattern": "(?P[^\\n]+?\\s+[a-z0-9]{9})\\s+(?PAccessories|Fitness|Clothing)\\s+(?P\\$\\s*\\d+(?:\\.\\d{1,2})?)\\s+(?PIn Stock|Low Stock|Out of Stock)", + "plain_multiline_row_pattern": "(?m)^(?P[^\\n]+\\n[a-z0-9]{9})\\s*\\n[\\s]*(?PAccessories|Fitness|Clothing)[\\s]+(?P\\$\\s*\\d+(?:\\.\\d{1,2})?)[\\s]+(?PIn Stock|Low Stock|Out of Stock)(?=\\n|$)", + "reference_kind": "fixed_demo", + "reference_rows": [ + { + "product": "Bamboo Watch f230fh0g3", + "category": "Accessories", + "price": "$65", + "status": "In Stock" + }, + { + "product": "Black Watch nvklal433", + "category": "Accessories", + "price": "$72", + "status": "In Stock" + }, + { + "product": "Blue Band zz21cz3c1", + "category": "Fitness", + "price": "$79", + "status": "Low Stock" + }, + { + "product": "Blue T-Shirt 244wgerg2", + "category": "Clothing", + "price": "$29", + "status": "In Stock" + }, + { + "product": "Bracelet h456wer53", + "category": "Accessories", + "price": "$15", + "status": "In Stock" + }, + { + "product": "Brown Purse av2231fwg", + "category": "Accessories", + "price": "$120", + "status": "Out of Stock" + }, + { + "product": "Chakra Bracelet bib36pfvm", + "category": "Accessories", + "price": "$32", + "status": "Low Stock" + }, + { + "product": "Galaxy Earrings mbvjkgip5", + "category": "Accessories", + "price": "$34", + "status": "In Stock" + } + ], + "ignored_column_labels": [ + "Rating" + ] + } + } + }, + "artifact": "Product Category Price Rating Status\nBamboo Watch f230fh0g3 Accessories $65.00 In Stock\nBlack Watch nvklal433 Accessories $72.00 In Stock\nBlue Band zz21cz3c1 Fitness $79.00 Low Stock\nBlue T-Shirt 244wgerg2 Clothing $29.00 In Stock\nBracelet h456wer53 Accessories $15.00 In Stock\nBrown Purse av2231fwg Accessories $120.00 Out of Stock\nChakra Bracelet bib36pfvm Accessories $32.00 Low Stock\nGalaxy Earrings mbvjkgip5 Accessories $34.00 In Stock\nPREVIEW", + "ok": true, + "expected_task_pass": true + }, + { + "id": "fixed-demo-product-markdown-stars", + "category": "acceptance-contract", + "root_cause": "fixed_demo_product_field_correspondence", + "failure_mode": "false-negative", + "case": { + "url": "https://example.test/fixed-products", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 7, + "table_columns": [ + { + "name": "product", + "label": "Product", + "type": "text" + }, + { + "name": "category", + "label": "Category", + "type": "text" + }, + { + "name": "price", + "label": "Price", + "type": "currency_usd" + }, + { + "name": "status", + "label": "Status", + "type": "text" } - }, - "artifact": "[Phone](https://www.amazon.ae/dp/B000000001)\nدرهم other text 659.99 unrelated\nدرهم other text 659.99 unrelated\nدرهم other text 659.99 unrelated\nدرهم other text 659.99 unrelated\nدرهم other text 659.99 unrelated\n", - "ok": true, - "expected_task_pass": false - }, - { - "id": "arabic-currency-visible-format-currency-only", - "category": "format-normalization", - "root_cause": "arabic_currency_order_and_direction_marks", - "failure_mode": "false-positive", - "case": { - "url": "https://www.amazon.ae/s?k=%D9%87%D8%A7%D8%AA%D9%81&language=ar_AE", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 1, - "hosts": [ - "www.amazon.ae" - ], - "entry_path": "/(?:[^/]+/)*(?:dp|gp/aw/d)/[A-Z0-9]{10}(?:/ref=[^/?#]+)?", - "stable_text_anchors": [], - "page_patterns": [ - "(?s)(?:(?:درهم[\\s\\u061c\\u200e\\u200f\\u202a-\\u202e\\u2066-\\u2069]*[0-9]+(?:[.,][0-9]+)*|[0-9]+(?:[.,][0-9]+)*[\\s\\u061c\\u200e\\u200f\\u202a-\\u202e\\u2066-\\u2069]*درهم).*?){5}" - ], - "entry_identity_pattern": "/(?:[^/]+/)*(?:dp|gp/aw/d)/([A-Z0-9]{10})(?:/|$)", - "name_validation": "nonempty", - "entry_scope": { - "html_selector": "[data-component-type=\"s-search-result\"][data-asin]", - "title_selector": "a h2", - "link_selector": "a:has(h2)" - } + ], + "record_identity_fields": [ + "product" + ], + "table_scope": { + "root_selector": ".doc-section-demo", + "root_index": 0, + "row_selector": "tbody tr", + "cell_selector": ":scope > td:not(:nth-child(4))", + "start_pattern": "(?im)^(?=[ \\t|]*Product[ \\t|]+Category)", + "end_pattern": "(?im)^[ \\t#]*(?:PREVIEW|Usage)[ \\t]*$", + "plain_row_pattern": "(?P[^\\n]+?\\s+[a-z0-9]{9})\\s+(?PAccessories|Fitness|Clothing)\\s+(?P\\$\\s*\\d+(?:\\.\\d{1,2})?)\\s+(?PIn Stock|Low Stock|Out of Stock)", + "plain_multiline_row_pattern": "(?m)^(?P[^\\n]+\\n[a-z0-9]{9})\\s*\\n[\\s]*(?PAccessories|Fitness|Clothing)[\\s]+(?P\\$\\s*\\d+(?:\\.\\d{1,2})?)[\\s]+(?PIn Stock|Low Stock|Out of Stock)(?=\\n|$)", + "reference_kind": "fixed_demo", + "reference_rows": [ + { + "product": "Bamboo Watch f230fh0g3", + "category": "Accessories", + "price": "$65", + "status": "In Stock" + }, + { + "product": "Black Watch nvklal433", + "category": "Accessories", + "price": "$72", + "status": "In Stock" + }, + { + "product": "Blue Band zz21cz3c1", + "category": "Fitness", + "price": "$79", + "status": "Low Stock" + }, + { + "product": "Blue T-Shirt 244wgerg2", + "category": "Clothing", + "price": "$29", + "status": "In Stock" + }, + { + "product": "Bracelet h456wer53", + "category": "Accessories", + "price": "$15", + "status": "In Stock" + }, + { + "product": "Brown Purse av2231fwg", + "category": "Accessories", + "price": "$120", + "status": "Out of Stock" + }, + { + "product": "Chakra Bracelet bib36pfvm", + "category": "Accessories", + "price": "$32", + "status": "Low Stock" + }, + { + "product": "Galaxy Earrings mbvjkgip5", + "category": "Accessories", + "price": "$34", + "status": "In Stock" + } + ], + "ignored_column_labels": [ + "Rating" + ] + } + } + }, + "artifact": "| Product | Category | Price | Rating | Status |\n| --- | --- | --- | --- | --- |\n| Bamboo Watch f230fh0g3 | Accessories | $65 | ★★★ | In Stock |\n| Black Watch nvklal433 | Accessories | $72 | ★★★ | In Stock |\n| Blue Band zz21cz3c1 | Fitness | $79 | ★★★ | Low Stock |\n| Blue T-Shirt 244wgerg2 | Clothing | $29 | ★★★ | In Stock |\n| Bracelet h456wer53 | Accessories | $15 | ★★★ | In Stock |\n| Brown Purse av2231fwg | Accessories | $120 | ★★★ | Out of Stock |\n| Chakra Bracelet bib36pfvm | Accessories | $32 | ★★★ | Low Stock |\n| Galaxy Earrings mbvjkgip5 | Accessories | $34 | ★★★ | In Stock |", + "ok": true, + "expected_task_pass": true + }, + { + "id": "fixed-demo-product-duplicates", + "category": "acceptance-contract", + "root_cause": "fixed_demo_product_field_correspondence", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/fixed-products", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 7, + "table_columns": [ + { + "name": "product", + "label": "Product", + "type": "text" + }, + { + "name": "category", + "label": "Category", + "type": "text" + }, + { + "name": "price", + "label": "Price", + "type": "currency_usd" + }, + { + "name": "status", + "label": "Status", + "type": "text" } - }, - "artifact": "[Phone](https://www.amazon.ae/dp/B000000001)\nدرهم\nدرهم\nدرهم\nدرهم\nدرهم\n", - "ok": true, - "expected_task_pass": false - }, - { - "id": "dynamic-014-en-stable-rsuite-table-blank-lines-complete", - "category": "structured-extraction", - "root_cause": "table_multiline_whitespace_and_scope_boundaries", - "failure_mode": "false-negative", - "case": { - "url": "https://rsuitejs.com/components/table/", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "table_row", - "minimum_records": 2, - "stable_text_anchors": [], - "table_columns": [ - { - "name": "id", - "label": "Id", - "type": "integer", - "minimum": 1 - }, - { - "name": "first_name", - "label": "First Name", - "type": "text" - }, - { - "name": "last_name", - "label": "Last Name", - "type": "text" - }, - { - "name": "gender", - "label": "Gender", - "type": "text", - "pattern": "(?i)female|male" - }, - { - "name": "age", - "label": "Age", - "type": "integer", - "minimum": 0, - "maximum": 150 - }, - { - "name": "postcode", - "label": "Postcode", - "type": "token", - "pattern": "\\d{5}(?:-\\d{4})?" - }, - { - "name": "email", - "label": "Email", - "type": "text", - "pattern": "[^\\s@]+@[^\\s@]+\\.[^\\s@]+" - } - ], - "record_identity_fields": [ - "id" - ], - "table_scope": { - "root_selector": ".rs-table", - "root_index": 0, - "row_selector": "[role=row]:not(.rs-table-row-header)", - "cell_selector": "[role=gridcell]:not([aria-colindex=\"8\"]) .rs-table-cell-content", - "start_pattern": "(?im)^(?=[ \\t|]*Id(?:[ \\t|]+|\\n[ \\t\\n]*)First Name)", - "end_pattern": "(?im)^[ \\t#]*(?:Appearance|Loading\\.\\.\\.)[ \\t]*(?:\\[.*)?$", - "plain_row_pattern": "(?P\\d+)\\s+(?P[^\\d\\n]+?)\\s+(?P[^\\d\\n]+?)\\s+(?P(?i:female|male))\\s+(?P\\d+)\\s+(?P\\d{5}(?:-\\d{4})?)\\s+(?P[^\\s@]+@[^\\s@]+\\.[^\\s@]+)(?:\\s+Edit)?", - "plain_multiline_row_pattern": "(?m)^(?P\\d+)\\n[ \\t\\n]*(?P[^\\d\\n]+)\\n[ \\t\\n]*(?P[^\\d\\n]+)\\n[ \\t\\n]*(?P(?i:female|male))\\n[ \\t\\n]*(?P\\d+)\\n[ \\t\\n]*(?P\\d{5}(?:-\\d{4})?)\\n[ \\t\\n]*(?P[^\\s@]+@[^\\s@]+\\.[^\\s@]+)\\n[ \\t\\n]*Edit[ \\t]*(?=\\n|$)" - } + ], + "record_identity_fields": [ + "product" + ], + "table_scope": { + "root_selector": ".doc-section-demo", + "root_index": 0, + "row_selector": "tbody tr", + "cell_selector": ":scope > td:not(:nth-child(4))", + "start_pattern": "(?im)^(?=[ \\t|]*Product[ \\t|]+Category)", + "end_pattern": "(?im)^[ \\t#]*(?:PREVIEW|Usage)[ \\t]*$", + "plain_row_pattern": "(?P[^\\n]+?\\s+[a-z0-9]{9})\\s+(?PAccessories|Fitness|Clothing)\\s+(?P\\$\\s*\\d+(?:\\.\\d{1,2})?)\\s+(?PIn Stock|Low Stock|Out of Stock)", + "plain_multiline_row_pattern": "(?m)^(?P[^\\n]+\\n[a-z0-9]{9})\\s*\\n[\\s]*(?PAccessories|Fitness|Clothing)[\\s]+(?P\\$\\s*\\d+(?:\\.\\d{1,2})?)[\\s]+(?PIn Stock|Low Stock|Out of Stock)(?=\\n|$)", + "reference_kind": "fixed_demo", + "reference_rows": [ + { + "product": "Bamboo Watch f230fh0g3", + "category": "Accessories", + "price": "$65", + "status": "In Stock" + }, + { + "product": "Black Watch nvklal433", + "category": "Accessories", + "price": "$72", + "status": "In Stock" + }, + { + "product": "Blue Band zz21cz3c1", + "category": "Fitness", + "price": "$79", + "status": "Low Stock" + }, + { + "product": "Blue T-Shirt 244wgerg2", + "category": "Clothing", + "price": "$29", + "status": "In Stock" + }, + { + "product": "Bracelet h456wer53", + "category": "Accessories", + "price": "$15", + "status": "In Stock" + }, + { + "product": "Brown Purse av2231fwg", + "category": "Accessories", + "price": "$120", + "status": "Out of Stock" + }, + { + "product": "Chakra Bracelet bib36pfvm", + "category": "Accessories", + "price": "$32", + "status": "Low Stock" + }, + { + "product": "Galaxy Earrings mbvjkgip5", + "category": "Accessories", + "price": "$34", + "status": "In Stock" + } + ], + "ignored_column_labels": [ + "Rating" + ] + } + } + }, + "artifact": "Product Category Price Rating Status\nBamboo Watch f230fh0g3 Accessories $65 In Stock\nBlack Watch nvklal433 Accessories $72 In Stock\nBlue Band zz21cz3c1 Fitness $79 Low Stock\nBlue T-Shirt 244wgerg2 Clothing $29 In Stock\nBracelet h456wer53 Accessories $15 In Stock\nBrown Purse av2231fwg Accessories $120 Out of Stock\nBamboo Watch f230fh0g3 Accessories $65 In Stock\nPREVIEW", + "ok": true, + "expected_task_pass": false + }, + { + "id": "nintendo-scoped-games-jp-new-games", + "category": "acceptance-contract", + "root_cause": "nintendo_real_card_routes_not_historical_membership", + "failure_mode": "false-negative", + "case": { + "url": "https://www.nintendo.com/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "www.nintendo.com", + "store-jp.nintendo.com" + ], + "entry_path": "(?:/item/software/[A-Za-z0-9]+|/jp/games/(?:switch2/[A-Za-z0-9-]+|zelda/(?:botw|totk)/edition|feature/nintendo-classics/[A-Za-z0-9_-]+)/index\\.html|/jp/xenoblade/index\\.html|/jp/topics/article/[0-9a-f-]{36})", + "stable_text_anchors": [], + "name_validation": "nonempty", + "entry_scope": { + "html_selector": ".nc3-c-softCard", + "title_selector": ".nc3-c-softCard__name", + "link_selector": "a[href]:has(.nc3-c-softCard__name)" + } + } + }, + "artifact": "
Future Game Alpha
Future Game Beta
", + "ok": true, + "expected_task_pass": true + }, + { + "id": "nintendo-scoped-games-jp-markdown", + "category": "acceptance-contract", + "root_cause": "nintendo_real_card_routes_not_historical_membership", + "failure_mode": "false-negative", + "case": { + "url": "https://www.nintendo.com/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "www.nintendo.com", + "store-jp.nintendo.com" + ], + "entry_path": "(?:/item/software/[A-Za-z0-9]+|/jp/games/(?:switch2/[A-Za-z0-9-]+|zelda/(?:botw|totk)/edition|feature/nintendo-classics/[A-Za-z0-9_-]+)/index\\.html|/jp/xenoblade/index\\.html|/jp/topics/article/[0-9a-f-]{36})", + "stable_text_anchors": [], + "name_validation": "nonempty", + "entry_scope": { + "html_selector": ".nc3-c-softCard", + "title_selector": ".nc3-c-softCard__name", + "link_selector": "a[href]:has(.nc3-c-softCard__name)" + } + } + }, + "artifact": "[Future Game Alpha](https://store-jp.nintendo.com/item/software/D70010000999991)\n[Future Game Beta](https://www.nintendo.com/jp/topics/article/01234567-1234-1234-1234-123456789abc)", + "ok": true, + "expected_task_pass": true + }, + { + "id": "nintendo-scoped-games-jp-empty-title", + "category": "acceptance-contract", + "root_cause": "nintendo_real_card_routes_not_historical_membership", + "failure_mode": "false-positive", + "case": { + "url": "https://www.nintendo.com/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "www.nintendo.com", + "store-jp.nintendo.com" + ], + "entry_path": "(?:/item/software/[A-Za-z0-9]+|/jp/games/(?:switch2/[A-Za-z0-9-]+|zelda/(?:botw|totk)/edition|feature/nintendo-classics/[A-Za-z0-9_-]+)/index\\.html|/jp/xenoblade/index\\.html|/jp/topics/article/[0-9a-f-]{36})", + "stable_text_anchors": [], + "name_validation": "nonempty", + "entry_scope": { + "html_selector": ".nc3-c-softCard", + "title_selector": ".nc3-c-softCard__name", + "link_selector": "a[href]:has(.nc3-c-softCard__name)" + } + } + }, + "artifact": "
Future Game Beta
", + "ok": true, + "expected_task_pass": false + }, + { + "id": "nintendo-scoped-games-jp-duplicates", + "category": "acceptance-contract", + "root_cause": "nintendo_real_card_routes_not_historical_membership", + "failure_mode": "false-positive", + "case": { + "url": "https://www.nintendo.com/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "www.nintendo.com", + "store-jp.nintendo.com" + ], + "entry_path": "(?:/item/software/[A-Za-z0-9]+|/jp/games/(?:switch2/[A-Za-z0-9-]+|zelda/(?:botw|totk)/edition|feature/nintendo-classics/[A-Za-z0-9_-]+)/index\\.html|/jp/xenoblade/index\\.html|/jp/topics/article/[0-9a-f-]{36})", + "stable_text_anchors": [], + "name_validation": "nonempty", + "entry_scope": { + "html_selector": ".nc3-c-softCard", + "title_selector": ".nc3-c-softCard__name", + "link_selector": "a[href]:has(.nc3-c-softCard__name)" + } + } + }, + "artifact": "
Future Game Alpha
Future Game Alpha
", + "ok": true, + "expected_task_pass": false + }, + { + "id": "nintendo-scoped-games-jp-navigation-only", + "category": "acceptance-contract", + "root_cause": "nintendo_real_card_routes_not_historical_membership", + "failure_mode": "false-positive", + "case": { + "url": "https://www.nintendo.com/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "www.nintendo.com", + "store-jp.nintendo.com" + ], + "entry_path": "(?:/item/software/[A-Za-z0-9]+|/jp/games/(?:switch2/[A-Za-z0-9-]+|zelda/(?:botw|totk)/edition|feature/nintendo-classics/[A-Za-z0-9_-]+)/index\\.html|/jp/xenoblade/index\\.html|/jp/topics/article/[0-9a-f-]{36})", + "stable_text_anchors": [], + "name_validation": "nonempty", + "entry_scope": { + "html_selector": ".nc3-c-softCard", + "title_selector": ".nc3-c-softCard__name", + "link_selector": "a[href]:has(.nc3-c-softCard__name)" + } + } + }, + "artifact": "", + "ok": true, + "expected_task_pass": false + }, + { + "id": "nintendo-scoped-games-jp-unrelated-path", + "category": "acceptance-contract", + "root_cause": "nintendo_real_card_routes_not_historical_membership", + "failure_mode": "false-positive", + "case": { + "url": "https://www.nintendo.com/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "www.nintendo.com", + "store-jp.nintendo.com" + ], + "entry_path": "(?:/item/software/[A-Za-z0-9]+|/jp/games/(?:switch2/[A-Za-z0-9-]+|zelda/(?:botw|totk)/edition|feature/nintendo-classics/[A-Za-z0-9_-]+)/index\\.html|/jp/xenoblade/index\\.html|/jp/topics/article/[0-9a-f-]{36})", + "stable_text_anchors": [], + "name_validation": "nonempty", + "entry_scope": { + "html_selector": ".nc3-c-softCard", + "title_selector": ".nc3-c-softCard__name", + "link_selector": "a[href]:has(.nc3-c-softCard__name)" + } + } + }, + "artifact": "
Help
Future Game Beta
", + "ok": true, + "expected_task_pass": false + }, + { + "id": "nintendo-scoped-games-es-new-games", + "category": "acceptance-contract", + "root_cause": "nintendo_real_card_routes_not_historical_membership", + "failure_mode": "false-negative", + "case": { + "url": "https://www.nintendo.com/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "www.nintendo.com" + ], + "entry_path": "/es-es/Juegos/(?:Juegos-de-Nintendo-Switch-2|Nintendo-Switch-2-Edition|Juegos-de-Nintendo-Switch|Programas-descargables-Nintendo-Switch|Game-Boy-Advance|Juegos-de-dispositivo-inteligente)/[^/]+-[0-9]+\\.html", + "stable_text_anchors": [], + "name_validation": "nonempty", + "entry_scope": { + "html_selector": ".searchresult_row", + "title_selector": ".page-title-text", + "link_selector": "a[href]:has(.page-title-text)" + } + } + }, + "artifact": "
Future Game Alpha
Future Game Beta
", + "ok": true, + "expected_task_pass": true + }, + { + "id": "nintendo-scoped-games-es-markdown", + "category": "acceptance-contract", + "root_cause": "nintendo_real_card_routes_not_historical_membership", + "failure_mode": "false-negative", + "case": { + "url": "https://www.nintendo.com/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "www.nintendo.com" + ], + "entry_path": "/es-es/Juegos/(?:Juegos-de-Nintendo-Switch-2|Nintendo-Switch-2-Edition|Juegos-de-Nintendo-Switch|Programas-descargables-Nintendo-Switch|Game-Boy-Advance|Juegos-de-dispositivo-inteligente)/[^/]+-[0-9]+\\.html", + "stable_text_anchors": [], + "name_validation": "nonempty", + "entry_scope": { + "html_selector": ".searchresult_row", + "title_selector": ".page-title-text", + "link_selector": "a[href]:has(.page-title-text)" + } + } + }, + "artifact": "[Future Game Alpha](https://www.nintendo.com/es-es/Juegos/Game-Boy-Advance/New-Game-999991.html)\n[Future Game Beta](https://www.nintendo.com/es-es/Juegos/Juegos-de-dispositivo-inteligente/Another-Game-999992.html)", + "ok": true, + "expected_task_pass": true + }, + { + "id": "nintendo-scoped-games-es-empty-title", + "category": "acceptance-contract", + "root_cause": "nintendo_real_card_routes_not_historical_membership", + "failure_mode": "false-positive", + "case": { + "url": "https://www.nintendo.com/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "www.nintendo.com" + ], + "entry_path": "/es-es/Juegos/(?:Juegos-de-Nintendo-Switch-2|Nintendo-Switch-2-Edition|Juegos-de-Nintendo-Switch|Programas-descargables-Nintendo-Switch|Game-Boy-Advance|Juegos-de-dispositivo-inteligente)/[^/]+-[0-9]+\\.html", + "stable_text_anchors": [], + "name_validation": "nonempty", + "entry_scope": { + "html_selector": ".searchresult_row", + "title_selector": ".page-title-text", + "link_selector": "a[href]:has(.page-title-text)" + } + } + }, + "artifact": "
Future Game Beta
", + "ok": true, + "expected_task_pass": false + }, + { + "id": "nintendo-scoped-games-es-duplicates", + "category": "acceptance-contract", + "root_cause": "nintendo_real_card_routes_not_historical_membership", + "failure_mode": "false-positive", + "case": { + "url": "https://www.nintendo.com/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "www.nintendo.com" + ], + "entry_path": "/es-es/Juegos/(?:Juegos-de-Nintendo-Switch-2|Nintendo-Switch-2-Edition|Juegos-de-Nintendo-Switch|Programas-descargables-Nintendo-Switch|Game-Boy-Advance|Juegos-de-dispositivo-inteligente)/[^/]+-[0-9]+\\.html", + "stable_text_anchors": [], + "name_validation": "nonempty", + "entry_scope": { + "html_selector": ".searchresult_row", + "title_selector": ".page-title-text", + "link_selector": "a[href]:has(.page-title-text)" + } + } + }, + "artifact": "
Future Game Alpha
Future Game Alpha
", + "ok": true, + "expected_task_pass": false + }, + { + "id": "nintendo-scoped-games-es-navigation-only", + "category": "acceptance-contract", + "root_cause": "nintendo_real_card_routes_not_historical_membership", + "failure_mode": "false-positive", + "case": { + "url": "https://www.nintendo.com/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "www.nintendo.com" + ], + "entry_path": "/es-es/Juegos/(?:Juegos-de-Nintendo-Switch-2|Nintendo-Switch-2-Edition|Juegos-de-Nintendo-Switch|Programas-descargables-Nintendo-Switch|Game-Boy-Advance|Juegos-de-dispositivo-inteligente)/[^/]+-[0-9]+\\.html", + "stable_text_anchors": [], + "name_validation": "nonempty", + "entry_scope": { + "html_selector": ".searchresult_row", + "title_selector": ".page-title-text", + "link_selector": "a[href]:has(.page-title-text)" + } + } + }, + "artifact": "", + "ok": true, + "expected_task_pass": false + }, + { + "id": "nintendo-scoped-games-es-unrelated-path", + "category": "acceptance-contract", + "root_cause": "nintendo_real_card_routes_not_historical_membership", + "failure_mode": "false-positive", + "case": { + "url": "https://www.nintendo.com/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "www.nintendo.com" + ], + "entry_path": "/es-es/Juegos/(?:Juegos-de-Nintendo-Switch-2|Nintendo-Switch-2-Edition|Juegos-de-Nintendo-Switch|Programas-descargables-Nintendo-Switch|Game-Boy-Advance|Juegos-de-dispositivo-inteligente)/[^/]+-[0-9]+\\.html", + "stable_text_anchors": [], + "name_validation": "nonempty", + "entry_scope": { + "html_selector": ".searchresult_row", + "title_selector": ".page-title-text", + "link_selector": "a[href]:has(.page-title-text)" + } + } + }, + "artifact": "
Help
Future Game Beta
", + "ok": true, + "expected_task_pass": false + }, + { + "id": "forbidden-html-surface-dormant-template", + "category": "answer-surface", + "root_cause": "dormant_html_templates_not_blocking_notices", + "failure_mode": "false-negative", + "case": { + "url": "https://example.test/catalog", + "task_kind": "body", + "expected": [ + "Game catalog" + ], + "forbidden_any": [ + "CAPTCHA", + "Access denied" + ], + "min_content_chars": 1 + }, + "artifact": "

Game catalog

Actual game list

", + "ok": true, + "expected_task_pass": true + }, + { + "id": "forbidden-html-surface-visible-challenge", + "category": "answer-surface", + "root_cause": "dormant_html_templates_not_blocking_notices", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/catalog", + "task_kind": "body", + "expected": [ + "Game catalog" + ], + "forbidden_any": [ + "CAPTCHA", + "Access denied" + ], + "min_content_chars": 1 + }, + "artifact": "

Game catalog

CAPTCHA required

", + "ok": true, + "expected_task_pass": false + }, + { + "id": "forbidden-html-surface-explicitly-hidden", + "category": "answer-surface", + "root_cause": "dormant_html_templates_not_blocking_notices", + "failure_mode": "false-negative", + "case": { + "url": "https://example.test/catalog", + "task_kind": "body", + "expected": [ + "Game catalog" + ], + "forbidden_any": [ + "CAPTCHA", + "Access denied" + ], + "min_content_chars": 1 + }, + "artifact": "
Game catalog
", + "ok": true, + "expected_task_pass": true + }, + { + "id": "forbidden-html-surface-actual-denial", + "category": "answer-surface", + "root_cause": "dormant_html_templates_not_blocking_notices", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/catalog", + "task_kind": "body", + "expected": [ + "Game catalog" + ], + "forbidden_any": [ + "CAPTCHA", + "Access denied" + ], + "min_content_chars": 1 + }, + "artifact": "
Game catalog
Access denied
", + "ok": true, + "expected_task_pass": false + }, + { + "id": "forbidden-html-surface-plain-challenge", + "category": "answer-surface", + "root_cause": "dormant_html_templates_not_blocking_notices", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/catalog", + "task_kind": "body", + "expected": [ + "Game catalog" + ], + "forbidden_any": [ + "CAPTCHA", + "Access denied" + ], + "min_content_chars": 1 + }, + "artifact": "Game catalog\nCAPTCHA required", + "ok": true, + "expected_task_pass": false + }, + { + "id": "nintendo-markdown-section-actual-game-section", + "category": "structured-extraction", + "root_cause": "game_card_announcements_not_navigation_news", + "failure_mode": "false-negative", + "case": { + "url": "https://www.nintendo.com/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "www.nintendo.com", + "store-jp.nintendo.com" + ], + "entry_path": "(?:/item/software/[A-Za-z0-9]+|/jp/games/(?:switch2/[A-Za-z0-9-]+|zelda/(?:botw|totk)/edition|feature/nintendo-classics/[A-Za-z0-9_-]+)/index\\.html|/jp/xenoblade/index\\.html|/jp/topics/article/[0-9a-f-]{36})", + "stable_text_anchors": [], + "name_validation": "nonempty", + "entry_scope": { + "html_selector": ".nc3-c-softCard", + "title_selector": ".nc3-c-softCard__name", + "link_selector": "a[href]:has(.nc3-c-softCard__name)", + "start_pattern": "^[ \\t#*·-]*ピックアップ[ \\t*]*$", + "end_pattern": "^[ \\t#*]*ゲームを楽しむ" + } + } + }, + "artifact": "## ピックアップ\n[Future Game Alpha](https://store-jp.nintendo.com/item/software/D70010000999991)\n[Future Game Beta](https://www.nintendo.com/jp/topics/article/01234567-1234-1234-1234-123456789abc)\n### ゲームを楽しむ", + "ok": true, + "expected_task_pass": true + }, + { + "id": "nintendo-markdown-section-navigation-news-only", + "category": "structured-extraction", + "root_cause": "game_card_announcements_not_navigation_news", + "failure_mode": "false-positive", + "case": { + "url": "https://www.nintendo.com/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "www.nintendo.com", + "store-jp.nintendo.com" + ], + "entry_path": "(?:/item/software/[A-Za-z0-9]+|/jp/games/(?:switch2/[A-Za-z0-9-]+|zelda/(?:botw|totk)/edition|feature/nintendo-classics/[A-Za-z0-9_-]+)/index\\.html|/jp/xenoblade/index\\.html|/jp/topics/article/[0-9a-f-]{36})", + "stable_text_anchors": [], + "name_validation": "nonempty", + "entry_scope": { + "html_selector": ".nc3-c-softCard", + "title_selector": ".nc3-c-softCard__name", + "link_selector": "a[href]:has(.nc3-c-softCard__name)", + "start_pattern": "^[ \\t#*·-]*ピックアップ[ \\t*]*$", + "end_pattern": "^[ \\t#*]*ゲームを楽しむ" + } + } + }, + "artifact": "[Future Game Alpha](https://store-jp.nintendo.com/item/software/D70010000999991)\n[Future Game Beta](https://www.nintendo.com/jp/topics/article/01234567-1234-1234-1234-123456789abc)\n## ピックアップ\nNo game records\n### ゲームを楽しむ", + "ok": true, + "expected_task_pass": false + }, + { + "id": "nintendo-markdown-section-links-after-game-section", + "category": "structured-extraction", + "root_cause": "game_card_announcements_not_navigation_news", + "failure_mode": "false-positive", + "case": { + "url": "https://www.nintendo.com/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "www.nintendo.com", + "store-jp.nintendo.com" + ], + "entry_path": "(?:/item/software/[A-Za-z0-9]+|/jp/games/(?:switch2/[A-Za-z0-9-]+|zelda/(?:botw|totk)/edition|feature/nintendo-classics/[A-Za-z0-9_-]+)/index\\.html|/jp/xenoblade/index\\.html|/jp/topics/article/[0-9a-f-]{36})", + "stable_text_anchors": [], + "name_validation": "nonempty", + "entry_scope": { + "html_selector": ".nc3-c-softCard", + "title_selector": ".nc3-c-softCard__name", + "link_selector": "a[href]:has(.nc3-c-softCard__name)", + "start_pattern": "^[ \\t#*·-]*ピックアップ[ \\t*]*$", + "end_pattern": "^[ \\t#*]*ゲームを楽しむ" + } + } + }, + "artifact": "## ピックアップ\nNo game records\n### ゲームを楽しむ\n[Future Game Alpha](https://store-jp.nintendo.com/item/software/D70010000999991)\n[Future Game Beta](https://www.nintendo.com/jp/topics/article/01234567-1234-1234-1234-123456789abc)", + "ok": true, + "expected_task_pass": false + }, + { + "id": "nintendo-markdown-section-bullet-heading", + "category": "structured-extraction", + "root_cause": "game_card_announcements_not_navigation_news", + "failure_mode": "false-negative", + "case": { + "url": "https://www.nintendo.com/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "www.nintendo.com", + "store-jp.nintendo.com" + ], + "entry_path": "(?:/item/software/[A-Za-z0-9]+|/jp/games/(?:switch2/[A-Za-z0-9-]+|zelda/(?:botw|totk)/edition|feature/nintendo-classics/[A-Za-z0-9_-]+)/index\\.html|/jp/xenoblade/index\\.html|/jp/topics/article/[0-9a-f-]{36})", + "stable_text_anchors": [], + "name_validation": "nonempty", + "entry_scope": { + "html_selector": ".nc3-c-softCard", + "title_selector": ".nc3-c-softCard__name", + "link_selector": "a[href]:has(.nc3-c-softCard__name)", + "start_pattern": "^[ \\t#*·-]*ピックアップ[ \\t*]*$", + "end_pattern": "^[ \\t#*]*ゲームを楽しむ" + } + } + }, + "artifact": "* ピックアップ\n[Future Game Alpha](https://store-jp.nintendo.com/item/software/D70010000999991)\n[Future Game Beta](https://www.nintendo.com/jp/topics/article/01234567-1234-1234-1234-123456789abc)", + "ok": true, + "expected_task_pass": true + }, + { + "id": "dynamic-026-bilibili-rank-markdown-scope-inside", + "category": "structured-extraction", + "root_cause": "rank_markdown_scope_excludes_navigation", + "failure_mode": "false-negative", + "case": { + "url": "https://www.bilibili.com/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "www.bilibili.com" + ], + "entry_path": "/video/(?:BV[A-Za-z0-9]+|av[0-9]+)", + "stable_text_anchors": [], + "entry_scope": { + "html_selector": ".rank-item a.title", + "start_pattern": "排行榜根据稿件内容质量，近期的数据综合展示，动态更新", + "end_pattern": "(?i)^[ \\t#*]*bilibili[ \\t*]*$" + }, + "name_validation": "nonempty" + } + }, + "artifact": "排行榜根据稿件内容质量，近期的数据综合展示，动态更新\n[Actual Name 0](https://www.bilibili.com/video/BV1Ab4111111)\n[Actual Name 1](https://www.bilibili.com/video/BV1Ab4222222)\nbilibili", + "ok": true, + "expected_task_pass": true + }, + { + "id": "dynamic-026-bilibili-rank-markdown-scope-navigation-only", + "category": "structured-extraction", + "root_cause": "rank_markdown_scope_excludes_navigation", + "failure_mode": "false-positive", + "case": { + "url": "https://www.bilibili.com/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "www.bilibili.com" + ], + "entry_path": "/video/(?:BV[A-Za-z0-9]+|av[0-9]+)", + "stable_text_anchors": [], + "entry_scope": { + "html_selector": ".rank-item a.title", + "start_pattern": "排行榜根据稿件内容质量，近期的数据综合展示，动态更新", + "end_pattern": "(?i)^[ \\t#*]*bilibili[ \\t*]*$" + }, + "name_validation": "nonempty" + } + }, + "artifact": "[Actual Name 0](https://www.bilibili.com/video/BV1Ab4111111)\n[Actual Name 1](https://www.bilibili.com/video/BV1Ab4222222)\n排行榜根据稿件内容质量，近期的数据综合展示，动态更新\nNo target records\nbilibili", + "ok": true, + "expected_task_pass": false + }, + { + "id": "dynamic-026-bilibili-rank-markdown-scope-footer-only", + "category": "structured-extraction", + "root_cause": "rank_markdown_scope_excludes_navigation", + "failure_mode": "false-positive", + "case": { + "url": "https://www.bilibili.com/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "www.bilibili.com" + ], + "entry_path": "/video/(?:BV[A-Za-z0-9]+|av[0-9]+)", + "stable_text_anchors": [], + "entry_scope": { + "html_selector": ".rank-item a.title", + "start_pattern": "排行榜根据稿件内容质量，近期的数据综合展示，动态更新", + "end_pattern": "(?i)^[ \\t#*]*bilibili[ \\t*]*$" + }, + "name_validation": "nonempty" + } + }, + "artifact": "排行榜根据稿件内容质量，近期的数据综合展示，动态更新\nNo target records\nbilibili\n[Actual Name 0](https://www.bilibili.com/video/BV1Ab4111111)\n[Actual Name 1](https://www.bilibili.com/video/BV1Ab4222222)", + "ok": true, + "expected_task_pass": false + }, + { + "id": "dynamic-048-en-github-trending-developers-markdown-scope-inside", + "category": "structured-extraction", + "root_cause": "rank_markdown_scope_excludes_navigation", + "failure_mode": "false-negative", + "case": { + "url": "https://github.com/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "github.com" + ], + "entry_path": "/(?!login$|signup$|features$|topics$|trending$)[A-Za-z0-9_.-]+", + "stable_text_anchors": [], + "entry_scope": { + "html_selector": "article.Box-row h1", + "start_pattern": "These are the developers building the hot tools today\\.", + "end_pattern": "^[ \\t#*]*Footer[ \\t*]*$" + }, + "name_validation": "nonempty" + } + }, + "artifact": "These are the developers building the hot tools today.\n[Actual Name 0](https://github.com/newalpha)\n[Actual Name 1](https://github.com/newbeta)\nFooter", + "ok": true, + "expected_task_pass": true + }, + { + "id": "dynamic-048-en-github-trending-developers-markdown-scope-navigation-only", + "category": "structured-extraction", + "root_cause": "rank_markdown_scope_excludes_navigation", + "failure_mode": "false-positive", + "case": { + "url": "https://github.com/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "github.com" + ], + "entry_path": "/(?!login$|signup$|features$|topics$|trending$)[A-Za-z0-9_.-]+", + "stable_text_anchors": [], + "entry_scope": { + "html_selector": "article.Box-row h1", + "start_pattern": "These are the developers building the hot tools today\\.", + "end_pattern": "^[ \\t#*]*Footer[ \\t*]*$" + }, + "name_validation": "nonempty" + } + }, + "artifact": "[Actual Name 0](https://github.com/newalpha)\n[Actual Name 1](https://github.com/newbeta)\nThese are the developers building the hot tools today.\nNo target records\nFooter", + "ok": true, + "expected_task_pass": false + }, + { + "id": "dynamic-048-en-github-trending-developers-markdown-scope-footer-only", + "category": "structured-extraction", + "root_cause": "rank_markdown_scope_excludes_navigation", + "failure_mode": "false-positive", + "case": { + "url": "https://github.com/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "github.com" + ], + "entry_path": "/(?!login$|signup$|features$|topics$|trending$)[A-Za-z0-9_.-]+", + "stable_text_anchors": [], + "entry_scope": { + "html_selector": "article.Box-row h1", + "start_pattern": "These are the developers building the hot tools today\\.", + "end_pattern": "^[ \\t#*]*Footer[ \\t*]*$" + }, + "name_validation": "nonempty" + } + }, + "artifact": "These are the developers building the hot tools today.\nNo target records\nFooter\n[Actual Name 0](https://github.com/newalpha)\n[Actual Name 1](https://github.com/newbeta)", + "ok": true, + "expected_task_pass": false + }, + { + "id": "dynamic-044-en-real-github-trending-markdown-scope-inside", + "category": "structured-extraction", + "root_cause": "rank_markdown_scope_excludes_navigation", + "failure_mode": "false-negative", + "case": { + "url": "https://github.com/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "github.com" + ], + "entry_path": "/(?!trending(?:/|$)|apps/|sponsors/|topics/|collections/|settings/|features/|marketplace/|orgs/|users/|login/|signup/)[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+", + "stable_text_anchors": [], + "entry_scope": { + "html_selector": "article.Box-row h2", + "start_pattern": "See what the GitHub community is most excited about today\\.", + "end_pattern": "^[ \\t#*]*Footer[ \\t*]*$" + }, + "name_validation": "nonempty" + } + }, + "artifact": "See what the GitHub community is most excited about today.\n[Actual Name 0](https://github.com/newalpha/project)\n[Actual Name 1](https://github.com/newbeta/project)\nFooter", + "ok": true, + "expected_task_pass": true + }, + { + "id": "dynamic-044-en-real-github-trending-markdown-scope-navigation-only", + "category": "structured-extraction", + "root_cause": "rank_markdown_scope_excludes_navigation", + "failure_mode": "false-positive", + "case": { + "url": "https://github.com/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "github.com" + ], + "entry_path": "/(?!trending(?:/|$)|apps/|sponsors/|topics/|collections/|settings/|features/|marketplace/|orgs/|users/|login/|signup/)[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+", + "stable_text_anchors": [], + "entry_scope": { + "html_selector": "article.Box-row h2", + "start_pattern": "See what the GitHub community is most excited about today\\.", + "end_pattern": "^[ \\t#*]*Footer[ \\t*]*$" + }, + "name_validation": "nonempty" + } + }, + "artifact": "[Actual Name 0](https://github.com/newalpha/project)\n[Actual Name 1](https://github.com/newbeta/project)\nSee what the GitHub community is most excited about today.\nNo target records\nFooter", + "ok": true, + "expected_task_pass": false + }, + { + "id": "dynamic-044-en-real-github-trending-markdown-scope-footer-only", + "category": "structured-extraction", + "root_cause": "rank_markdown_scope_excludes_navigation", + "failure_mode": "false-positive", + "case": { + "url": "https://github.com/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "github.com" + ], + "entry_path": "/(?!trending(?:/|$)|apps/|sponsors/|topics/|collections/|settings/|features/|marketplace/|orgs/|users/|login/|signup/)[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+", + "stable_text_anchors": [], + "entry_scope": { + "html_selector": "article.Box-row h2", + "start_pattern": "See what the GitHub community is most excited about today\\.", + "end_pattern": "^[ \\t#*]*Footer[ \\t*]*$" + }, + "name_validation": "nonempty" + } + }, + "artifact": "See what the GitHub community is most excited about today.\nNo target records\nFooter\n[Actual Name 0](https://github.com/newalpha/project)\n[Actual Name 1](https://github.com/newbeta/project)", + "ok": true, + "expected_task_pass": false + }, + { + "id": "auxiliary-video-links-two-main-plus-popup", + "category": "structured-extraction", + "root_cause": "hidden_related_video_popup_not_main_rank", + "failure_mode": "false-negative", + "case": { + "url": "https://www.bilibili.com/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "www.bilibili.com" + ], + "entry_path": "/video/(?:BV[A-Za-z0-9]+|av[0-9]+)", + "stable_text_anchors": [], + "entry_scope": { + "html_selector": ".rank-item a.title", + "start_pattern": "排行榜根据稿件内容质量，近期的数据综合展示，动态更新", + "end_pattern": "(?i)^[ \\t#*]*bilibili[ \\t*]*$", + "auxiliary_links_end_pattern": "全部上榜视频" + }, + "name_validation": "nonempty" + } + }, + "artifact": "排行榜根据稿件内容质量，近期的数据综合展示，动态更新\n[Main Alpha](https://www.bilibili.com/video/BV1Ab4111111)\n123 views\n[Other One](https://www.bilibili.com/video/BV1Ab4333333)\n[Other Two](https://www.bilibili.com/video/BV1Ab4444444)\n全部上榜视频\n[Main Beta](https://www.bilibili.com/video/BV1Ab4222222)", + "ok": true, + "expected_task_pass": true + }, + { + "id": "auxiliary-video-links-popup-cannot-fill", + "category": "structured-extraction", + "root_cause": "hidden_related_video_popup_not_main_rank", + "failure_mode": "false-positive", + "case": { + "url": "https://www.bilibili.com/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "www.bilibili.com" + ], + "entry_path": "/video/(?:BV[A-Za-z0-9]+|av[0-9]+)", + "stable_text_anchors": [], + "entry_scope": { + "html_selector": ".rank-item a.title", + "start_pattern": "排行榜根据稿件内容质量，近期的数据综合展示，动态更新", + "end_pattern": "(?i)^[ \\t#*]*bilibili[ \\t*]*$", + "auxiliary_links_end_pattern": "全部上榜视频" + }, + "name_validation": "nonempty" + } + }, + "artifact": "排行榜根据稿件内容质量，近期的数据综合展示，动态更新\n[Main Alpha](https://www.bilibili.com/video/BV1Ab4111111)\n123 views\n[Other One](https://www.bilibili.com/video/BV1Ab4333333)\n[Other Two](https://www.bilibili.com/video/BV1Ab4444444)\n全部上榜视频", + "ok": true, + "expected_task_pass": false + }, + { + "id": "auxiliary-video-links-normal-two-main", + "category": "structured-extraction", + "root_cause": "hidden_related_video_popup_not_main_rank", + "failure_mode": "false-negative", + "case": { + "url": "https://www.bilibili.com/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "www.bilibili.com" + ], + "entry_path": "/video/(?:BV[A-Za-z0-9]+|av[0-9]+)", + "stable_text_anchors": [], + "entry_scope": { + "html_selector": ".rank-item a.title", + "start_pattern": "排行榜根据稿件内容质量，近期的数据综合展示，动态更新", + "end_pattern": "(?i)^[ \\t#*]*bilibili[ \\t*]*$", + "auxiliary_links_end_pattern": "全部上榜视频" + }, + "name_validation": "nonempty" + } + }, + "artifact": "排行榜根据稿件内容质量，近期的数据综合展示，动态更新\n[Main Alpha](https://www.bilibili.com/video/BV1Ab4111111)\n[Main Beta](https://www.bilibili.com/video/BV1Ab4222222)", + "ok": true, + "expected_task_pass": true + }, + { + "id": "markdown-title-metadata-inner-double", + "category": "format-normalization", + "root_cause": "quoted_title_metadata_not_link_destination", + "failure_mode": "false-negative", + "case": { + "url": "https://www.bilibili.com/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 1, + "hosts": [ + "www.bilibili.com" + ], + "entry_path": "/video/(?:BV[A-Za-z0-9]+|av[0-9]+)", + "stable_text_anchors": [], + "entry_scope": { + "html_selector": ".rank-item a.title", + "start_pattern": "排行榜根据稿件内容质量，近期的数据综合展示，动态更新", + "end_pattern": "(?i)^[ \\t#*]*bilibili[ \\t*]*$", + "auxiliary_links_end_pattern": "全部上榜视频" + }, + "name_validation": "nonempty" + } + }, + "artifact": "排行榜根据稿件内容质量，近期的数据综合展示，动态更新\n[Actual \"Name\"](https://www.bilibili.com/video/BV1Ab4111111 \"Actual \"Name\"\")", + "ok": true, + "expected_task_pass": true + }, + { + "id": "markdown-title-metadata-inner-apostrophe", + "category": "format-normalization", + "root_cause": "quoted_title_metadata_not_link_destination", + "failure_mode": "false-negative", + "case": { + "url": "https://www.bilibili.com/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 1, + "hosts": [ + "www.bilibili.com" + ], + "entry_path": "/video/(?:BV[A-Za-z0-9]+|av[0-9]+)", + "stable_text_anchors": [], + "entry_scope": { + "html_selector": ".rank-item a.title", + "start_pattern": "排行榜根据稿件内容质量，近期的数据综合展示，动态更新", + "end_pattern": "(?i)^[ \\t#*]*bilibili[ \\t*]*$", + "auxiliary_links_end_pattern": "全部上榜视频" + }, + "name_validation": "nonempty" + } + }, + "artifact": "排行榜根据稿件内容质量，近期的数据综合展示，动态更新\n[Actual Name](https://www.bilibili.com/video/BV1Ab4111111 \"User's title\")", + "ok": true, + "expected_task_pass": true + }, + { + "id": "markdown-title-metadata-wrong-path", + "category": "format-normalization", + "root_cause": "quoted_title_metadata_not_link_destination", + "failure_mode": "false-positive", + "case": { + "url": "https://www.bilibili.com/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 1, + "hosts": [ + "www.bilibili.com" + ], + "entry_path": "/video/(?:BV[A-Za-z0-9]+|av[0-9]+)", + "stable_text_anchors": [], + "entry_scope": { + "html_selector": ".rank-item a.title", + "start_pattern": "排行榜根据稿件内容质量，近期的数据综合展示，动态更新", + "end_pattern": "(?i)^[ \\t#*]*bilibili[ \\t*]*$", + "auxiliary_links_end_pattern": "全部上榜视频" + }, + "name_validation": "nonempty" + } + }, + "artifact": "排行榜根据稿件内容质量，近期的数据综合展示，动态更新\n[Actual Name](https://www.bilibili.com/help \"Actual \"Name\"\")", + "ok": true, + "expected_task_pass": false + }, + { + "id": "markdown-title-metadata-bare-url", + "category": "format-normalization", + "root_cause": "quoted_title_metadata_not_link_destination", + "failure_mode": "false-positive", + "case": { + "url": "https://www.bilibili.com/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 1, + "hosts": [ + "www.bilibili.com" + ], + "entry_path": "/video/(?:BV[A-Za-z0-9]+|av[0-9]+)", + "stable_text_anchors": [], + "entry_scope": { + "html_selector": ".rank-item a.title", + "start_pattern": "排行榜根据稿件内容质量，近期的数据综合展示，动态更新", + "end_pattern": "(?i)^[ \\t#*]*bilibili[ \\t*]*$", + "auxiliary_links_end_pattern": "全部上榜视频" + }, + "name_validation": "nonempty" + } + }, + "artifact": "排行榜根据稿件内容质量，近期的数据综合展示，动态更新\nActual Name https://www.bilibili.com/video/BV1Ab4111111", + "ok": true, + "expected_task_pass": false + }, + { + "id": "adjacent-song-pair-escaped-markdown-complete", + "category": "structured-extraction", + "root_cause": "adjacent_markdown_links_same_song_identity", + "failure_mode": "false-negative", + "case": { + "url": "https://www.melon.com/chart/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "www.melon.com" + ], + "entry_path": "/song/detail\\.htm", + "stable_text_anchors": [], + "identity_query_keys": [ + "songId" + ], + "entry_scope": { + "html_selector": "tr[data-song-no]", + "markdown_title_identity_pattern": "javascript:melon\\.play\\.playSong\\([^,]*,\\s*([0-9]+)\\);?", + "markdown_detail_identity_pattern": "[?&]songId=([0-9]+)(?:&|$)" + }, + "name_validation": "nonempty", + "title_exclusion_patterns": [ + "곡정보" + ] + } + }, + "artifact": "[곡정보](/song/detail.htm?songId=101)\n\n[새 노래 Alpha](javascript:melon.play.playSong\\('1000002721',101\\);)\n\n[곡정보](/song/detail.htm?songId=102)\n\n[새 노래 Beta](javascript:melon.play.playSong\\('1000002721',102\\);)", + "ok": true, + "expected_task_pass": true + }, + { + "id": "adjacent-song-pair-mismatched-identity", + "category": "structured-extraction", + "root_cause": "adjacent_markdown_links_same_song_identity", + "failure_mode": "false-positive", + "case": { + "url": "https://www.melon.com/chart/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "www.melon.com" + ], + "entry_path": "/song/detail\\.htm", + "stable_text_anchors": [], + "identity_query_keys": [ + "songId" + ], + "entry_scope": { + "html_selector": "tr[data-song-no]", + "markdown_title_identity_pattern": "javascript:melon\\.play\\.playSong\\([^,]*,\\s*([0-9]+)\\);?", + "markdown_detail_identity_pattern": "[?&]songId=([0-9]+)(?:&|$)" + }, + "name_validation": "nonempty", + "title_exclusion_patterns": [ + "곡정보" + ] + } + }, + "artifact": "[곡정보](/song/detail.htm?songId=101)\n\n[새 노래 Alpha](javascript:melon.play.playSong\\('1000002721',101\\);)\n\n[곡정보](/song/detail.htm?songId=999)\n\n[새 노래 Beta](javascript:melon.play.playSong\\('1000002721',102\\);)", + "ok": true, + "expected_task_pass": false + }, + { + "id": "same-row-song-pair-complete", + "category": "structured-extraction", + "root_cause": "visible_song_name_and_detail_link_same_row_identity", + "failure_mode": "false-negative", + "case": { + "url": "https://www.melon.com/chart/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "www.melon.com" + ], + "entry_path": "/song/detail\\.htm", + "stable_text_anchors": [], + "identity_query_keys": [ + "songId" + ], + "entry_scope": { + "html_selector": "tr[data-song-no]", + "title_selector": ".rank01 a", + "link_selector": "a.song_info[href]", + "markdown_title_identity_pattern": "javascript:melon\\.play\\.playSong\\([^,]*,\\s*([0-9]+)\\);?", + "markdown_detail_identity_pattern": "[?&]songId=([0-9]+)(?:&|$)" + }, + "name_validation": "nonempty", + "title_exclusion_patterns": [ + "곡정보" + ] + } + }, + "artifact": "| [곡정보](/song/detail.htm?songId=101) | [새 노래 Alpha](javascript:melon.play.playSong%28'1000002721',101%29;) |\n| [곡정보](/song/detail.htm?songId=102) | [새 노래 Beta](javascript:melon.play.playSong%28'1000002721',102%29;) |", + "ok": true, + "expected_task_pass": true + }, + { + "id": "same-row-song-pair-mismatched-id", + "category": "structured-extraction", + "root_cause": "visible_song_name_and_detail_link_same_row_identity", + "failure_mode": "false-positive", + "case": { + "url": "https://www.melon.com/chart/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "www.melon.com" + ], + "entry_path": "/song/detail\\.htm", + "stable_text_anchors": [], + "identity_query_keys": [ + "songId" + ], + "entry_scope": { + "html_selector": "tr[data-song-no]", + "title_selector": ".rank01 a", + "link_selector": "a.song_info[href]", + "markdown_title_identity_pattern": "javascript:melon\\.play\\.playSong\\([^,]*,\\s*([0-9]+)\\);?", + "markdown_detail_identity_pattern": "[?&]songId=([0-9]+)(?:&|$)" + }, + "name_validation": "nonempty", + "title_exclusion_patterns": [ + "곡정보" + ] + } + }, + "artifact": "| [곡정보](/song/detail.htm?songId=101) | [새 노래 Alpha](javascript:melon.play.playSong%28'1000002721',101%29;) |\n| [곡정보](/song/detail.htm?songId=999) | [새 노래 Beta](javascript:melon.play.playSong%28'1000002721',102%29;) |", + "ok": true, + "expected_task_pass": false + }, + { + "id": "same-row-song-pair-cross-row", + "category": "structured-extraction", + "root_cause": "visible_song_name_and_detail_link_same_row_identity", + "failure_mode": "false-positive", + "case": { + "url": "https://www.melon.com/chart/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "www.melon.com" + ], + "entry_path": "/song/detail\\.htm", + "stable_text_anchors": [], + "identity_query_keys": [ + "songId" + ], + "entry_scope": { + "html_selector": "tr[data-song-no]", + "title_selector": ".rank01 a", + "link_selector": "a.song_info[href]", + "markdown_title_identity_pattern": "javascript:melon\\.play\\.playSong\\([^,]*,\\s*([0-9]+)\\);?", + "markdown_detail_identity_pattern": "[?&]songId=([0-9]+)(?:&|$)" + }, + "name_validation": "nonempty", + "title_exclusion_patterns": [ + "곡정보" + ] + } + }, + "artifact": "| [곡정보](/song/detail.htm?songId=101) | [새 노래 Alpha](javascript:melon.play.playSong%28'1000002721',101%29;) |\n| [곡정보](/song/detail.htm?songId=102) |\n| [새 노래 Beta](javascript:melon.play.playSong%28'1000002721',102%29;) |", + "ok": true, + "expected_task_pass": false + }, + { + "id": "same-row-song-pair-duplicates", + "category": "structured-extraction", + "root_cause": "visible_song_name_and_detail_link_same_row_identity", + "failure_mode": "false-positive", + "case": { + "url": "https://www.melon.com/chart/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "www.melon.com" + ], + "entry_path": "/song/detail\\.htm", + "stable_text_anchors": [], + "identity_query_keys": [ + "songId" + ], + "entry_scope": { + "html_selector": "tr[data-song-no]", + "title_selector": ".rank01 a", + "link_selector": "a.song_info[href]", + "markdown_title_identity_pattern": "javascript:melon\\.play\\.playSong\\([^,]*,\\s*([0-9]+)\\);?", + "markdown_detail_identity_pattern": "[?&]songId=([0-9]+)(?:&|$)" + }, + "name_validation": "nonempty", + "title_exclusion_patterns": [ + "곡정보" + ] + } + }, + "artifact": "| [곡정보](/song/detail.htm?songId=101) | [새 노래 Alpha](javascript:melon.play.playSong%28'1000002721',101%29;) |\n| [곡정보](/song/detail.htm?songId=101) | [새 노래 Alpha](javascript:melon.play.playSong%28'1000002721',101%29;) |", + "ok": true, + "expected_task_pass": false + }, + { + "id": "same-row-song-pair-empty-name", + "category": "structured-extraction", + "root_cause": "visible_song_name_and_detail_link_same_row_identity", + "failure_mode": "false-positive", + "case": { + "url": "https://www.melon.com/chart/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "www.melon.com" + ], + "entry_path": "/song/detail\\.htm", + "stable_text_anchors": [], + "identity_query_keys": [ + "songId" + ], + "entry_scope": { + "html_selector": "tr[data-song-no]", + "title_selector": ".rank01 a", + "link_selector": "a.song_info[href]", + "markdown_title_identity_pattern": "javascript:melon\\.play\\.playSong\\([^,]*,\\s*([0-9]+)\\);?", + "markdown_detail_identity_pattern": "[?&]songId=([0-9]+)(?:&|$)" + }, + "name_validation": "nonempty", + "title_exclusion_patterns": [ + "곡정보" + ] + } + }, + "artifact": "| [곡정보](/song/detail.htm?songId=101) | [새 노래 Alpha](javascript:melon.play.playSong%28'1000002721',101%29;) |\n| [곡정보](/song/detail.htm?songId=102) | [](javascript:melon.play.playSong%28'1000002721',102%29;) |", + "ok": true, + "expected_task_pass": false + }, + { + "id": "same-row-song-pair-direct-named-links", + "category": "structured-extraction", + "root_cause": "visible_song_name_and_detail_link_same_row_identity", + "failure_mode": "false-negative", + "case": { + "url": "https://www.melon.com/chart/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "www.melon.com" + ], + "entry_path": "/song/detail\\.htm", + "stable_text_anchors": [], + "identity_query_keys": [ + "songId" + ], + "entry_scope": { + "html_selector": "tr[data-song-no]", + "title_selector": ".rank01 a", + "link_selector": "a.song_info[href]", + "markdown_title_identity_pattern": "javascript:melon\\.play\\.playSong\\([^,]*,\\s*([0-9]+)\\);?", + "markdown_detail_identity_pattern": "[?&]songId=([0-9]+)(?:&|$)" + }, + "name_validation": "nonempty", + "title_exclusion_patterns": [ + "곡정보" + ] + } + }, + "artifact": "[새 노래 Alpha](/song/detail.htm?songId=101)\n[새 노래 Beta](/song/detail.htm?songId=102)", + "ok": true, + "expected_task_pass": true + }, + { + "id": "dynamic-052-bilibili-com-v-footer-scope-inside", + "category": "structured-extraction", + "root_cause": "footer_links_not_dynamic_list_records", + "failure_mode": "false-negative", + "case": { + "url": "https://www.bilibili.com/v/popular/all/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "www.bilibili.com" + ], + "entry_path": "/video/(?:BV[A-Za-z0-9]+|av[0-9]+)", + "stable_text_anchors": [], + "entry_scope": { + "html_selector": ".video-card", + "title_selector": "p.video-name", + "link_selector": "a[href*=\"/video/\"]", + "start_pattern": "各个领域中新奇好玩的优质内容都在这里", + "end_pattern": "(?i)^[ \\t#*]*bilibili[ \\t*]*$" + }, + "name_validation": "nonempty" + } + }, + "artifact": "各个领域中新奇好玩的优质内容都在这里~\n[条目0](//www.bilibili.com/video/BV1Ab4111111)\n[条目1](//www.bilibili.com/video/BV1Ab4222222)\nbilibili", + "ok": true, + "expected_task_pass": true + }, + { + "id": "dynamic-052-bilibili-com-v-footer-scope-footer-only", + "category": "structured-extraction", + "root_cause": "footer_links_not_dynamic_list_records", + "failure_mode": "false-positive", + "case": { + "url": "https://www.bilibili.com/v/popular/all/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "www.bilibili.com" + ], + "entry_path": "/video/(?:BV[A-Za-z0-9]+|av[0-9]+)", + "stable_text_anchors": [], + "entry_scope": { + "html_selector": ".video-card", + "title_selector": "p.video-name", + "link_selector": "a[href*=\"/video/\"]", + "start_pattern": "各个领域中新奇好玩的优质内容都在这里", + "end_pattern": "(?i)^[ \\t#*]*bilibili[ \\t*]*$" + }, + "name_validation": "nonempty" + } + }, + "artifact": "各个领域中新奇好玩的优质内容都在这里~\nbilibili\n[条目0](//www.bilibili.com/video/BV1Ab4111111)\n[条目1](//www.bilibili.com/video/BV1Ab4222222)", + "ok": true, + "expected_task_pass": false + }, + { + "id": "dynamic-052-bilibili-com-v-footer-scope-navigation-only", + "category": "structured-extraction", + "root_cause": "footer_links_not_dynamic_list_records", + "failure_mode": "false-positive", + "case": { + "url": "https://www.bilibili.com/v/popular/all/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "www.bilibili.com" + ], + "entry_path": "/video/(?:BV[A-Za-z0-9]+|av[0-9]+)", + "stable_text_anchors": [], + "entry_scope": { + "html_selector": ".video-card", + "title_selector": "p.video-name", + "link_selector": "a[href*=\"/video/\"]", + "start_pattern": "各个领域中新奇好玩的优质内容都在这里", + "end_pattern": "(?i)^[ \\t#*]*bilibili[ \\t*]*$" + }, + "name_validation": "nonempty" + } + }, + "artifact": "[条目0](//www.bilibili.com/video/BV1Ab4111111)\n[条目1](//www.bilibili.com/video/BV1Ab4222222)\n各个领域中新奇好玩的优质内容都在这里~\nbilibili", + "ok": true, + "expected_task_pass": false + }, + { + "id": "dynamic-052-bilibili-com-v-footer-scope-no-footer", + "category": "structured-extraction", + "root_cause": "footer_links_not_dynamic_list_records", + "failure_mode": "false-negative", + "case": { + "url": "https://www.bilibili.com/v/popular/all/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "www.bilibili.com" + ], + "entry_path": "/video/(?:BV[A-Za-z0-9]+|av[0-9]+)", + "stable_text_anchors": [], + "entry_scope": { + "html_selector": ".video-card", + "title_selector": "p.video-name", + "link_selector": "a[href*=\"/video/\"]", + "start_pattern": "各个领域中新奇好玩的优质内容都在这里", + "end_pattern": "(?i)^[ \\t#*]*bilibili[ \\t*]*$" + }, + "name_validation": "nonempty" + } + }, + "artifact": "各个领域中新奇好玩的优质内容都在这里~\n[条目0](//www.bilibili.com/video/BV1Ab4111111)\n[条目1](//www.bilibili.com/video/BV1Ab4222222)", + "ok": true, + "expected_task_pass": true + }, + { + "id": "dynamic-063-s-weibo-com-top-footer-scope-inside", + "category": "structured-extraction", + "root_cause": "footer_links_not_dynamic_list_records", + "failure_mode": "false-negative", + "case": { + "url": "https://s.weibo.com/top/summary", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "s.weibo.com" + ], + "entry_path": "/weibo", + "stable_text_anchors": [], + "identity_query_keys": [ + "q" + ], + "entry_scope": { + "html_selector": "#pl_top_realtimehot tbody tr td.td-02 a[href]", + "start_pattern": "序号[^\\n]*关键词", + "end_pattern": "^[ \\t]*(?:\\[关于微博\\]\\(|关于微博[ \\t]+微博帮助)" + }, + "name_validation": "nonempty" + } + }, + "artifact": "序号 关键词\n[条目0](https://s.weibo.com/weibo?q=newone)\n[条目1](https://s.weibo.com/weibo?q=newtwo)\n关于微博 微博帮助", + "ok": true, + "expected_task_pass": true + }, + { + "id": "dynamic-063-s-weibo-com-top-footer-scope-footer-only", + "category": "structured-extraction", + "root_cause": "footer_links_not_dynamic_list_records", + "failure_mode": "false-positive", + "case": { + "url": "https://s.weibo.com/top/summary", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "s.weibo.com" + ], + "entry_path": "/weibo", + "stable_text_anchors": [], + "identity_query_keys": [ + "q" + ], + "entry_scope": { + "html_selector": "#pl_top_realtimehot tbody tr td.td-02 a[href]", + "start_pattern": "序号[^\\n]*关键词", + "end_pattern": "^[ \\t]*(?:\\[关于微博\\]\\(|关于微博[ \\t]+微博帮助)" + }, + "name_validation": "nonempty" + } + }, + "artifact": "序号 关键词\n关于微博 微博帮助\n[条目0](https://s.weibo.com/weibo?q=newone)\n[条目1](https://s.weibo.com/weibo?q=newtwo)", + "ok": true, + "expected_task_pass": false + }, + { + "id": "dynamic-063-s-weibo-com-top-footer-scope-navigation-only", + "category": "structured-extraction", + "root_cause": "footer_links_not_dynamic_list_records", + "failure_mode": "false-positive", + "case": { + "url": "https://s.weibo.com/top/summary", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "s.weibo.com" + ], + "entry_path": "/weibo", + "stable_text_anchors": [], + "identity_query_keys": [ + "q" + ], + "entry_scope": { + "html_selector": "#pl_top_realtimehot tbody tr td.td-02 a[href]", + "start_pattern": "序号[^\\n]*关键词", + "end_pattern": "^[ \\t]*(?:\\[关于微博\\]\\(|关于微博[ \\t]+微博帮助)" + }, + "name_validation": "nonempty" + } + }, + "artifact": "[条目0](https://s.weibo.com/weibo?q=newone)\n[条目1](https://s.weibo.com/weibo?q=newtwo)\n序号 关键词\n关于微博 微博帮助", + "ok": true, + "expected_task_pass": false + }, + { + "id": "dynamic-063-s-weibo-com-top-footer-scope-no-footer", + "category": "structured-extraction", + "root_cause": "footer_links_not_dynamic_list_records", + "failure_mode": "false-negative", + "case": { + "url": "https://s.weibo.com/top/summary", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "s.weibo.com" + ], + "entry_path": "/weibo", + "stable_text_anchors": [], + "identity_query_keys": [ + "q" + ], + "entry_scope": { + "html_selector": "#pl_top_realtimehot tbody tr td.td-02 a[href]", + "start_pattern": "序号[^\\n]*关键词", + "end_pattern": "^[ \\t]*(?:\\[关于微博\\]\\(|关于微博[ \\t]+微博帮助)" + }, + "name_validation": "nonempty" + } + }, + "artifact": "序号 关键词\n[条目0](https://s.weibo.com/weibo?q=newone)\n[条目1](https://s.weibo.com/weibo?q=newtwo)", + "ok": true, + "expected_task_pass": true + }, + { + "id": "arabic-currency-visible-format-prefix", + "category": "format-normalization", + "root_cause": "arabic_currency_order_and_direction_marks", + "failure_mode": "false-negative", + "case": { + "url": "https://www.amazon.ae/s?k=%D9%87%D8%A7%D8%AA%D9%81&language=ar_AE", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 1, + "hosts": [ + "www.amazon.ae" + ], + "entry_path": "/(?:[^/]+/)*(?:dp|gp/aw/d)/[A-Z0-9]{10}(?:/ref=[^/?#]+)?", + "stable_text_anchors": [], + "page_patterns": [ + "(?s)(?:(?:درهم[\\s\\u061c\\u200e\\u200f\\u202a-\\u202e\\u2066-\\u2069]*[0-9]+(?:[.,][0-9]+)*|[0-9]+(?:[.,][0-9]+)*[\\s\\u061c\\u200e\\u200f\\u202a-\\u202e\\u2066-\\u2069]*درهم).*?){5}" + ], + "entry_identity_pattern": "/(?:[^/]+/)*(?:dp|gp/aw/d)/([A-Z0-9]{10})(?:/|$)", + "name_validation": "nonempty", + "entry_scope": { + "html_selector": "[data-component-type=\"s-search-result\"][data-asin]", + "title_selector": "a h2", + "link_selector": "a:has(h2)" + } + } + }, + "artifact": "[Phone](https://www.amazon.ae/dp/B000000001)\nدرهم 659.99\nدرهم 659.99\nدرهم 659.99\nدرهم 659.99\nدرهم 659.99\n", + "ok": true, + "expected_task_pass": true + }, + { + "id": "arabic-currency-visible-format-suffix", + "category": "format-normalization", + "root_cause": "arabic_currency_order_and_direction_marks", + "failure_mode": "false-negative", + "case": { + "url": "https://www.amazon.ae/s?k=%D9%87%D8%A7%D8%AA%D9%81&language=ar_AE", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 1, + "hosts": [ + "www.amazon.ae" + ], + "entry_path": "/(?:[^/]+/)*(?:dp|gp/aw/d)/[A-Z0-9]{10}(?:/ref=[^/?#]+)?", + "stable_text_anchors": [], + "page_patterns": [ + "(?s)(?:(?:درهم[\\s\\u061c\\u200e\\u200f\\u202a-\\u202e\\u2066-\\u2069]*[0-9]+(?:[.,][0-9]+)*|[0-9]+(?:[.,][0-9]+)*[\\s\\u061c\\u200e\\u200f\\u202a-\\u202e\\u2066-\\u2069]*درهم).*?){5}" + ], + "entry_identity_pattern": "/(?:[^/]+/)*(?:dp|gp/aw/d)/([A-Z0-9]{10})(?:/|$)", + "name_validation": "nonempty", + "entry_scope": { + "html_selector": "[data-component-type=\"s-search-result\"][data-asin]", + "title_selector": "a h2", + "link_selector": "a:has(h2)" + } + } + }, + "artifact": "[Phone](https://www.amazon.ae/dp/B000000001)\n659.99 درهم\n659.99 درهم\n659.99 درهم\n659.99 درهم\n659.99 درهم\n", + "ok": true, + "expected_task_pass": true + }, + { + "id": "arabic-currency-visible-format-direction-marks", + "category": "format-normalization", + "root_cause": "arabic_currency_order_and_direction_marks", + "failure_mode": "false-negative", + "case": { + "url": "https://www.amazon.ae/s?k=%D9%87%D8%A7%D8%AA%D9%81&language=ar_AE", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 1, + "hosts": [ + "www.amazon.ae" + ], + "entry_path": "/(?:[^/]+/)*(?:dp|gp/aw/d)/[A-Z0-9]{10}(?:/ref=[^/?#]+)?", + "stable_text_anchors": [], + "page_patterns": [ + "(?s)(?:(?:درهم[\\s\\u061c\\u200e\\u200f\\u202a-\\u202e\\u2066-\\u2069]*[0-9]+(?:[.,][0-9]+)*|[0-9]+(?:[.,][0-9]+)*[\\s\\u061c\\u200e\\u200f\\u202a-\\u202e\\u2066-\\u2069]*درهم).*?){5}" + ], + "entry_identity_pattern": "/(?:[^/]+/)*(?:dp|gp/aw/d)/([A-Z0-9]{10})(?:/|$)", + "name_validation": "nonempty", + "entry_scope": { + "html_selector": "[data-component-type=\"s-search-result\"][data-asin]", + "title_selector": "a h2", + "link_selector": "a:has(h2)" + } + } + }, + "artifact": "[Phone](https://www.amazon.ae/dp/B000000001)\n‏659.99 درهم‎\n‏659.99 درهم‎\n‏659.99 درهم‎\n‏659.99 درهم‎\n‏659.99 درهم‎\n", + "ok": true, + "expected_task_pass": true + }, + { + "id": "arabic-currency-visible-format-prefix-direction", + "category": "format-normalization", + "root_cause": "arabic_currency_order_and_direction_marks", + "failure_mode": "false-negative", + "case": { + "url": "https://www.amazon.ae/s?k=%D9%87%D8%A7%D8%AA%D9%81&language=ar_AE", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 1, + "hosts": [ + "www.amazon.ae" + ], + "entry_path": "/(?:[^/]+/)*(?:dp|gp/aw/d)/[A-Z0-9]{10}(?:/ref=[^/?#]+)?", + "stable_text_anchors": [], + "page_patterns": [ + "(?s)(?:(?:درهم[\\s\\u061c\\u200e\\u200f\\u202a-\\u202e\\u2066-\\u2069]*[0-9]+(?:[.,][0-9]+)*|[0-9]+(?:[.,][0-9]+)*[\\s\\u061c\\u200e\\u200f\\u202a-\\u202e\\u2066-\\u2069]*درهم).*?){5}" + ], + "entry_identity_pattern": "/(?:[^/]+/)*(?:dp|gp/aw/d)/([A-Z0-9]{10})(?:/|$)", + "name_validation": "nonempty", + "entry_scope": { + "html_selector": "[data-component-type=\"s-search-result\"][data-asin]", + "title_selector": "a h2", + "link_selector": "a:has(h2)" + } + } + }, + "artifact": "[Phone](https://www.amazon.ae/dp/B000000001)\nدرهم‎659.99‏\nدرهم‎659.99‏\nدرهم‎659.99‏\nدرهم‎659.99‏\nدرهم‎659.99‏\n", + "ok": true, + "expected_task_pass": true + }, + { + "id": "arabic-currency-visible-format-four-only", + "category": "format-normalization", + "root_cause": "arabic_currency_order_and_direction_marks", + "failure_mode": "false-positive", + "case": { + "url": "https://www.amazon.ae/s?k=%D9%87%D8%A7%D8%AA%D9%81&language=ar_AE", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 1, + "hosts": [ + "www.amazon.ae" + ], + "entry_path": "/(?:[^/]+/)*(?:dp|gp/aw/d)/[A-Z0-9]{10}(?:/ref=[^/?#]+)?", + "stable_text_anchors": [], + "page_patterns": [ + "(?s)(?:(?:درهم[\\s\\u061c\\u200e\\u200f\\u202a-\\u202e\\u2066-\\u2069]*[0-9]+(?:[.,][0-9]+)*|[0-9]+(?:[.,][0-9]+)*[\\s\\u061c\\u200e\\u200f\\u202a-\\u202e\\u2066-\\u2069]*درهم).*?){5}" + ], + "entry_identity_pattern": "/(?:[^/]+/)*(?:dp|gp/aw/d)/([A-Z0-9]{10})(?:/|$)", + "name_validation": "nonempty", + "entry_scope": { + "html_selector": "[data-component-type=\"s-search-result\"][data-asin]", + "title_selector": "a h2", + "link_selector": "a:has(h2)" + } + } + }, + "artifact": "[Phone](https://www.amazon.ae/dp/B000000001)\n659.99 درهم\n659.99 درهم\n659.99 درهم\n659.99 درهم\n", + "ok": true, + "expected_task_pass": false + }, + { + "id": "arabic-currency-visible-format-unrelated-digits", + "category": "format-normalization", + "root_cause": "arabic_currency_order_and_direction_marks", + "failure_mode": "false-positive", + "case": { + "url": "https://www.amazon.ae/s?k=%D9%87%D8%A7%D8%AA%D9%81&language=ar_AE", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 1, + "hosts": [ + "www.amazon.ae" + ], + "entry_path": "/(?:[^/]+/)*(?:dp|gp/aw/d)/[A-Z0-9]{10}(?:/ref=[^/?#]+)?", + "stable_text_anchors": [], + "page_patterns": [ + "(?s)(?:(?:درهم[\\s\\u061c\\u200e\\u200f\\u202a-\\u202e\\u2066-\\u2069]*[0-9]+(?:[.,][0-9]+)*|[0-9]+(?:[.,][0-9]+)*[\\s\\u061c\\u200e\\u200f\\u202a-\\u202e\\u2066-\\u2069]*درهم).*?){5}" + ], + "entry_identity_pattern": "/(?:[^/]+/)*(?:dp|gp/aw/d)/([A-Z0-9]{10})(?:/|$)", + "name_validation": "nonempty", + "entry_scope": { + "html_selector": "[data-component-type=\"s-search-result\"][data-asin]", + "title_selector": "a h2", + "link_selector": "a:has(h2)" + } + } + }, + "artifact": "[Phone](https://www.amazon.ae/dp/B000000001)\nدرهم other text 659.99 unrelated\nدرهم other text 659.99 unrelated\nدرهم other text 659.99 unrelated\nدرهم other text 659.99 unrelated\nدرهم other text 659.99 unrelated\n", + "ok": true, + "expected_task_pass": false + }, + { + "id": "arabic-currency-visible-format-currency-only", + "category": "format-normalization", + "root_cause": "arabic_currency_order_and_direction_marks", + "failure_mode": "false-positive", + "case": { + "url": "https://www.amazon.ae/s?k=%D9%87%D8%A7%D8%AA%D9%81&language=ar_AE", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 1, + "hosts": [ + "www.amazon.ae" + ], + "entry_path": "/(?:[^/]+/)*(?:dp|gp/aw/d)/[A-Z0-9]{10}(?:/ref=[^/?#]+)?", + "stable_text_anchors": [], + "page_patterns": [ + "(?s)(?:(?:درهم[\\s\\u061c\\u200e\\u200f\\u202a-\\u202e\\u2066-\\u2069]*[0-9]+(?:[.,][0-9]+)*|[0-9]+(?:[.,][0-9]+)*[\\s\\u061c\\u200e\\u200f\\u202a-\\u202e\\u2066-\\u2069]*درهم).*?){5}" + ], + "entry_identity_pattern": "/(?:[^/]+/)*(?:dp|gp/aw/d)/([A-Z0-9]{10})(?:/|$)", + "name_validation": "nonempty", + "entry_scope": { + "html_selector": "[data-component-type=\"s-search-result\"][data-asin]", + "title_selector": "a h2", + "link_selector": "a:has(h2)" + } + } + }, + "artifact": "[Phone](https://www.amazon.ae/dp/B000000001)\nدرهم\nدرهم\nدرهم\nدرهم\nدرهم\n", + "ok": true, + "expected_task_pass": false + }, + { + "id": "dynamic-014-en-stable-rsuite-table-blank-lines-complete", + "category": "structured-extraction", + "root_cause": "table_multiline_whitespace_and_scope_boundaries", + "failure_mode": "false-negative", + "case": { + "url": "https://rsuitejs.com/components/table/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 2, + "stable_text_anchors": [], + "table_columns": [ + { + "name": "id", + "label": "Id", + "type": "integer", + "minimum": 1 + }, + { + "name": "first_name", + "label": "First Name", + "type": "text" + }, + { + "name": "last_name", + "label": "Last Name", + "type": "text" + }, + { + "name": "gender", + "label": "Gender", + "type": "text", + "pattern": "(?i)female|male" + }, + { + "name": "age", + "label": "Age", + "type": "integer", + "minimum": 0, + "maximum": 150 + }, + { + "name": "postcode", + "label": "Postcode", + "type": "token", + "pattern": "\\d{5}(?:-\\d{4})?" + }, + { + "name": "email", + "label": "Email", + "type": "text", + "pattern": "[^\\s@]+@[^\\s@]+\\.[^\\s@]+" } - }, - "artifact": "Id\n\nFirst Name\n\nLast Name\n\nGender\n\nAge\n\nPostcode\n\nEmail\n\n\n1\n\nCynthia\n\nWalter\n\nfemale\n\n32\n\n59447-0258\n\nVan.Tromp@hotmail.com\n\nEdit\n\n2\n\nMarcella\n\nProhaska\n\nfemale\n\n23\n\n81933-1082\n\nGeo18@yahoo.com\n\nEdit\n\nLoading...", - "ok": true, - "expected_task_pass": true - }, - { - "id": "dynamic-014-en-stable-rsuite-table-blank-lines-missing-field", - "category": "structured-extraction", - "root_cause": "table_multiline_whitespace_and_scope_boundaries", - "failure_mode": "false-positive", - "case": { - "url": "https://rsuitejs.com/components/table/", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "table_row", - "minimum_records": 2, - "stable_text_anchors": [], - "table_columns": [ - { - "name": "id", - "label": "Id", - "type": "integer", - "minimum": 1 - }, - { - "name": "first_name", - "label": "First Name", - "type": "text" - }, - { - "name": "last_name", - "label": "Last Name", - "type": "text" - }, - { - "name": "gender", - "label": "Gender", - "type": "text", - "pattern": "(?i)female|male" - }, - { - "name": "age", - "label": "Age", - "type": "integer", - "minimum": 0, - "maximum": 150 - }, - { - "name": "postcode", - "label": "Postcode", - "type": "token", - "pattern": "\\d{5}(?:-\\d{4})?" - }, - { - "name": "email", - "label": "Email", - "type": "text", - "pattern": "[^\\s@]+@[^\\s@]+\\.[^\\s@]+" - } - ], - "record_identity_fields": [ - "id" - ], - "table_scope": { - "root_selector": ".rs-table", - "root_index": 0, - "row_selector": "[role=row]:not(.rs-table-row-header)", - "cell_selector": "[role=gridcell]:not([aria-colindex=\"8\"]) .rs-table-cell-content", - "start_pattern": "(?im)^(?=[ \\t|]*Id(?:[ \\t|]+|\\n[ \\t\\n]*)First Name)", - "end_pattern": "(?im)^[ \\t#]*(?:Appearance|Loading\\.\\.\\.)[ \\t]*(?:\\[.*)?$", - "plain_row_pattern": "(?P\\d+)\\s+(?P[^\\d\\n]+?)\\s+(?P[^\\d\\n]+?)\\s+(?P(?i:female|male))\\s+(?P\\d+)\\s+(?P\\d{5}(?:-\\d{4})?)\\s+(?P[^\\s@]+@[^\\s@]+\\.[^\\s@]+)(?:\\s+Edit)?", - "plain_multiline_row_pattern": "(?m)^(?P\\d+)\\n[ \\t\\n]*(?P[^\\d\\n]+)\\n[ \\t\\n]*(?P[^\\d\\n]+)\\n[ \\t\\n]*(?P(?i:female|male))\\n[ \\t\\n]*(?P\\d+)\\n[ \\t\\n]*(?P\\d{5}(?:-\\d{4})?)\\n[ \\t\\n]*(?P[^\\s@]+@[^\\s@]+\\.[^\\s@]+)\\n[ \\t\\n]*Edit[ \\t]*(?=\\n|$)" - } + ], + "record_identity_fields": [ + "id" + ], + "table_scope": { + "root_selector": ".rs-table", + "root_index": 0, + "row_selector": "[role=row]:not(.rs-table-row-header)", + "cell_selector": "[role=gridcell]:not([aria-colindex=\"8\"]) .rs-table-cell-content", + "start_pattern": "(?im)^(?=[ \\t|]*Id(?:[ \\t|]+|\\n[ \\t\\n]*)First Name)", + "end_pattern": "(?im)^[ \\t#]*(?:Appearance|Loading\\.\\.\\.)[ \\t]*(?:\\[.*)?$", + "plain_row_pattern": "(?P\\d+)\\s+(?P[^\\d\\n]+?)\\s+(?P[^\\d\\n]+?)\\s+(?P(?i:female|male))\\s+(?P\\d+)\\s+(?P\\d{5}(?:-\\d{4})?)\\s+(?P[^\\s@]+@[^\\s@]+\\.[^\\s@]+)(?:\\s+Edit)?", + "plain_multiline_row_pattern": "(?m)^(?P\\d+)\\n[ \\t\\n]*(?P[^\\d\\n]+)\\n[ \\t\\n]*(?P[^\\d\\n]+)\\n[ \\t\\n]*(?P(?i:female|male))\\n[ \\t\\n]*(?P\\d+)\\n[ \\t\\n]*(?P\\d{5}(?:-\\d{4})?)\\n[ \\t\\n]*(?P[^\\s@]+@[^\\s@]+\\.[^\\s@]+)\\n[ \\t\\n]*Edit[ \\t]*(?=\\n|$)" + } + } + }, + "artifact": "Id\n\nFirst Name\n\nLast Name\n\nGender\n\nAge\n\nPostcode\n\nEmail\n\n\n1\n\nCynthia\n\nWalter\n\nfemale\n\n32\n\n59447-0258\n\nVan.Tromp@hotmail.com\n\nEdit\n\n2\n\nMarcella\n\nProhaska\n\nfemale\n\n23\n\n81933-1082\n\nGeo18@yahoo.com\n\nEdit\n\nLoading...", + "ok": true, + "expected_task_pass": true + }, + { + "id": "dynamic-014-en-stable-rsuite-table-blank-lines-missing-field", + "category": "structured-extraction", + "root_cause": "table_multiline_whitespace_and_scope_boundaries", + "failure_mode": "false-positive", + "case": { + "url": "https://rsuitejs.com/components/table/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 2, + "stable_text_anchors": [], + "table_columns": [ + { + "name": "id", + "label": "Id", + "type": "integer", + "minimum": 1 + }, + { + "name": "first_name", + "label": "First Name", + "type": "text" + }, + { + "name": "last_name", + "label": "Last Name", + "type": "text" + }, + { + "name": "gender", + "label": "Gender", + "type": "text", + "pattern": "(?i)female|male" + }, + { + "name": "age", + "label": "Age", + "type": "integer", + "minimum": 0, + "maximum": 150 + }, + { + "name": "postcode", + "label": "Postcode", + "type": "token", + "pattern": "\\d{5}(?:-\\d{4})?" + }, + { + "name": "email", + "label": "Email", + "type": "text", + "pattern": "[^\\s@]+@[^\\s@]+\\.[^\\s@]+" } - }, - "artifact": "Id\n\nFirst Name\n\nLast Name\n\nGender\n\nAge\n\nPostcode\n\nEmail\n\n\n1\n\nCynthia\n\nWalter\n\nfemale\n\n32\n\n59447-0258\n\nEdit\n\n2\n\nMarcella\n\nProhaska\n\nfemale\n\n23\n\n81933-1082\n\nGeo18@yahoo.com\n\nEdit\n\nLoading...", - "ok": true, - "expected_task_pass": false - }, - { - "id": "dynamic-014-en-stable-rsuite-table-blank-lines-duplicated", - "category": "structured-extraction", - "root_cause": "table_multiline_whitespace_and_scope_boundaries", - "failure_mode": "false-positive", - "case": { - "url": "https://rsuitejs.com/components/table/", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "table_row", - "minimum_records": 2, - "stable_text_anchors": [], - "table_columns": [ - { - "name": "id", - "label": "Id", - "type": "integer", - "minimum": 1 - }, - { - "name": "first_name", - "label": "First Name", - "type": "text" - }, - { - "name": "last_name", - "label": "Last Name", - "type": "text" - }, - { - "name": "gender", - "label": "Gender", - "type": "text", - "pattern": "(?i)female|male" - }, - { - "name": "age", - "label": "Age", - "type": "integer", - "minimum": 0, - "maximum": 150 - }, - { - "name": "postcode", - "label": "Postcode", - "type": "token", - "pattern": "\\d{5}(?:-\\d{4})?" - }, - { - "name": "email", - "label": "Email", - "type": "text", - "pattern": "[^\\s@]+@[^\\s@]+\\.[^\\s@]+" - } - ], - "record_identity_fields": [ - "id" - ], - "table_scope": { - "root_selector": ".rs-table", - "root_index": 0, - "row_selector": "[role=row]:not(.rs-table-row-header)", - "cell_selector": "[role=gridcell]:not([aria-colindex=\"8\"]) .rs-table-cell-content", - "start_pattern": "(?im)^(?=[ \\t|]*Id(?:[ \\t|]+|\\n[ \\t\\n]*)First Name)", - "end_pattern": "(?im)^[ \\t#]*(?:Appearance|Loading\\.\\.\\.)[ \\t]*(?:\\[.*)?$", - "plain_row_pattern": "(?P\\d+)\\s+(?P[^\\d\\n]+?)\\s+(?P[^\\d\\n]+?)\\s+(?P(?i:female|male))\\s+(?P\\d+)\\s+(?P\\d{5}(?:-\\d{4})?)\\s+(?P[^\\s@]+@[^\\s@]+\\.[^\\s@]+)(?:\\s+Edit)?", - "plain_multiline_row_pattern": "(?m)^(?P\\d+)\\n[ \\t\\n]*(?P[^\\d\\n]+)\\n[ \\t\\n]*(?P[^\\d\\n]+)\\n[ \\t\\n]*(?P(?i:female|male))\\n[ \\t\\n]*(?P\\d+)\\n[ \\t\\n]*(?P\\d{5}(?:-\\d{4})?)\\n[ \\t\\n]*(?P[^\\s@]+@[^\\s@]+\\.[^\\s@]+)\\n[ \\t\\n]*Edit[ \\t]*(?=\\n|$)" - } + ], + "record_identity_fields": [ + "id" + ], + "table_scope": { + "root_selector": ".rs-table", + "root_index": 0, + "row_selector": "[role=row]:not(.rs-table-row-header)", + "cell_selector": "[role=gridcell]:not([aria-colindex=\"8\"]) .rs-table-cell-content", + "start_pattern": "(?im)^(?=[ \\t|]*Id(?:[ \\t|]+|\\n[ \\t\\n]*)First Name)", + "end_pattern": "(?im)^[ \\t#]*(?:Appearance|Loading\\.\\.\\.)[ \\t]*(?:\\[.*)?$", + "plain_row_pattern": "(?P\\d+)\\s+(?P[^\\d\\n]+?)\\s+(?P[^\\d\\n]+?)\\s+(?P(?i:female|male))\\s+(?P\\d+)\\s+(?P\\d{5}(?:-\\d{4})?)\\s+(?P[^\\s@]+@[^\\s@]+\\.[^\\s@]+)(?:\\s+Edit)?", + "plain_multiline_row_pattern": "(?m)^(?P\\d+)\\n[ \\t\\n]*(?P[^\\d\\n]+)\\n[ \\t\\n]*(?P[^\\d\\n]+)\\n[ \\t\\n]*(?P(?i:female|male))\\n[ \\t\\n]*(?P\\d+)\\n[ \\t\\n]*(?P\\d{5}(?:-\\d{4})?)\\n[ \\t\\n]*(?P[^\\s@]+@[^\\s@]+\\.[^\\s@]+)\\n[ \\t\\n]*Edit[ \\t]*(?=\\n|$)" + } + } + }, + "artifact": "Id\n\nFirst Name\n\nLast Name\n\nGender\n\nAge\n\nPostcode\n\nEmail\n\n\n1\n\nCynthia\n\nWalter\n\nfemale\n\n32\n\n59447-0258\n\nEdit\n\n2\n\nMarcella\n\nProhaska\n\nfemale\n\n23\n\n81933-1082\n\nGeo18@yahoo.com\n\nEdit\n\nLoading...", + "ok": true, + "expected_task_pass": false + }, + { + "id": "dynamic-014-en-stable-rsuite-table-blank-lines-duplicated", + "category": "structured-extraction", + "root_cause": "table_multiline_whitespace_and_scope_boundaries", + "failure_mode": "false-positive", + "case": { + "url": "https://rsuitejs.com/components/table/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 2, + "stable_text_anchors": [], + "table_columns": [ + { + "name": "id", + "label": "Id", + "type": "integer", + "minimum": 1 + }, + { + "name": "first_name", + "label": "First Name", + "type": "text" + }, + { + "name": "last_name", + "label": "Last Name", + "type": "text" + }, + { + "name": "gender", + "label": "Gender", + "type": "text", + "pattern": "(?i)female|male" + }, + { + "name": "age", + "label": "Age", + "type": "integer", + "minimum": 0, + "maximum": 150 + }, + { + "name": "postcode", + "label": "Postcode", + "type": "token", + "pattern": "\\d{5}(?:-\\d{4})?" + }, + { + "name": "email", + "label": "Email", + "type": "text", + "pattern": "[^\\s@]+@[^\\s@]+\\.[^\\s@]+" } - }, - "artifact": "Id\n\nFirst Name\n\nLast Name\n\nGender\n\nAge\n\nPostcode\n\nEmail\n\n\n1\n\nCynthia\n\nWalter\n\nfemale\n\n32\n\n59447-0258\n\nVan.Tromp@hotmail.com\n\nEdit\n\n1\n\nCynthia\n\nWalter\n\nfemale\n\n32\n\n59447-0258\n\nVan.Tromp@hotmail.com\n\nEdit\n\nLoading...", - "ok": true, - "expected_task_pass": false - }, - { - "id": "dynamic-014-en-stable-rsuite-table-blank-lines-one-record", - "category": "structured-extraction", - "root_cause": "table_multiline_whitespace_and_scope_boundaries", - "failure_mode": "false-positive", - "case": { - "url": "https://rsuitejs.com/components/table/", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "table_row", - "minimum_records": 2, - "stable_text_anchors": [], - "table_columns": [ - { - "name": "id", - "label": "Id", - "type": "integer", - "minimum": 1 - }, - { - "name": "first_name", - "label": "First Name", - "type": "text" - }, - { - "name": "last_name", - "label": "Last Name", - "type": "text" - }, - { - "name": "gender", - "label": "Gender", - "type": "text", - "pattern": "(?i)female|male" - }, - { - "name": "age", - "label": "Age", - "type": "integer", - "minimum": 0, - "maximum": 150 - }, - { - "name": "postcode", - "label": "Postcode", - "type": "token", - "pattern": "\\d{5}(?:-\\d{4})?" - }, - { - "name": "email", - "label": "Email", - "type": "text", - "pattern": "[^\\s@]+@[^\\s@]+\\.[^\\s@]+" - } - ], - "record_identity_fields": [ - "id" - ], - "table_scope": { - "root_selector": ".rs-table", - "root_index": 0, - "row_selector": "[role=row]:not(.rs-table-row-header)", - "cell_selector": "[role=gridcell]:not([aria-colindex=\"8\"]) .rs-table-cell-content", - "start_pattern": "(?im)^(?=[ \\t|]*Id(?:[ \\t|]+|\\n[ \\t\\n]*)First Name)", - "end_pattern": "(?im)^[ \\t#]*(?:Appearance|Loading\\.\\.\\.)[ \\t]*(?:\\[.*)?$", - "plain_row_pattern": "(?P\\d+)\\s+(?P[^\\d\\n]+?)\\s+(?P[^\\d\\n]+?)\\s+(?P(?i:female|male))\\s+(?P\\d+)\\s+(?P\\d{5}(?:-\\d{4})?)\\s+(?P[^\\s@]+@[^\\s@]+\\.[^\\s@]+)(?:\\s+Edit)?", - "plain_multiline_row_pattern": "(?m)^(?P\\d+)\\n[ \\t\\n]*(?P[^\\d\\n]+)\\n[ \\t\\n]*(?P[^\\d\\n]+)\\n[ \\t\\n]*(?P(?i:female|male))\\n[ \\t\\n]*(?P\\d+)\\n[ \\t\\n]*(?P\\d{5}(?:-\\d{4})?)\\n[ \\t\\n]*(?P[^\\s@]+@[^\\s@]+\\.[^\\s@]+)\\n[ \\t\\n]*Edit[ \\t]*(?=\\n|$)" - } + ], + "record_identity_fields": [ + "id" + ], + "table_scope": { + "root_selector": ".rs-table", + "root_index": 0, + "row_selector": "[role=row]:not(.rs-table-row-header)", + "cell_selector": "[role=gridcell]:not([aria-colindex=\"8\"]) .rs-table-cell-content", + "start_pattern": "(?im)^(?=[ \\t|]*Id(?:[ \\t|]+|\\n[ \\t\\n]*)First Name)", + "end_pattern": "(?im)^[ \\t#]*(?:Appearance|Loading\\.\\.\\.)[ \\t]*(?:\\[.*)?$", + "plain_row_pattern": "(?P\\d+)\\s+(?P[^\\d\\n]+?)\\s+(?P[^\\d\\n]+?)\\s+(?P(?i:female|male))\\s+(?P\\d+)\\s+(?P\\d{5}(?:-\\d{4})?)\\s+(?P[^\\s@]+@[^\\s@]+\\.[^\\s@]+)(?:\\s+Edit)?", + "plain_multiline_row_pattern": "(?m)^(?P\\d+)\\n[ \\t\\n]*(?P[^\\d\\n]+)\\n[ \\t\\n]*(?P[^\\d\\n]+)\\n[ \\t\\n]*(?P(?i:female|male))\\n[ \\t\\n]*(?P\\d+)\\n[ \\t\\n]*(?P\\d{5}(?:-\\d{4})?)\\n[ \\t\\n]*(?P[^\\s@]+@[^\\s@]+\\.[^\\s@]+)\\n[ \\t\\n]*Edit[ \\t]*(?=\\n|$)" + } + } + }, + "artifact": "Id\n\nFirst Name\n\nLast Name\n\nGender\n\nAge\n\nPostcode\n\nEmail\n\n\n1\n\nCynthia\n\nWalter\n\nfemale\n\n32\n\n59447-0258\n\nVan.Tromp@hotmail.com\n\nEdit\n\n1\n\nCynthia\n\nWalter\n\nfemale\n\n32\n\n59447-0258\n\nVan.Tromp@hotmail.com\n\nEdit\n\nLoading...", + "ok": true, + "expected_task_pass": false + }, + { + "id": "dynamic-014-en-stable-rsuite-table-blank-lines-one-record", + "category": "structured-extraction", + "root_cause": "table_multiline_whitespace_and_scope_boundaries", + "failure_mode": "false-positive", + "case": { + "url": "https://rsuitejs.com/components/table/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 2, + "stable_text_anchors": [], + "table_columns": [ + { + "name": "id", + "label": "Id", + "type": "integer", + "minimum": 1 + }, + { + "name": "first_name", + "label": "First Name", + "type": "text" + }, + { + "name": "last_name", + "label": "Last Name", + "type": "text" + }, + { + "name": "gender", + "label": "Gender", + "type": "text", + "pattern": "(?i)female|male" + }, + { + "name": "age", + "label": "Age", + "type": "integer", + "minimum": 0, + "maximum": 150 + }, + { + "name": "postcode", + "label": "Postcode", + "type": "token", + "pattern": "\\d{5}(?:-\\d{4})?" + }, + { + "name": "email", + "label": "Email", + "type": "text", + "pattern": "[^\\s@]+@[^\\s@]+\\.[^\\s@]+" } - }, - "artifact": "Id\n\nFirst Name\n\nLast Name\n\nGender\n\nAge\n\nPostcode\n\nEmail\n\n\n1\n\nCynthia\n\nWalter\n\nfemale\n\n32\n\n59447-0258\n\nVan.Tromp@hotmail.com\n\nEdit\n\nLoading...", - "ok": true, - "expected_task_pass": false - }, - { - "id": "dynamic-025-025-tabulator-vertical-virtual-dom-blank-lines-complete", - "category": "structured-extraction", - "root_cause": "table_multiline_whitespace_and_scope_boundaries", - "failure_mode": "false-negative", - "case": { - "url": "https://www.tabulator.info/examples/6.x/", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "table_row", - "minimum_records": 2, - "stable_text_anchors": [], - "table_columns": [ - { - "name": "id", - "label": "ID", - "type": "integer", - "minimum": 1 - }, - { - "name": "name", - "label": "Name", - "type": "text" - }, - { - "name": "progress", - "label": "Progress", - "type": "integer", - "minimum": 0, - "maximum": 100 - }, - { - "name": "gender", - "label": "Gender", - "type": "text" - }, - { - "name": "rating", - "label": "Rating", - "type": "integer", - "minimum": 0, - "maximum": 5 - }, - { - "name": "color", - "label": "Favourite Color", - "type": "text" - }, - { - "name": "birth_date", - "label": "Date Of Birth", - "type": "date_dmy" - } - ], - "record_identity_fields": [ - "id" - ], - "table_scope": { - "row_selector": "#example-table-virtual .tabulator-row", - "cell_selector": ".tabulator-cell", - "start_pattern": "The table below has 1000 rows[^\\n]*", - "end_pattern": "View Source", - "plain_row_pattern": "(?P\\d+)\\s*(?P[^\\d\\n]+?)\\s*(?P\\d{1,3})\\s*(?Pfemale|male)\\s*(?P\\d)\\s*(?P[^\\d\\n]+?)\\s*(?P\\d{2}/\\d{2}/\\d{4})", - "plain_multiline_row_pattern": "(?m)^(?P\\d+)\\n[ \\t\\n]*(?P[^\\d\\n]+)\\n[ \\t\\n]*(?P\\d{1,3})\\n[ \\t\\n]*(?Pfemale|male)\\n[ \\t\\n]*(?P\\d)\\n[ \\t\\n]*(?P[^\\d\\n]+)\\n[ \\t\\n]*(?P\\d{2}/\\d{2}/\\d{4})(?=\\n|$)" - } + ], + "record_identity_fields": [ + "id" + ], + "table_scope": { + "root_selector": ".rs-table", + "root_index": 0, + "row_selector": "[role=row]:not(.rs-table-row-header)", + "cell_selector": "[role=gridcell]:not([aria-colindex=\"8\"]) .rs-table-cell-content", + "start_pattern": "(?im)^(?=[ \\t|]*Id(?:[ \\t|]+|\\n[ \\t\\n]*)First Name)", + "end_pattern": "(?im)^[ \\t#]*(?:Appearance|Loading\\.\\.\\.)[ \\t]*(?:\\[.*)?$", + "plain_row_pattern": "(?P\\d+)\\s+(?P[^\\d\\n]+?)\\s+(?P[^\\d\\n]+?)\\s+(?P(?i:female|male))\\s+(?P\\d+)\\s+(?P\\d{5}(?:-\\d{4})?)\\s+(?P[^\\s@]+@[^\\s@]+\\.[^\\s@]+)(?:\\s+Edit)?", + "plain_multiline_row_pattern": "(?m)^(?P\\d+)\\n[ \\t\\n]*(?P[^\\d\\n]+)\\n[ \\t\\n]*(?P[^\\d\\n]+)\\n[ \\t\\n]*(?P(?i:female|male))\\n[ \\t\\n]*(?P\\d+)\\n[ \\t\\n]*(?P\\d{5}(?:-\\d{4})?)\\n[ \\t\\n]*(?P[^\\s@]+@[^\\s@]+\\.[^\\s@]+)\\n[ \\t\\n]*Edit[ \\t]*(?=\\n|$)" + } + } + }, + "artifact": "Id\n\nFirst Name\n\nLast Name\n\nGender\n\nAge\n\nPostcode\n\nEmail\n\n\n1\n\nCynthia\n\nWalter\n\nfemale\n\n32\n\n59447-0258\n\nVan.Tromp@hotmail.com\n\nEdit\n\nLoading...", + "ok": true, + "expected_task_pass": false + }, + { + "id": "dynamic-025-025-tabulator-vertical-virtual-dom-blank-lines-complete", + "category": "structured-extraction", + "root_cause": "table_multiline_whitespace_and_scope_boundaries", + "failure_mode": "false-negative", + "case": { + "url": "https://www.tabulator.info/examples/6.x/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 2, + "stable_text_anchors": [], + "table_columns": [ + { + "name": "id", + "label": "ID", + "type": "integer", + "minimum": 1 + }, + { + "name": "name", + "label": "Name", + "type": "text" + }, + { + "name": "progress", + "label": "Progress", + "type": "integer", + "minimum": 0, + "maximum": 100 + }, + { + "name": "gender", + "label": "Gender", + "type": "text" + }, + { + "name": "rating", + "label": "Rating", + "type": "integer", + "minimum": 0, + "maximum": 5 + }, + { + "name": "color", + "label": "Favourite Color", + "type": "text" + }, + { + "name": "birth_date", + "label": "Date Of Birth", + "type": "date_dmy" } - }, - "artifact": "The table below has 1000 rows, each row is loaded as it becomes visible as you scroll\n\n\n1\n\nMs. Darrick McLaughlin\n\n70\n\nfemale\n\n4\n\nivory\n\n30/12/2025\n\n2\n\nMrs. Monte Pouros\n\n53\n\nfemale\n\n4\n\nlavender\n\n24/03/2026\n\nView Source", - "ok": true, - "expected_task_pass": true - }, - { - "id": "dynamic-025-025-tabulator-vertical-virtual-dom-blank-lines-missing-field", - "category": "structured-extraction", - "root_cause": "table_multiline_whitespace_and_scope_boundaries", - "failure_mode": "false-positive", - "case": { - "url": "https://www.tabulator.info/examples/6.x/", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "table_row", - "minimum_records": 2, - "stable_text_anchors": [], - "table_columns": [ - { - "name": "id", - "label": "ID", - "type": "integer", - "minimum": 1 - }, - { - "name": "name", - "label": "Name", - "type": "text" - }, - { - "name": "progress", - "label": "Progress", - "type": "integer", - "minimum": 0, - "maximum": 100 - }, - { - "name": "gender", - "label": "Gender", - "type": "text" - }, - { - "name": "rating", - "label": "Rating", - "type": "integer", - "minimum": 0, - "maximum": 5 - }, - { - "name": "color", - "label": "Favourite Color", - "type": "text" - }, - { - "name": "birth_date", - "label": "Date Of Birth", - "type": "date_dmy" - } - ], - "record_identity_fields": [ - "id" - ], - "table_scope": { - "row_selector": "#example-table-virtual .tabulator-row", - "cell_selector": ".tabulator-cell", - "start_pattern": "The table below has 1000 rows[^\\n]*", - "end_pattern": "View Source", - "plain_row_pattern": "(?P\\d+)\\s*(?P[^\\d\\n]+?)\\s*(?P\\d{1,3})\\s*(?Pfemale|male)\\s*(?P\\d)\\s*(?P[^\\d\\n]+?)\\s*(?P\\d{2}/\\d{2}/\\d{4})", - "plain_multiline_row_pattern": "(?m)^(?P\\d+)\\n[ \\t\\n]*(?P[^\\d\\n]+)\\n[ \\t\\n]*(?P\\d{1,3})\\n[ \\t\\n]*(?Pfemale|male)\\n[ \\t\\n]*(?P\\d)\\n[ \\t\\n]*(?P[^\\d\\n]+)\\n[ \\t\\n]*(?P\\d{2}/\\d{2}/\\d{4})(?=\\n|$)" - } + ], + "record_identity_fields": [ + "id" + ], + "table_scope": { + "row_selector": "#example-table-virtual .tabulator-row", + "cell_selector": ".tabulator-cell", + "start_pattern": "The table below has 1000 rows[^\\n]*", + "end_pattern": "View Source", + "plain_row_pattern": "(?P\\d+)\\s*(?P[^\\d\\n]+?)\\s*(?P\\d{1,3})\\s*(?Pfemale|male)\\s*(?P\\d)\\s*(?P[^\\d\\n]+?)\\s*(?P\\d{2}/\\d{2}/\\d{4})", + "plain_multiline_row_pattern": "(?m)^(?P\\d+)\\n[ \\t\\n]*(?P[^\\d\\n]+)\\n[ \\t\\n]*(?P\\d{1,3})\\n[ \\t\\n]*(?Pfemale|male)\\n[ \\t\\n]*(?P\\d)\\n[ \\t\\n]*(?P[^\\d\\n]+)\\n[ \\t\\n]*(?P\\d{2}/\\d{2}/\\d{4})(?=\\n|$)" + } + } + }, + "artifact": "The table below has 1000 rows, each row is loaded as it becomes visible as you scroll\n\n\n1\n\nMs. Darrick McLaughlin\n\n70\n\nfemale\n\n4\n\nivory\n\n30/12/2025\n\n2\n\nMrs. Monte Pouros\n\n53\n\nfemale\n\n4\n\nlavender\n\n24/03/2026\n\nView Source", + "ok": true, + "expected_task_pass": true + }, + { + "id": "dynamic-025-025-tabulator-vertical-virtual-dom-blank-lines-missing-field", + "category": "structured-extraction", + "root_cause": "table_multiline_whitespace_and_scope_boundaries", + "failure_mode": "false-positive", + "case": { + "url": "https://www.tabulator.info/examples/6.x/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 2, + "stable_text_anchors": [], + "table_columns": [ + { + "name": "id", + "label": "ID", + "type": "integer", + "minimum": 1 + }, + { + "name": "name", + "label": "Name", + "type": "text" + }, + { + "name": "progress", + "label": "Progress", + "type": "integer", + "minimum": 0, + "maximum": 100 + }, + { + "name": "gender", + "label": "Gender", + "type": "text" + }, + { + "name": "rating", + "label": "Rating", + "type": "integer", + "minimum": 0, + "maximum": 5 + }, + { + "name": "color", + "label": "Favourite Color", + "type": "text" + }, + { + "name": "birth_date", + "label": "Date Of Birth", + "type": "date_dmy" } - }, - "artifact": "The table below has 1000 rows, each row is loaded as it becomes visible as you scroll\n\n\n1\n\nMs. Darrick McLaughlin\n\n70\n\nfemale\n\n4\n\nivory\n\n2\n\nMrs. Monte Pouros\n\n53\n\nfemale\n\n4\n\nlavender\n\n24/03/2026\n\nView Source", - "ok": true, - "expected_task_pass": false - }, - { - "id": "dynamic-025-025-tabulator-vertical-virtual-dom-blank-lines-duplicated", - "category": "structured-extraction", - "root_cause": "table_multiline_whitespace_and_scope_boundaries", - "failure_mode": "false-positive", - "case": { - "url": "https://www.tabulator.info/examples/6.x/", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "table_row", - "minimum_records": 2, - "stable_text_anchors": [], - "table_columns": [ - { - "name": "id", - "label": "ID", - "type": "integer", - "minimum": 1 - }, - { - "name": "name", - "label": "Name", - "type": "text" - }, - { - "name": "progress", - "label": "Progress", - "type": "integer", - "minimum": 0, - "maximum": 100 - }, - { - "name": "gender", - "label": "Gender", - "type": "text" - }, - { - "name": "rating", - "label": "Rating", - "type": "integer", - "minimum": 0, - "maximum": 5 - }, - { - "name": "color", - "label": "Favourite Color", - "type": "text" - }, - { - "name": "birth_date", - "label": "Date Of Birth", - "type": "date_dmy" - } - ], - "record_identity_fields": [ - "id" - ], - "table_scope": { - "row_selector": "#example-table-virtual .tabulator-row", - "cell_selector": ".tabulator-cell", - "start_pattern": "The table below has 1000 rows[^\\n]*", - "end_pattern": "View Source", - "plain_row_pattern": "(?P\\d+)\\s*(?P[^\\d\\n]+?)\\s*(?P\\d{1,3})\\s*(?Pfemale|male)\\s*(?P\\d)\\s*(?P[^\\d\\n]+?)\\s*(?P\\d{2}/\\d{2}/\\d{4})", - "plain_multiline_row_pattern": "(?m)^(?P\\d+)\\n[ \\t\\n]*(?P[^\\d\\n]+)\\n[ \\t\\n]*(?P\\d{1,3})\\n[ \\t\\n]*(?Pfemale|male)\\n[ \\t\\n]*(?P\\d)\\n[ \\t\\n]*(?P[^\\d\\n]+)\\n[ \\t\\n]*(?P\\d{2}/\\d{2}/\\d{4})(?=\\n|$)" - } + ], + "record_identity_fields": [ + "id" + ], + "table_scope": { + "row_selector": "#example-table-virtual .tabulator-row", + "cell_selector": ".tabulator-cell", + "start_pattern": "The table below has 1000 rows[^\\n]*", + "end_pattern": "View Source", + "plain_row_pattern": "(?P\\d+)\\s*(?P[^\\d\\n]+?)\\s*(?P\\d{1,3})\\s*(?Pfemale|male)\\s*(?P\\d)\\s*(?P[^\\d\\n]+?)\\s*(?P\\d{2}/\\d{2}/\\d{4})", + "plain_multiline_row_pattern": "(?m)^(?P\\d+)\\n[ \\t\\n]*(?P[^\\d\\n]+)\\n[ \\t\\n]*(?P\\d{1,3})\\n[ \\t\\n]*(?Pfemale|male)\\n[ \\t\\n]*(?P\\d)\\n[ \\t\\n]*(?P[^\\d\\n]+)\\n[ \\t\\n]*(?P\\d{2}/\\d{2}/\\d{4})(?=\\n|$)" + } + } + }, + "artifact": "The table below has 1000 rows, each row is loaded as it becomes visible as you scroll\n\n\n1\n\nMs. Darrick McLaughlin\n\n70\n\nfemale\n\n4\n\nivory\n\n2\n\nMrs. Monte Pouros\n\n53\n\nfemale\n\n4\n\nlavender\n\n24/03/2026\n\nView Source", + "ok": true, + "expected_task_pass": false + }, + { + "id": "dynamic-025-025-tabulator-vertical-virtual-dom-blank-lines-duplicated", + "category": "structured-extraction", + "root_cause": "table_multiline_whitespace_and_scope_boundaries", + "failure_mode": "false-positive", + "case": { + "url": "https://www.tabulator.info/examples/6.x/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 2, + "stable_text_anchors": [], + "table_columns": [ + { + "name": "id", + "label": "ID", + "type": "integer", + "minimum": 1 + }, + { + "name": "name", + "label": "Name", + "type": "text" + }, + { + "name": "progress", + "label": "Progress", + "type": "integer", + "minimum": 0, + "maximum": 100 + }, + { + "name": "gender", + "label": "Gender", + "type": "text" + }, + { + "name": "rating", + "label": "Rating", + "type": "integer", + "minimum": 0, + "maximum": 5 + }, + { + "name": "color", + "label": "Favourite Color", + "type": "text" + }, + { + "name": "birth_date", + "label": "Date Of Birth", + "type": "date_dmy" } - }, - "artifact": "The table below has 1000 rows, each row is loaded as it becomes visible as you scroll\n\n\n1\n\nMs. Darrick McLaughlin\n\n70\n\nfemale\n\n4\n\nivory\n\n30/12/2025\n\n1\n\nMs. Darrick McLaughlin\n\n70\n\nfemale\n\n4\n\nivory\n\n30/12/2025\n\nView Source", - "ok": true, - "expected_task_pass": false - }, - { - "id": "dynamic-025-025-tabulator-vertical-virtual-dom-blank-lines-one-record", - "category": "structured-extraction", - "root_cause": "table_multiline_whitespace_and_scope_boundaries", - "failure_mode": "false-positive", - "case": { - "url": "https://www.tabulator.info/examples/6.x/", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "table_row", - "minimum_records": 2, - "stable_text_anchors": [], - "table_columns": [ - { - "name": "id", - "label": "ID", - "type": "integer", - "minimum": 1 - }, - { - "name": "name", - "label": "Name", - "type": "text" - }, - { - "name": "progress", - "label": "Progress", - "type": "integer", - "minimum": 0, - "maximum": 100 - }, - { - "name": "gender", - "label": "Gender", - "type": "text" - }, - { - "name": "rating", - "label": "Rating", - "type": "integer", - "minimum": 0, - "maximum": 5 - }, - { - "name": "color", - "label": "Favourite Color", - "type": "text" - }, - { - "name": "birth_date", - "label": "Date Of Birth", - "type": "date_dmy" - } - ], - "record_identity_fields": [ - "id" - ], - "table_scope": { - "row_selector": "#example-table-virtual .tabulator-row", - "cell_selector": ".tabulator-cell", - "start_pattern": "The table below has 1000 rows[^\\n]*", - "end_pattern": "View Source", - "plain_row_pattern": "(?P\\d+)\\s*(?P[^\\d\\n]+?)\\s*(?P\\d{1,3})\\s*(?Pfemale|male)\\s*(?P\\d)\\s*(?P[^\\d\\n]+?)\\s*(?P\\d{2}/\\d{2}/\\d{4})", - "plain_multiline_row_pattern": "(?m)^(?P\\d+)\\n[ \\t\\n]*(?P[^\\d\\n]+)\\n[ \\t\\n]*(?P\\d{1,3})\\n[ \\t\\n]*(?Pfemale|male)\\n[ \\t\\n]*(?P\\d)\\n[ \\t\\n]*(?P[^\\d\\n]+)\\n[ \\t\\n]*(?P\\d{2}/\\d{2}/\\d{4})(?=\\n|$)" - } + ], + "record_identity_fields": [ + "id" + ], + "table_scope": { + "row_selector": "#example-table-virtual .tabulator-row", + "cell_selector": ".tabulator-cell", + "start_pattern": "The table below has 1000 rows[^\\n]*", + "end_pattern": "View Source", + "plain_row_pattern": "(?P\\d+)\\s*(?P[^\\d\\n]+?)\\s*(?P\\d{1,3})\\s*(?Pfemale|male)\\s*(?P\\d)\\s*(?P[^\\d\\n]+?)\\s*(?P\\d{2}/\\d{2}/\\d{4})", + "plain_multiline_row_pattern": "(?m)^(?P\\d+)\\n[ \\t\\n]*(?P[^\\d\\n]+)\\n[ \\t\\n]*(?P\\d{1,3})\\n[ \\t\\n]*(?Pfemale|male)\\n[ \\t\\n]*(?P\\d)\\n[ \\t\\n]*(?P[^\\d\\n]+)\\n[ \\t\\n]*(?P\\d{2}/\\d{2}/\\d{4})(?=\\n|$)" + } + } + }, + "artifact": "The table below has 1000 rows, each row is loaded as it becomes visible as you scroll\n\n\n1\n\nMs. Darrick McLaughlin\n\n70\n\nfemale\n\n4\n\nivory\n\n30/12/2025\n\n1\n\nMs. Darrick McLaughlin\n\n70\n\nfemale\n\n4\n\nivory\n\n30/12/2025\n\nView Source", + "ok": true, + "expected_task_pass": false + }, + { + "id": "dynamic-025-025-tabulator-vertical-virtual-dom-blank-lines-one-record", + "category": "structured-extraction", + "root_cause": "table_multiline_whitespace_and_scope_boundaries", + "failure_mode": "false-positive", + "case": { + "url": "https://www.tabulator.info/examples/6.x/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 2, + "stable_text_anchors": [], + "table_columns": [ + { + "name": "id", + "label": "ID", + "type": "integer", + "minimum": 1 + }, + { + "name": "name", + "label": "Name", + "type": "text" + }, + { + "name": "progress", + "label": "Progress", + "type": "integer", + "minimum": 0, + "maximum": 100 + }, + { + "name": "gender", + "label": "Gender", + "type": "text" + }, + { + "name": "rating", + "label": "Rating", + "type": "integer", + "minimum": 0, + "maximum": 5 + }, + { + "name": "color", + "label": "Favourite Color", + "type": "text" + }, + { + "name": "birth_date", + "label": "Date Of Birth", + "type": "date_dmy" } - }, - "artifact": "The table below has 1000 rows, each row is loaded as it becomes visible as you scroll\n\n\n1\n\nMs. Darrick McLaughlin\n\n70\n\nfemale\n\n4\n\nivory\n\n30/12/2025\n\nView Source", - "ok": true, - "expected_task_pass": false - }, - { - "id": "table-loading-label-outside-target-not-global-failure", - "category": "structured-extraction", - "root_cause": "unrelated_example_loading_not_target_failure", - "failure_mode": "false-negative", - "case": { - "url": "https://www.tabulator.info/examples/6.x/", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "table_row", - "minimum_records": 2, - "stable_text_anchors": [], - "table_columns": [ - { - "name": "id", - "label": "ID", - "type": "integer", - "minimum": 1 - }, - { - "name": "name", - "label": "Name", - "type": "text" - }, - { - "name": "progress", - "label": "Progress", - "type": "integer", - "minimum": 0, - "maximum": 100 - }, - { - "name": "gender", - "label": "Gender", - "type": "text" - }, - { - "name": "rating", - "label": "Rating", - "type": "integer", - "minimum": 0, - "maximum": 5 - }, - { - "name": "color", - "label": "Favourite Color", - "type": "text" - }, - { - "name": "birth_date", - "label": "Date Of Birth", - "type": "date_dmy" - } - ], - "record_identity_fields": [ - "id" - ], - "table_scope": { - "row_selector": "#example-table-virtual .tabulator-row", - "cell_selector": ".tabulator-cell", - "start_pattern": "The table below has 1000 rows[^\\n]*", - "end_pattern": "View Source", - "plain_row_pattern": "(?P\\d+)\\s*(?P[^\\d\\n]+?)\\s*(?P\\d{1,3})\\s*(?Pfemale|male)\\s*(?P\\d)\\s*(?P[^\\d\\n]+?)\\s*(?P\\d{2}/\\d{2}/\\d{4})", - "plain_multiline_row_pattern": "(?m)^(?P\\d+)\\n[ \\t\\n]*(?P[^\\d\\n]+)\\n[ \\t\\n]*(?P\\d{1,3})\\n[ \\t\\n]*(?Pfemale|male)\\n[ \\t\\n]*(?P\\d)\\n[ \\t\\n]*(?P[^\\d\\n]+)\\n[ \\t\\n]*(?P\\d{2}/\\d{2}/\\d{4})(?=\\n|$)" - } + ], + "record_identity_fields": [ + "id" + ], + "table_scope": { + "row_selector": "#example-table-virtual .tabulator-row", + "cell_selector": ".tabulator-cell", + "start_pattern": "The table below has 1000 rows[^\\n]*", + "end_pattern": "View Source", + "plain_row_pattern": "(?P\\d+)\\s*(?P[^\\d\\n]+?)\\s*(?P\\d{1,3})\\s*(?Pfemale|male)\\s*(?P\\d)\\s*(?P[^\\d\\n]+?)\\s*(?P\\d{2}/\\d{2}/\\d{4})", + "plain_multiline_row_pattern": "(?m)^(?P\\d+)\\n[ \\t\\n]*(?P[^\\d\\n]+)\\n[ \\t\\n]*(?P\\d{1,3})\\n[ \\t\\n]*(?Pfemale|male)\\n[ \\t\\n]*(?P\\d)\\n[ \\t\\n]*(?P[^\\d\\n]+)\\n[ \\t\\n]*(?P\\d{2}/\\d{2}/\\d{4})(?=\\n|$)" + } + } + }, + "artifact": "The table below has 1000 rows, each row is loaded as it becomes visible as you scroll\n\n\n1\n\nMs. Darrick McLaughlin\n\n70\n\nfemale\n\n4\n\nivory\n\n30/12/2025\n\nView Source", + "ok": true, + "expected_task_pass": false + }, + { + "id": "table-loading-label-outside-target-not-global-failure", + "category": "structured-extraction", + "root_cause": "unrelated_example_loading_not_target_failure", + "failure_mode": "false-negative", + "case": { + "url": "https://www.tabulator.info/examples/6.x/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 2, + "stable_text_anchors": [], + "table_columns": [ + { + "name": "id", + "label": "ID", + "type": "integer", + "minimum": 1 + }, + { + "name": "name", + "label": "Name", + "type": "text" + }, + { + "name": "progress", + "label": "Progress", + "type": "integer", + "minimum": 0, + "maximum": 100 + }, + { + "name": "gender", + "label": "Gender", + "type": "text" + }, + { + "name": "rating", + "label": "Rating", + "type": "integer", + "minimum": 0, + "maximum": 5 }, - "forbidden_any": [ - "Access denied", - "403 Forbidden", - "Verify you are human", - "Checking your browser", - "CAPTCHA", - "Just a moment" - ] - }, - "artifact": "The table below has 1000 rows, each row is loaded as it becomes visible as you scroll\n\n\n1\n\nMs. Darrick McLaughlin\n\n70\n\nfemale\n\n4\n\nivory\n\n30/12/2025\n\n2\n\nMrs. Monte Pouros\n\n53\n\nfemale\n\n4\n\nlavender\n\n24/03/2026\n\nView Source\nOther example\nLoading Example...", - "ok": true, - "expected_task_pass": true - }, - { - "id": "table-target-loading-without-records-rejected", - "category": "structured-extraction", - "root_cause": "unrelated_example_loading_not_target_failure", - "failure_mode": "false-positive", - "case": { - "url": "https://www.tabulator.info/examples/6.x/", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "table_row", - "minimum_records": 2, - "stable_text_anchors": [], - "table_columns": [ - { - "name": "id", - "label": "ID", - "type": "integer", - "minimum": 1 - }, - { - "name": "name", - "label": "Name", - "type": "text" - }, - { - "name": "progress", - "label": "Progress", - "type": "integer", - "minimum": 0, - "maximum": 100 - }, - { - "name": "gender", - "label": "Gender", - "type": "text" - }, - { - "name": "rating", - "label": "Rating", - "type": "integer", - "minimum": 0, - "maximum": 5 - }, - { - "name": "color", - "label": "Favourite Color", - "type": "text" - }, - { - "name": "birth_date", - "label": "Date Of Birth", - "type": "date_dmy" - } - ], - "record_identity_fields": [ - "id" - ], - "table_scope": { - "row_selector": "#example-table-virtual .tabulator-row", - "cell_selector": ".tabulator-cell", - "start_pattern": "The table below has 1000 rows[^\\n]*", - "end_pattern": "View Source", - "plain_row_pattern": "(?P\\d+)\\s*(?P[^\\d\\n]+?)\\s*(?P\\d{1,3})\\s*(?Pfemale|male)\\s*(?P\\d)\\s*(?P[^\\d\\n]+?)\\s*(?P\\d{2}/\\d{2}/\\d{4})", - "plain_multiline_row_pattern": "(?m)^(?P\\d+)\\n[ \\t\\n]*(?P[^\\d\\n]+)\\n[ \\t\\n]*(?P\\d{1,3})\\n[ \\t\\n]*(?Pfemale|male)\\n[ \\t\\n]*(?P\\d)\\n[ \\t\\n]*(?P[^\\d\\n]+)\\n[ \\t\\n]*(?P\\d{2}/\\d{2}/\\d{4})(?=\\n|$)" - } + { + "name": "color", + "label": "Favourite Color", + "type": "text" }, - "forbidden_any": [ - "Access denied", - "403 Forbidden", - "Verify you are human", - "Checking your browser", - "CAPTCHA", - "Just a moment" - ] - }, - "artifact": "The table below has 1000 rows\nLoading Example...\nView Source", - "ok": true, - "expected_task_pass": false - }, - { - "id": "heading-only-body-region-image-between-headings", - "category": "body-scope", - "root_cause": "separate_headings_not_contiguous_body_fragment", - "failure_mode": "false-negative", - "case": { - "url": "https://nutui.jd.com/h5/vue/4x/", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "body_text", - "minimum_records": 0, - "stable_text_anchors": [ - { - "id": "nutui-independent-heading-0", - "alternatives": [ - "扫码体验" - ], - "source_location": "main heading", - "observation_status": "independent source visible headings; match separately, no cross-heading adjacency requirement" - }, - { - "id": "nutui-independent-heading-1", - "alternatives": [ - "链接" - ], - "source_location": "main heading", - "observation_status": "independent source visible headings; match separately, no cross-heading adjacency requirement" - } - ] - }, - "body_requirements": { - "version": 3, - "unit": "region_compact_hexagram_recall", - "minimum_coverage": 0.8, - "minimum_floor": 0.6, - "minimum_region_passes": 4, - "segmentation_method": "sections", - "regions": [ - { - "id": "body-region-1", - "text": "nutui nutui vue 组 件 库 适 用 于 vue 技 术 栈 的 h5 应 用 开 箱 即 用 帮 助 研 发 快 速 开 发 用 户 界 面 提 升 开 发 效 率 改 善 开 发 体 验", - "source_text": "NutUI\n\nNutUI-Vue 组件库，适用于 Vue 技术栈的 H5 应用，开箱即用，帮助研发快速开发用户界面，提升开发效率，改善开发体验。", - "segmentation_method": "sections" - }, - { - "id": "body-region-2", - "text": "特 性 80 高 质 量 组 件 覆 盖 移 动 端 主 流 场 景 基 于 京 东 app 10 0 视 觉 规 范 支 持 按 需 引 用 详 尽 的 文 档 和 示 例 支 持 typescript 支 持 webstorm vscode 组 件 属 性 高 亮 支 持 动 态 定 制 主 题 支 持 暗 黑 模 式 支 持 国 际 化 单 元 测 试 覆 盖 率 超 过 80 保 障 稳 定 性 提 供 sketch 设 计 资 源", - "source_text": "特性\n\n🚀 80+ 高质量组件，覆盖移动端主流场景\n\n📖 基于京东 APP 10.0 视觉规范\n\n🍭 支持按需引用\n\n📖 详尽的文档和示例\n\n💪 支持 TypeScript\n\n💡 支持 webstorm，vscode 组件属性高亮\n\n💪 支持动态定制主题\n\n🍭 支持暗黑模式\n\n🌍 支持国际化\n\n🍭 单元测试覆盖率超过 80%，保障稳定性\n\n📖 提供 Sketch 设计资源", - "segmentation_method": "sections" - }, - { - "id": "body-region-4", - "text": "官 方 生 态 由 nutui 官 方 团 队 维 护 的 项 目 如 下 项 目 描 述 nutui nutui nutui 移 动 端 h5 版 本 nutui nutui taro nutui taro 多 端 版 本 nutui icons vue nutui nutui 使 用 的 图 标 库 nutui icons vue taro nutui nutui taro 使 用 的 图 标 库 nutui touch emulator 在 桌 面 端 使 用 nutui 的 辅 助 库 nutui playground nutui 在 线 playground nutui demo nutui 官 方 示 例 合 集", - "source_text": "官方生态\n\n由 NutUI 官方团队维护的项目如下：\n\n项目 描述\n\n@nutui/nutui NutUI 移动端 H5 版本\n\n@nutui/nutui-taro NutUI Taro 多端版本\n\n@nutui/icons-vue @nutui/nutui 使用的图标库\n\n@nutui/icons-vue-taro @nutui/nutui-taro 使用的图标库\n\n@nutui/touch-emulator 在桌面端使用 NutUI 的辅助库\n\n@nutui/playground NutUI 在线 Playground\n\nnutui-demo NutUI 官方示例合集", - "segmentation_method": "sections" - }, - { - "id": "body-region-5", - "text": "社 区 生 态 由 社 区 维 护 的 项 目 如 下 欢 迎 补 充 项 目 描 述 yang1206 uniapp nutui 基 于 nutui 4 x 版 本 开 发 的 uni app 组 件 库 jwaterwater uni nutui 基 于 nutui 3 x 版 本 开 发 的 uni app 组 件 库 贡 献 核 心 人 员 以 下 是 nutui 的 核 心 贡 献 者 们 richard1015 drjingfubo love forever yangxiaolu1993 snandy eiinu ymm ailululu lkjh3214 yangjinjun3 贡 献 者 们 感 谢 以 下 小 伙 伴 们 为 nutui 发 展 做 出 的 贡 献 共 建 指 南 贡 献 指 南 pr 开 发 者 开 源 协 议 mit", - "source_text": "社区生态\n\n由社区维护的项目如下，欢迎补充：\n\n项目 描述\n\nyang1206/uniapp-nutui 基于 NutUI 4.x 版本开发的 uni-app 组件库\n\njwaterwater/uni-nutui 基于 NutUI 3.x 版本开发的 uni-app 组件库\n\n贡献\n\n核心人员\n\n以下是 NutUI 的核心贡献者们：\n\nrichard1015\n\nDrjingfubo\n\nlove_forever\n\nyangxiaolu1993\n\nsnandy\n\nEiinu\n\nYmm\n\nailululu\n\nlkjh3214\n\nyangjinjun3\n\n贡献者们\n\n感谢以下小伙伴们为 NutUI 发展做出的贡献：\n\n共建指南\n\n贡献指南\n\nPR\n\n开发者\n\n开源协议\n\nMIT", - "segmentation_method": "sections" - } - ] - } - }, - "artifact": "NutUI\n\nNutUI-Vue 组件库，适用于 Vue 技术栈的 H5 应用，开箱即用，帮助研发快速开发用户界面，提升开发效率，改善开发体验。\n\n特性\n\n🚀 80+ 高质量组件，覆盖移动端主流场景\n\n📖 基于京东 APP 10.0 视觉规范\n\n🍭 支持按需引用\n\n📖 详尽的文档和示例\n\n💪 支持 TypeScript\n\n💡 支持 webstorm，vscode 组件属性高亮\n\n💪 支持动态定制主题\n\n🍭 支持暗黑模式\n\n🌍 支持国际化\n\n🍭 单元测试覆盖率超过 80%，保障稳定性\n\n📖 提供 Sketch 设计资源\n\n官方生态\n\n由 NutUI 官方团队维护的项目如下：\n\n项目 描述\n\n@nutui/nutui NutUI 移动端 H5 版本\n\n@nutui/nutui-taro NutUI Taro 多端版本\n\n@nutui/icons-vue @nutui/nutui 使用的图标库\n\n@nutui/icons-vue-taro @nutui/nutui-taro 使用的图标库\n\n@nutui/touch-emulator 在桌面端使用 NutUI 的辅助库\n\n@nutui/playground NutUI 在线 Playground\n\nnutui-demo NutUI 官方示例合集\n\n社区生态\n\n由社区维护的项目如下，欢迎补充：\n\n项目 描述\n\nyang1206/uniapp-nutui 基于 NutUI 4.x 版本开发的 uni-app 组件库\n\njwaterwater/uni-nutui 基于 NutUI 3.x 版本开发的 uni-app 组件库\n\n贡献\n\n核心人员\n\n以下是 NutUI 的核心贡献者们：\n\nrichard1015\n\nDrjingfubo\n\nlove_forever\n\nyangxiaolu1993\n\nsnandy\n\nEiinu\n\nYmm\n\nailululu\n\nlkjh3214\n\nyangjinjun3\n\n贡献者们\n\n感谢以下小伙伴们为 NutUI 发展做出的贡献：\n\n共建指南\n\n贡献指南\n\nPR\n\n开发者\n\n开源协议\n\nMIT\n\n## 扫码体验\n\n![NutUI](https://example.org/qr.png)\n\n## 链接", - "ok": true, - "expected_task_pass": true - }, - { - "id": "heading-only-body-region-missing-heading", - "category": "body-scope", - "root_cause": "separate_headings_not_contiguous_body_fragment", - "failure_mode": "false-positive", - "case": { - "url": "https://nutui.jd.com/h5/vue/4x/", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "body_text", - "minimum_records": 0, - "stable_text_anchors": [ - { - "id": "nutui-independent-heading-0", - "alternatives": [ - "扫码体验" - ], - "source_location": "main heading", - "observation_status": "independent source visible headings; match separately, no cross-heading adjacency requirement" - }, - { - "id": "nutui-independent-heading-1", - "alternatives": [ - "链接" - ], - "source_location": "main heading", - "observation_status": "independent source visible headings; match separately, no cross-heading adjacency requirement" - } - ] - }, - "body_requirements": { - "version": 3, - "unit": "region_compact_hexagram_recall", - "minimum_coverage": 0.8, - "minimum_floor": 0.6, - "minimum_region_passes": 4, - "segmentation_method": "sections", - "regions": [ - { - "id": "body-region-1", - "text": "nutui nutui vue 组 件 库 适 用 于 vue 技 术 栈 的 h5 应 用 开 箱 即 用 帮 助 研 发 快 速 开 发 用 户 界 面 提 升 开 发 效 率 改 善 开 发 体 验", - "source_text": "NutUI\n\nNutUI-Vue 组件库，适用于 Vue 技术栈的 H5 应用，开箱即用，帮助研发快速开发用户界面，提升开发效率，改善开发体验。", - "segmentation_method": "sections" - }, - { - "id": "body-region-2", - "text": "特 性 80 高 质 量 组 件 覆 盖 移 动 端 主 流 场 景 基 于 京 东 app 10 0 视 觉 规 范 支 持 按 需 引 用 详 尽 的 文 档 和 示 例 支 持 typescript 支 持 webstorm vscode 组 件 属 性 高 亮 支 持 动 态 定 制 主 题 支 持 暗 黑 模 式 支 持 国 际 化 单 元 测 试 覆 盖 率 超 过 80 保 障 稳 定 性 提 供 sketch 设 计 资 源", - "source_text": "特性\n\n🚀 80+ 高质量组件，覆盖移动端主流场景\n\n📖 基于京东 APP 10.0 视觉规范\n\n🍭 支持按需引用\n\n📖 详尽的文档和示例\n\n💪 支持 TypeScript\n\n💡 支持 webstorm，vscode 组件属性高亮\n\n💪 支持动态定制主题\n\n🍭 支持暗黑模式\n\n🌍 支持国际化\n\n🍭 单元测试覆盖率超过 80%，保障稳定性\n\n📖 提供 Sketch 设计资源", - "segmentation_method": "sections" - }, - { - "id": "body-region-4", - "text": "官 方 生 态 由 nutui 官 方 团 队 维 护 的 项 目 如 下 项 目 描 述 nutui nutui nutui 移 动 端 h5 版 本 nutui nutui taro nutui taro 多 端 版 本 nutui icons vue nutui nutui 使 用 的 图 标 库 nutui icons vue taro nutui nutui taro 使 用 的 图 标 库 nutui touch emulator 在 桌 面 端 使 用 nutui 的 辅 助 库 nutui playground nutui 在 线 playground nutui demo nutui 官 方 示 例 合 集", - "source_text": "官方生态\n\n由 NutUI 官方团队维护的项目如下：\n\n项目 描述\n\n@nutui/nutui NutUI 移动端 H5 版本\n\n@nutui/nutui-taro NutUI Taro 多端版本\n\n@nutui/icons-vue @nutui/nutui 使用的图标库\n\n@nutui/icons-vue-taro @nutui/nutui-taro 使用的图标库\n\n@nutui/touch-emulator 在桌面端使用 NutUI 的辅助库\n\n@nutui/playground NutUI 在线 Playground\n\nnutui-demo NutUI 官方示例合集", - "segmentation_method": "sections" - }, - { - "id": "body-region-5", - "text": "社 区 生 态 由 社 区 维 护 的 项 目 如 下 欢 迎 补 充 项 目 描 述 yang1206 uniapp nutui 基 于 nutui 4 x 版 本 开 发 的 uni app 组 件 库 jwaterwater uni nutui 基 于 nutui 3 x 版 本 开 发 的 uni app 组 件 库 贡 献 核 心 人 员 以 下 是 nutui 的 核 心 贡 献 者 们 richard1015 drjingfubo love forever yangxiaolu1993 snandy eiinu ymm ailululu lkjh3214 yangjinjun3 贡 献 者 们 感 谢 以 下 小 伙 伴 们 为 nutui 发 展 做 出 的 贡 献 共 建 指 南 贡 献 指 南 pr 开 发 者 开 源 协 议 mit", - "source_text": "社区生态\n\n由社区维护的项目如下，欢迎补充：\n\n项目 描述\n\nyang1206/uniapp-nutui 基于 NutUI 4.x 版本开发的 uni-app 组件库\n\njwaterwater/uni-nutui 基于 NutUI 3.x 版本开发的 uni-app 组件库\n\n贡献\n\n核心人员\n\n以下是 NutUI 的核心贡献者们：\n\nrichard1015\n\nDrjingfubo\n\nlove_forever\n\nyangxiaolu1993\n\nsnandy\n\nEiinu\n\nYmm\n\nailululu\n\nlkjh3214\n\nyangjinjun3\n\n贡献者们\n\n感谢以下小伙伴们为 NutUI 发展做出的贡献：\n\n共建指南\n\n贡献指南\n\nPR\n\n开发者\n\n开源协议\n\nMIT", - "segmentation_method": "sections" - } - ] - } - }, - "artifact": "NutUI\n\nNutUI-Vue 组件库，适用于 Vue 技术栈的 H5 应用，开箱即用，帮助研发快速开发用户界面，提升开发效率，改善开发体验。\n\n特性\n\n🚀 80+ 高质量组件，覆盖移动端主流场景\n\n📖 基于京东 APP 10.0 视觉规范\n\n🍭 支持按需引用\n\n📖 详尽的文档和示例\n\n💪 支持 TypeScript\n\n💡 支持 webstorm，vscode 组件属性高亮\n\n💪 支持动态定制主题\n\n🍭 支持暗黑模式\n\n🌍 支持国际化\n\n🍭 单元测试覆盖率超过 80%，保障稳定性\n\n📖 提供 Sketch 设计资源\n\n官方生态\n\n由 NutUI 官方团队维护的项目如下：\n\n项目 描述\n\n@nutui/nutui NutUI 移动端 H5 版本\n\n@nutui/nutui-taro NutUI Taro 多端版本\n\n@nutui/icons-vue @nutui/nutui 使用的图标库\n\n@nutui/icons-vue-taro @nutui/nutui-taro 使用的图标库\n\n@nutui/touch-emulator 在桌面端使用 NutUI 的辅助库\n\n@nutui/playground NutUI 在线 Playground\n\nnutui-demo NutUI 官方示例合集\n\n社区生态\n\n由社区维护的项目如下，欢迎补充：\n\n项目 描述\n\nyang1206/uniapp-nutui 基于 NutUI 4.x 版本开发的 uni-app 组件库\n\njwaterwater/uni-nutui 基于 NutUI 3.x 版本开发的 uni-app 组件库\n\n贡献\n\n核心人员\n\n以下是 NutUI 的核心贡献者们：\n\nrichard1015\n\nDrjingfubo\n\nlove_forever\n\nyangxiaolu1993\n\nsnandy\n\nEiinu\n\nYmm\n\nailululu\n\nlkjh3214\n\nyangjinjun3\n\n贡献者们\n\n感谢以下小伙伴们为 NutUI 发展做出的贡献：\n\n共建指南\n\n贡献指南\n\nPR\n\n开发者\n\n开源协议\n\nMIT\n\n## 链接", - "ok": true, - "expected_task_pass": false - }, - { - "id": "heading-only-body-region-headings-only", - "category": "body-scope", - "root_cause": "separate_headings_not_contiguous_body_fragment", - "failure_mode": "false-positive", - "case": { - "url": "https://nutui.jd.com/h5/vue/4x/", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "body_text", - "minimum_records": 0, - "stable_text_anchors": [ - { - "id": "nutui-independent-heading-0", - "alternatives": [ - "扫码体验" - ], - "source_location": "main heading", - "observation_status": "independent source visible headings; match separately, no cross-heading adjacency requirement" - }, - { - "id": "nutui-independent-heading-1", - "alternatives": [ - "链接" - ], - "source_location": "main heading", - "observation_status": "independent source visible headings; match separately, no cross-heading adjacency requirement" - } - ] - }, - "body_requirements": { - "version": 3, - "unit": "region_compact_hexagram_recall", - "minimum_coverage": 0.8, - "minimum_floor": 0.6, - "minimum_region_passes": 4, - "segmentation_method": "sections", - "regions": [ - { - "id": "body-region-1", - "text": "nutui nutui vue 组 件 库 适 用 于 vue 技 术 栈 的 h5 应 用 开 箱 即 用 帮 助 研 发 快 速 开 发 用 户 界 面 提 升 开 发 效 率 改 善 开 发 体 验", - "source_text": "NutUI\n\nNutUI-Vue 组件库，适用于 Vue 技术栈的 H5 应用，开箱即用，帮助研发快速开发用户界面，提升开发效率，改善开发体验。", - "segmentation_method": "sections" - }, - { - "id": "body-region-2", - "text": "特 性 80 高 质 量 组 件 覆 盖 移 动 端 主 流 场 景 基 于 京 东 app 10 0 视 觉 规 范 支 持 按 需 引 用 详 尽 的 文 档 和 示 例 支 持 typescript 支 持 webstorm vscode 组 件 属 性 高 亮 支 持 动 态 定 制 主 题 支 持 暗 黑 模 式 支 持 国 际 化 单 元 测 试 覆 盖 率 超 过 80 保 障 稳 定 性 提 供 sketch 设 计 资 源", - "source_text": "特性\n\n🚀 80+ 高质量组件，覆盖移动端主流场景\n\n📖 基于京东 APP 10.0 视觉规范\n\n🍭 支持按需引用\n\n📖 详尽的文档和示例\n\n💪 支持 TypeScript\n\n💡 支持 webstorm，vscode 组件属性高亮\n\n💪 支持动态定制主题\n\n🍭 支持暗黑模式\n\n🌍 支持国际化\n\n🍭 单元测试覆盖率超过 80%，保障稳定性\n\n📖 提供 Sketch 设计资源", - "segmentation_method": "sections" - }, - { - "id": "body-region-4", - "text": "官 方 生 态 由 nutui 官 方 团 队 维 护 的 项 目 如 下 项 目 描 述 nutui nutui nutui 移 动 端 h5 版 本 nutui nutui taro nutui taro 多 端 版 本 nutui icons vue nutui nutui 使 用 的 图 标 库 nutui icons vue taro nutui nutui taro 使 用 的 图 标 库 nutui touch emulator 在 桌 面 端 使 用 nutui 的 辅 助 库 nutui playground nutui 在 线 playground nutui demo nutui 官 方 示 例 合 集", - "source_text": "官方生态\n\n由 NutUI 官方团队维护的项目如下：\n\n项目 描述\n\n@nutui/nutui NutUI 移动端 H5 版本\n\n@nutui/nutui-taro NutUI Taro 多端版本\n\n@nutui/icons-vue @nutui/nutui 使用的图标库\n\n@nutui/icons-vue-taro @nutui/nutui-taro 使用的图标库\n\n@nutui/touch-emulator 在桌面端使用 NutUI 的辅助库\n\n@nutui/playground NutUI 在线 Playground\n\nnutui-demo NutUI 官方示例合集", - "segmentation_method": "sections" - }, - { - "id": "body-region-5", - "text": "社 区 生 态 由 社 区 维 护 的 项 目 如 下 欢 迎 补 充 项 目 描 述 yang1206 uniapp nutui 基 于 nutui 4 x 版 本 开 发 的 uni app 组 件 库 jwaterwater uni nutui 基 于 nutui 3 x 版 本 开 发 的 uni app 组 件 库 贡 献 核 心 人 员 以 下 是 nutui 的 核 心 贡 献 者 们 richard1015 drjingfubo love forever yangxiaolu1993 snandy eiinu ymm ailululu lkjh3214 yangjinjun3 贡 献 者 们 感 谢 以 下 小 伙 伴 们 为 nutui 发 展 做 出 的 贡 献 共 建 指 南 贡 献 指 南 pr 开 发 者 开 源 协 议 mit", - "source_text": "社区生态\n\n由社区维护的项目如下，欢迎补充：\n\n项目 描述\n\nyang1206/uniapp-nutui 基于 NutUI 4.x 版本开发的 uni-app 组件库\n\njwaterwater/uni-nutui 基于 NutUI 3.x 版本开发的 uni-app 组件库\n\n贡献\n\n核心人员\n\n以下是 NutUI 的核心贡献者们：\n\nrichard1015\n\nDrjingfubo\n\nlove_forever\n\nyangxiaolu1993\n\nsnandy\n\nEiinu\n\nYmm\n\nailululu\n\nlkjh3214\n\nyangjinjun3\n\n贡献者们\n\n感谢以下小伙伴们为 NutUI 发展做出的贡献：\n\n共建指南\n\n贡献指南\n\nPR\n\n开发者\n\n开源协议\n\nMIT", - "segmentation_method": "sections" - } - ] - } - }, - "artifact": "## 扫码体验\n\n![NutUI](https://example.org/qr.png)\n\n## 链接", - "ok": true, - "expected_task_pass": false - }, - { - "id": "detail-url-unreserved-encoding-unreserved-encoded", - "category": "format-normalization", - "root_cause": "unreserved_url_encoding_preserves_detail_identity", - "failure_mode": "false-negative", - "case": { - "url": "https://www.allocine.fr/film/", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 2, - "hosts": [ - "www.allocine.fr" - ], - "entry_path": "/film/fichefilm_gen_cfilm=[0-9]+\\.html", - "stable_text_anchors": [], - "entry_scope": { - "html_selector": "main a.meta-title-link, main a.list-entity-item-link, main a.light, main a.show-name" - }, - "name_validation": "nonempty" - } - }, - "artifact": "[Film Alpha](/film/fichefilm%5Fgen%5Fcfilm=123.html)\n[Film Beta](/film/fichefilm_gen_cfilm=456.html)", - "ok": true, - "expected_task_pass": true - }, - { - "id": "detail-url-unreserved-encoding-encoded-duplicate", - "category": "format-normalization", - "root_cause": "unreserved_url_encoding_preserves_detail_identity", - "failure_mode": "false-positive", - "case": { - "url": "https://www.allocine.fr/film/", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 2, - "hosts": [ - "www.allocine.fr" - ], - "entry_path": "/film/fichefilm_gen_cfilm=[0-9]+\\.html", - "stable_text_anchors": [], - "entry_scope": { - "html_selector": "main a.meta-title-link, main a.list-entity-item-link, main a.light, main a.show-name" - }, - "name_validation": "nonempty" - } - }, - "artifact": "[Film Alpha](/film/fichefilm%5Fgen%5Fcfilm=123.html)\n[Film Alpha](/film/fichefilm_gen_cfilm=123.html)", - "ok": true, - "expected_task_pass": false - }, - { - "id": "detail-url-unreserved-encoding-reserved-separator", - "category": "format-normalization", - "root_cause": "unreserved_url_encoding_preserves_detail_identity", - "failure_mode": "false-positive", - "case": { - "url": "https://www.allocine.fr/film/", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 2, - "hosts": [ - "www.allocine.fr" - ], - "entry_path": "/film/fichefilm_gen_cfilm=[0-9]+\\.html", - "stable_text_anchors": [], - "entry_scope": { - "html_selector": "main a.meta-title-link, main a.list-entity-item-link, main a.light, main a.show-name" - }, - "name_validation": "nonempty" - } - }, - "artifact": "[Film Alpha](/film/fichefilm%5Fgen%5Fcfilm=123.html)\n[Film Beta](/film%2Ffichefilm_gen_cfilm=456.html)", - "ok": true, - "expected_task_pass": false - }, - { - "id": "detail-url-unreserved-encoding-empty-name", - "category": "format-normalization", - "root_cause": "unreserved_url_encoding_preserves_detail_identity", - "failure_mode": "false-positive", - "case": { - "url": "https://www.allocine.fr/film/", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 2, - "hosts": [ - "www.allocine.fr" - ], - "entry_path": "/film/fichefilm_gen_cfilm=[0-9]+\\.html", - "stable_text_anchors": [], - "entry_scope": { - "html_selector": "main a.meta-title-link, main a.list-entity-item-link, main a.light, main a.show-name" - }, - "name_validation": "nonempty" - } - }, - "artifact": "[Film Alpha](/film/fichefilm%5Fgen%5Fcfilm=123.html)\n[](/film/fichefilm_gen_cfilm=456.html)", - "ok": true, - "expected_task_pass": false - }, - { - "id": "detail-url-unreserved-encoding-wrong-detail-path", - "category": "format-normalization", - "root_cause": "unreserved_url_encoding_preserves_detail_identity", - "failure_mode": "false-positive", - "case": { - "url": "https://www.allocine.fr/film/", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 2, - "hosts": [ - "www.allocine.fr" - ], - "entry_path": "/film/fichefilm_gen_cfilm=[0-9]+\\.html", - "stable_text_anchors": [], - "entry_scope": { - "html_selector": "main a.meta-title-link, main a.list-entity-item-link, main a.light, main a.show-name" - }, - "name_validation": "nonempty" - } - }, - "artifact": "[Film Alpha](/film/fichefilm%5Fgen%5Fcfilm=123.html)\n[Film Beta](/film/fichefilm_gen_cfilm=456.json)", - "ok": true, - "expected_task_pass": false - }, - { - "id": "named-link-separated-list-comma-list", - "category": "format-normalization", - "root_cause": "punctuation_separated_explicit_named_links", - "failure_mode": "false-negative", - "case": { - "url": "https://www.allocine.fr/film/", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 2, - "hosts": [ - "www.allocine.fr" - ], - "entry_path": "/film/fichefilm_gen_cfilm=[0-9]+\\.html", - "stable_text_anchors": [], - "entry_scope": { - "html_selector": "main a.meta-title-link, main a.list-entity-item-link, main a.light, main a.show-name" - }, - "name_validation": "nonempty" + { + "name": "birth_date", + "label": "Date Of Birth", + "type": "date_dmy" } + ], + "record_identity_fields": [ + "id" + ], + "table_scope": { + "row_selector": "#example-table-virtual .tabulator-row", + "cell_selector": ".tabulator-cell", + "start_pattern": "The table below has 1000 rows[^\\n]*", + "end_pattern": "View Source", + "plain_row_pattern": "(?P\\d+)\\s*(?P[^\\d\\n]+?)\\s*(?P\\d{1,3})\\s*(?Pfemale|male)\\s*(?P\\d)\\s*(?P[^\\d\\n]+?)\\s*(?P\\d{2}/\\d{2}/\\d{4})", + "plain_multiline_row_pattern": "(?m)^(?P\\d+)\\n[ \\t\\n]*(?P[^\\d\\n]+)\\n[ \\t\\n]*(?P\\d{1,3})\\n[ \\t\\n]*(?Pfemale|male)\\n[ \\t\\n]*(?P\\d)\\n[ \\t\\n]*(?P[^\\d\\n]+)\\n[ \\t\\n]*(?P\\d{2}/\\d{2}/\\d{4})(?=\\n|$)" + } }, - "artifact": "[Film Alpha](/film/fichefilm_gen_cfilm=123.html), [Film Beta](/film/fichefilm_gen_cfilm=456.html)", - "ok": true, - "expected_task_pass": true - }, - { - "id": "named-link-separated-list-semicolon-list", - "category": "format-normalization", - "root_cause": "punctuation_separated_explicit_named_links", - "failure_mode": "false-negative", - "case": { - "url": "https://www.allocine.fr/film/", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 2, - "hosts": [ - "www.allocine.fr" - ], - "entry_path": "/film/fichefilm_gen_cfilm=[0-9]+\\.html", - "stable_text_anchors": [], - "entry_scope": { - "html_selector": "main a.meta-title-link, main a.list-entity-item-link, main a.light, main a.show-name" - }, - "name_validation": "nonempty" + "forbidden_any": [ + "Access denied", + "403 Forbidden", + "Verify you are human", + "Checking your browser", + "CAPTCHA", + "Just a moment" + ] + }, + "artifact": "The table below has 1000 rows, each row is loaded as it becomes visible as you scroll\n\n\n1\n\nMs. Darrick McLaughlin\n\n70\n\nfemale\n\n4\n\nivory\n\n30/12/2025\n\n2\n\nMrs. Monte Pouros\n\n53\n\nfemale\n\n4\n\nlavender\n\n24/03/2026\n\nView Source\nOther example\nLoading Example...", + "ok": true, + "expected_task_pass": true + }, + { + "id": "table-target-loading-without-records-rejected", + "category": "structured-extraction", + "root_cause": "unrelated_example_loading_not_target_failure", + "failure_mode": "false-positive", + "case": { + "url": "https://www.tabulator.info/examples/6.x/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 2, + "stable_text_anchors": [], + "table_columns": [ + { + "name": "id", + "label": "ID", + "type": "integer", + "minimum": 1 + }, + { + "name": "name", + "label": "Name", + "type": "text" + }, + { + "name": "progress", + "label": "Progress", + "type": "integer", + "minimum": 0, + "maximum": 100 + }, + { + "name": "gender", + "label": "Gender", + "type": "text" + }, + { + "name": "rating", + "label": "Rating", + "type": "integer", + "minimum": 0, + "maximum": 5 + }, + { + "name": "color", + "label": "Favourite Color", + "type": "text" + }, + { + "name": "birth_date", + "label": "Date Of Birth", + "type": "date_dmy" } + ], + "record_identity_fields": [ + "id" + ], + "table_scope": { + "row_selector": "#example-table-virtual .tabulator-row", + "cell_selector": ".tabulator-cell", + "start_pattern": "The table below has 1000 rows[^\\n]*", + "end_pattern": "View Source", + "plain_row_pattern": "(?P\\d+)\\s*(?P[^\\d\\n]+?)\\s*(?P\\d{1,3})\\s*(?Pfemale|male)\\s*(?P\\d)\\s*(?P[^\\d\\n]+?)\\s*(?P\\d{2}/\\d{2}/\\d{4})", + "plain_multiline_row_pattern": "(?m)^(?P\\d+)\\n[ \\t\\n]*(?P[^\\d\\n]+)\\n[ \\t\\n]*(?P\\d{1,3})\\n[ \\t\\n]*(?Pfemale|male)\\n[ \\t\\n]*(?P\\d)\\n[ \\t\\n]*(?P[^\\d\\n]+)\\n[ \\t\\n]*(?P\\d{2}/\\d{2}/\\d{4})(?=\\n|$)" + } }, - "artifact": "[Film Alpha](/film/fichefilm_gen_cfilm=123.html); [Film Beta](/film/fichefilm_gen_cfilm=456.html)", - "ok": true, - "expected_task_pass": true - }, - { - "id": "named-link-separated-list-prose-recommendation", - "category": "format-normalization", - "root_cause": "punctuation_separated_explicit_named_links", - "failure_mode": "false-positive", - "case": { - "url": "https://www.allocine.fr/film/", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 2, - "hosts": [ - "www.allocine.fr" - ], - "entry_path": "/film/fichefilm_gen_cfilm=[0-9]+\\.html", - "stable_text_anchors": [], - "entry_scope": { - "html_selector": "main a.meta-title-link, main a.list-entity-item-link, main a.light, main a.show-name" - }, - "name_validation": "nonempty" + "forbidden_any": [ + "Access denied", + "403 Forbidden", + "Verify you are human", + "Checking your browser", + "CAPTCHA", + "Just a moment" + ] + }, + "artifact": "The table below has 1000 rows\nLoading Example...\nView Source", + "ok": true, + "expected_task_pass": false + }, + { + "id": "heading-only-body-region-image-between-headings", + "category": "body-scope", + "root_cause": "separate_headings_not_contiguous_body_fragment", + "failure_mode": "false-negative", + "case": { + "url": "https://nutui.jd.com/h5/vue/4x/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "body_text", + "minimum_records": 0, + "stable_text_anchors": [ + { + "id": "nutui-independent-heading-0", + "alternatives": [ + "扫码体验" + ], + "source_location": "main heading", + "observation_status": "independent source visible headings; match separately, no cross-heading adjacency requirement" + }, + { + "id": "nutui-independent-heading-1", + "alternatives": [ + "链接" + ], + "source_location": "main heading", + "observation_status": "independent source visible headings; match separately, no cross-heading adjacency requirement" } + ] }, - "artifact": "Watch [Film Alpha](/film/fichefilm_gen_cfilm=123.html) or [Film Beta](/film/fichefilm_gen_cfilm=456.html)", - "ok": true, - "expected_task_pass": false - }, - { - "id": "named-link-separated-list-prose-between-links", - "category": "format-normalization", - "root_cause": "punctuation_separated_explicit_named_links", - "failure_mode": "false-positive", - "case": { - "url": "https://www.allocine.fr/film/", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 2, - "hosts": [ - "www.allocine.fr" - ], - "entry_path": "/film/fichefilm_gen_cfilm=[0-9]+\\.html", - "stable_text_anchors": [], - "entry_scope": { - "html_selector": "main a.meta-title-link, main a.list-entity-item-link, main a.light, main a.show-name" - }, - "name_validation": "nonempty" + "body_requirements": { + "version": 3, + "unit": "region_compact_hexagram_recall", + "minimum_coverage": 0.8, + "minimum_floor": 0.6, + "minimum_region_passes": 4, + "segmentation_method": "sections", + "regions": [ + { + "id": "body-region-1", + "text": "nutui nutui vue 组 件 库 适 用 于 vue 技 术 栈 的 h5 应 用 开 箱 即 用 帮 助 研 发 快 速 开 发 用 户 界 面 提 升 开 发 效 率 改 善 开 发 体 验", + "source_text": "NutUI\n\nNutUI-Vue 组件库，适用于 Vue 技术栈的 H5 应用，开箱即用，帮助研发快速开发用户界面，提升开发效率，改善开发体验。", + "segmentation_method": "sections" + }, + { + "id": "body-region-2", + "text": "特 性 80 高 质 量 组 件 覆 盖 移 动 端 主 流 场 景 基 于 京 东 app 10 0 视 觉 规 范 支 持 按 需 引 用 详 尽 的 文 档 和 示 例 支 持 typescript 支 持 webstorm vscode 组 件 属 性 高 亮 支 持 动 态 定 制 主 题 支 持 暗 黑 模 式 支 持 国 际 化 单 元 测 试 覆 盖 率 超 过 80 保 障 稳 定 性 提 供 sketch 设 计 资 源", + "source_text": "特性\n\n🚀 80+ 高质量组件，覆盖移动端主流场景\n\n📖 基于京东 APP 10.0 视觉规范\n\n🍭 支持按需引用\n\n📖 详尽的文档和示例\n\n💪 支持 TypeScript\n\n💡 支持 webstorm，vscode 组件属性高亮\n\n💪 支持动态定制主题\n\n🍭 支持暗黑模式\n\n🌍 支持国际化\n\n🍭 单元测试覆盖率超过 80%，保障稳定性\n\n📖 提供 Sketch 设计资源", + "segmentation_method": "sections" + }, + { + "id": "body-region-4", + "text": "官 方 生 态 由 nutui 官 方 团 队 维 护 的 项 目 如 下 项 目 描 述 nutui nutui nutui 移 动 端 h5 版 本 nutui nutui taro nutui taro 多 端 版 本 nutui icons vue nutui nutui 使 用 的 图 标 库 nutui icons vue taro nutui nutui taro 使 用 的 图 标 库 nutui touch emulator 在 桌 面 端 使 用 nutui 的 辅 助 库 nutui playground nutui 在 线 playground nutui demo nutui 官 方 示 例 合 集", + "source_text": "官方生态\n\n由 NutUI 官方团队维护的项目如下：\n\n项目 描述\n\n@nutui/nutui NutUI 移动端 H5 版本\n\n@nutui/nutui-taro NutUI Taro 多端版本\n\n@nutui/icons-vue @nutui/nutui 使用的图标库\n\n@nutui/icons-vue-taro @nutui/nutui-taro 使用的图标库\n\n@nutui/touch-emulator 在桌面端使用 NutUI 的辅助库\n\n@nutui/playground NutUI 在线 Playground\n\nnutui-demo NutUI 官方示例合集", + "segmentation_method": "sections" + }, + { + "id": "body-region-5", + "text": "社 区 生 态 由 社 区 维 护 的 项 目 如 下 欢 迎 补 充 项 目 描 述 yang1206 uniapp nutui 基 于 nutui 4 x 版 本 开 发 的 uni app 组 件 库 jwaterwater uni nutui 基 于 nutui 3 x 版 本 开 发 的 uni app 组 件 库 贡 献 核 心 人 员 以 下 是 nutui 的 核 心 贡 献 者 们 richard1015 drjingfubo love forever yangxiaolu1993 snandy eiinu ymm ailululu lkjh3214 yangjinjun3 贡 献 者 们 感 谢 以 下 小 伙 伴 们 为 nutui 发 展 做 出 的 贡 献 共 建 指 南 贡 献 指 南 pr 开 发 者 开 源 协 议 mit", + "source_text": "社区生态\n\n由社区维护的项目如下，欢迎补充：\n\n项目 描述\n\nyang1206/uniapp-nutui 基于 NutUI 4.x 版本开发的 uni-app 组件库\n\njwaterwater/uni-nutui 基于 NutUI 3.x 版本开发的 uni-app 组件库\n\n贡献\n\n核心人员\n\n以下是 NutUI 的核心贡献者们：\n\nrichard1015\n\nDrjingfubo\n\nlove_forever\n\nyangxiaolu1993\n\nsnandy\n\nEiinu\n\nYmm\n\nailululu\n\nlkjh3214\n\nyangjinjun3\n\n贡献者们\n\n感谢以下小伙伴们为 NutUI 发展做出的贡献：\n\n共建指南\n\n贡献指南\n\nPR\n\n开发者\n\n开源协议\n\nMIT", + "segmentation_method": "sections" } - }, - "artifact": "[Film Alpha](/film/fichefilm_gen_cfilm=123.html) is better than [Film Beta](/film/fichefilm_gen_cfilm=456.html)", - "ok": true, - "expected_task_pass": false - }, - { - "id": "joyn-movie-ownership-fresh-movies", - "category": "structured-extraction", - "root_cause": "movie_records_not_genre_navigation_or_placeholder", - "failure_mode": "false-negative", - "case": { - "url": "https://www.joyn.de/filme", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 2, - "hosts": [ - "www.joyn.de" - ], - "entry_path": "/filme/(?!genre(?:/|$))[^/]+", - "stable_text_anchors": [], - "name_validation": "nonempty", - "entry_scope": { - "html_selector": "main div[class*=\"CardStandard_StandardCardWrapper\"]", - "title_selector": "h3", - "link_selector": "a[data-testid=\"LL--movie\"]" - } + ] + } + }, + "artifact": "NutUI\n\nNutUI-Vue 组件库，适用于 Vue 技术栈的 H5 应用，开箱即用，帮助研发快速开发用户界面，提升开发效率，改善开发体验。\n\n特性\n\n🚀 80+ 高质量组件，覆盖移动端主流场景\n\n📖 基于京东 APP 10.0 视觉规范\n\n🍭 支持按需引用\n\n📖 详尽的文档和示例\n\n💪 支持 TypeScript\n\n💡 支持 webstorm，vscode 组件属性高亮\n\n💪 支持动态定制主题\n\n🍭 支持暗黑模式\n\n🌍 支持国际化\n\n🍭 单元测试覆盖率超过 80%，保障稳定性\n\n📖 提供 Sketch 设计资源\n\n官方生态\n\n由 NutUI 官方团队维护的项目如下：\n\n项目 描述\n\n@nutui/nutui NutUI 移动端 H5 版本\n\n@nutui/nutui-taro NutUI Taro 多端版本\n\n@nutui/icons-vue @nutui/nutui 使用的图标库\n\n@nutui/icons-vue-taro @nutui/nutui-taro 使用的图标库\n\n@nutui/touch-emulator 在桌面端使用 NutUI 的辅助库\n\n@nutui/playground NutUI 在线 Playground\n\nnutui-demo NutUI 官方示例合集\n\n社区生态\n\n由社区维护的项目如下，欢迎补充：\n\n项目 描述\n\nyang1206/uniapp-nutui 基于 NutUI 4.x 版本开发的 uni-app 组件库\n\njwaterwater/uni-nutui 基于 NutUI 3.x 版本开发的 uni-app 组件库\n\n贡献\n\n核心人员\n\n以下是 NutUI 的核心贡献者们：\n\nrichard1015\n\nDrjingfubo\n\nlove_forever\n\nyangxiaolu1993\n\nsnandy\n\nEiinu\n\nYmm\n\nailululu\n\nlkjh3214\n\nyangjinjun3\n\n贡献者们\n\n感谢以下小伙伴们为 NutUI 发展做出的贡献：\n\n共建指南\n\n贡献指南\n\nPR\n\n开发者\n\n开源协议\n\nMIT\n\n## 扫码体验\n\n![NutUI](https://example.org/qr.png)\n\n## 链接", + "ok": true, + "expected_task_pass": true + }, + { + "id": "heading-only-body-region-missing-heading", + "category": "body-scope", + "root_cause": "separate_headings_not_contiguous_body_fragment", + "failure_mode": "false-positive", + "case": { + "url": "https://nutui.jd.com/h5/vue/4x/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "body_text", + "minimum_records": 0, + "stable_text_anchors": [ + { + "id": "nutui-independent-heading-0", + "alternatives": [ + "扫码体验" + ], + "source_location": "main heading", + "observation_status": "independent source visible headings; match separately, no cross-heading adjacency requirement" + }, + { + "id": "nutui-independent-heading-1", + "alternatives": [ + "链接" + ], + "source_location": "main heading", + "observation_status": "independent source visible headings; match separately, no cross-heading adjacency requirement" } + ] }, - "artifact": "[New Movie](/filme/new-movie)\n[Another Movie](/filme/another-movie)", - "ok": true, - "expected_task_pass": true - }, - { - "id": "joyn-movie-ownership-genres-only", - "category": "structured-extraction", - "root_cause": "movie_records_not_genre_navigation_or_placeholder", - "failure_mode": "false-positive", - "case": { - "url": "https://www.joyn.de/filme", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 2, - "hosts": [ - "www.joyn.de" - ], - "entry_path": "/filme/(?!genre(?:/|$))[^/]+", - "stable_text_anchors": [], - "name_validation": "nonempty", - "entry_scope": { - "html_selector": "main div[class*=\"CardStandard_StandardCardWrapper\"]", - "title_selector": "h3", - "link_selector": "a[data-testid=\"LL--movie\"]" - } + "body_requirements": { + "version": 3, + "unit": "region_compact_hexagram_recall", + "minimum_coverage": 0.8, + "minimum_floor": 0.6, + "minimum_region_passes": 4, + "segmentation_method": "sections", + "regions": [ + { + "id": "body-region-1", + "text": "nutui nutui vue 组 件 库 适 用 于 vue 技 术 栈 的 h5 应 用 开 箱 即 用 帮 助 研 发 快 速 开 发 用 户 界 面 提 升 开 发 效 率 改 善 开 发 体 验", + "source_text": "NutUI\n\nNutUI-Vue 组件库，适用于 Vue 技术栈的 H5 应用，开箱即用，帮助研发快速开发用户界面，提升开发效率，改善开发体验。", + "segmentation_method": "sections" + }, + { + "id": "body-region-2", + "text": "特 性 80 高 质 量 组 件 覆 盖 移 动 端 主 流 场 景 基 于 京 东 app 10 0 视 觉 规 范 支 持 按 需 引 用 详 尽 的 文 档 和 示 例 支 持 typescript 支 持 webstorm vscode 组 件 属 性 高 亮 支 持 动 态 定 制 主 题 支 持 暗 黑 模 式 支 持 国 际 化 单 元 测 试 覆 盖 率 超 过 80 保 障 稳 定 性 提 供 sketch 设 计 资 源", + "source_text": "特性\n\n🚀 80+ 高质量组件，覆盖移动端主流场景\n\n📖 基于京东 APP 10.0 视觉规范\n\n🍭 支持按需引用\n\n📖 详尽的文档和示例\n\n💪 支持 TypeScript\n\n💡 支持 webstorm，vscode 组件属性高亮\n\n💪 支持动态定制主题\n\n🍭 支持暗黑模式\n\n🌍 支持国际化\n\n🍭 单元测试覆盖率超过 80%，保障稳定性\n\n📖 提供 Sketch 设计资源", + "segmentation_method": "sections" + }, + { + "id": "body-region-4", + "text": "官 方 生 态 由 nutui 官 方 团 队 维 护 的 项 目 如 下 项 目 描 述 nutui nutui nutui 移 动 端 h5 版 本 nutui nutui taro nutui taro 多 端 版 本 nutui icons vue nutui nutui 使 用 的 图 标 库 nutui icons vue taro nutui nutui taro 使 用 的 图 标 库 nutui touch emulator 在 桌 面 端 使 用 nutui 的 辅 助 库 nutui playground nutui 在 线 playground nutui demo nutui 官 方 示 例 合 集", + "source_text": "官方生态\n\n由 NutUI 官方团队维护的项目如下：\n\n项目 描述\n\n@nutui/nutui NutUI 移动端 H5 版本\n\n@nutui/nutui-taro NutUI Taro 多端版本\n\n@nutui/icons-vue @nutui/nutui 使用的图标库\n\n@nutui/icons-vue-taro @nutui/nutui-taro 使用的图标库\n\n@nutui/touch-emulator 在桌面端使用 NutUI 的辅助库\n\n@nutui/playground NutUI 在线 Playground\n\nnutui-demo NutUI 官方示例合集", + "segmentation_method": "sections" + }, + { + "id": "body-region-5", + "text": "社 区 生 态 由 社 区 维 护 的 项 目 如 下 欢 迎 补 充 项 目 描 述 yang1206 uniapp nutui 基 于 nutui 4 x 版 本 开 发 的 uni app 组 件 库 jwaterwater uni nutui 基 于 nutui 3 x 版 本 开 发 的 uni app 组 件 库 贡 献 核 心 人 员 以 下 是 nutui 的 核 心 贡 献 者 们 richard1015 drjingfubo love forever yangxiaolu1993 snandy eiinu ymm ailululu lkjh3214 yangjinjun3 贡 献 者 们 感 谢 以 下 小 伙 伴 们 为 nutui 发 展 做 出 的 贡 献 共 建 指 南 贡 献 指 南 pr 开 发 者 开 源 协 议 mit", + "source_text": "社区生态\n\n由社区维护的项目如下，欢迎补充：\n\n项目 描述\n\nyang1206/uniapp-nutui 基于 NutUI 4.x 版本开发的 uni-app 组件库\n\njwaterwater/uni-nutui 基于 NutUI 3.x 版本开发的 uni-app 组件库\n\n贡献\n\n核心人员\n\n以下是 NutUI 的核心贡献者们：\n\nrichard1015\n\nDrjingfubo\n\nlove_forever\n\nyangxiaolu1993\n\nsnandy\n\nEiinu\n\nYmm\n\nailululu\n\nlkjh3214\n\nyangjinjun3\n\n贡献者们\n\n感谢以下小伙伴们为 NutUI 发展做出的贡献：\n\n共建指南\n\n贡献指南\n\nPR\n\n开发者\n\n开源协议\n\nMIT", + "segmentation_method": "sections" } - }, - "artifact": "[Drama](/filme/genre/drama)\n[Comedy](/filme/genre/comedy)", - "ok": true, - "expected_task_pass": false - }, - { - "id": "joyn-movie-ownership-duplicated-movie", - "category": "structured-extraction", - "root_cause": "movie_records_not_genre_navigation_or_placeholder", - "failure_mode": "false-positive", - "case": { - "url": "https://www.joyn.de/filme", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 2, - "hosts": [ - "www.joyn.de" - ], - "entry_path": "/filme/(?!genre(?:/|$))[^/]+", - "stable_text_anchors": [], - "name_validation": "nonempty", - "entry_scope": { - "html_selector": "main div[class*=\"CardStandard_StandardCardWrapper\"]", - "title_selector": "h3", - "link_selector": "a[data-testid=\"LL--movie\"]" - } + ] + } + }, + "artifact": "NutUI\n\nNutUI-Vue 组件库，适用于 Vue 技术栈的 H5 应用，开箱即用，帮助研发快速开发用户界面，提升开发效率，改善开发体验。\n\n特性\n\n🚀 80+ 高质量组件，覆盖移动端主流场景\n\n📖 基于京东 APP 10.0 视觉规范\n\n🍭 支持按需引用\n\n📖 详尽的文档和示例\n\n💪 支持 TypeScript\n\n💡 支持 webstorm，vscode 组件属性高亮\n\n💪 支持动态定制主题\n\n🍭 支持暗黑模式\n\n🌍 支持国际化\n\n🍭 单元测试覆盖率超过 80%，保障稳定性\n\n📖 提供 Sketch 设计资源\n\n官方生态\n\n由 NutUI 官方团队维护的项目如下：\n\n项目 描述\n\n@nutui/nutui NutUI 移动端 H5 版本\n\n@nutui/nutui-taro NutUI Taro 多端版本\n\n@nutui/icons-vue @nutui/nutui 使用的图标库\n\n@nutui/icons-vue-taro @nutui/nutui-taro 使用的图标库\n\n@nutui/touch-emulator 在桌面端使用 NutUI 的辅助库\n\n@nutui/playground NutUI 在线 Playground\n\nnutui-demo NutUI 官方示例合集\n\n社区生态\n\n由社区维护的项目如下，欢迎补充：\n\n项目 描述\n\nyang1206/uniapp-nutui 基于 NutUI 4.x 版本开发的 uni-app 组件库\n\njwaterwater/uni-nutui 基于 NutUI 3.x 版本开发的 uni-app 组件库\n\n贡献\n\n核心人员\n\n以下是 NutUI 的核心贡献者们：\n\nrichard1015\n\nDrjingfubo\n\nlove_forever\n\nyangxiaolu1993\n\nsnandy\n\nEiinu\n\nYmm\n\nailululu\n\nlkjh3214\n\nyangjinjun3\n\n贡献者们\n\n感谢以下小伙伴们为 NutUI 发展做出的贡献：\n\n共建指南\n\n贡献指南\n\nPR\n\n开发者\n\n开源协议\n\nMIT\n\n## 链接", + "ok": true, + "expected_task_pass": false + }, + { + "id": "heading-only-body-region-headings-only", + "category": "body-scope", + "root_cause": "separate_headings_not_contiguous_body_fragment", + "failure_mode": "false-positive", + "case": { + "url": "https://nutui.jd.com/h5/vue/4x/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "body_text", + "minimum_records": 0, + "stable_text_anchors": [ + { + "id": "nutui-independent-heading-0", + "alternatives": [ + "扫码体验" + ], + "source_location": "main heading", + "observation_status": "independent source visible headings; match separately, no cross-heading adjacency requirement" + }, + { + "id": "nutui-independent-heading-1", + "alternatives": [ + "链接" + ], + "source_location": "main heading", + "observation_status": "independent source visible headings; match separately, no cross-heading adjacency requirement" } + ] }, - "artifact": "[New Movie](/filme/new-movie)\n[New Movie](/filme/new-movie)", - "ok": true, - "expected_task_pass": false - }, - { - "id": "joyn-movie-ownership-empty-name", - "category": "structured-extraction", - "root_cause": "movie_records_not_genre_navigation_or_placeholder", - "failure_mode": "false-positive", - "case": { - "url": "https://www.joyn.de/filme", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 2, - "hosts": [ - "www.joyn.de" - ], - "entry_path": "/filme/(?!genre(?:/|$))[^/]+", - "stable_text_anchors": [], - "name_validation": "nonempty", - "entry_scope": { - "html_selector": "main div[class*=\"CardStandard_StandardCardWrapper\"]", - "title_selector": "h3", - "link_selector": "a[data-testid=\"LL--movie\"]" - } + "body_requirements": { + "version": 3, + "unit": "region_compact_hexagram_recall", + "minimum_coverage": 0.8, + "minimum_floor": 0.6, + "minimum_region_passes": 4, + "segmentation_method": "sections", + "regions": [ + { + "id": "body-region-1", + "text": "nutui nutui vue 组 件 库 适 用 于 vue 技 术 栈 的 h5 应 用 开 箱 即 用 帮 助 研 发 快 速 开 发 用 户 界 面 提 升 开 发 效 率 改 善 开 发 体 验", + "source_text": "NutUI\n\nNutUI-Vue 组件库，适用于 Vue 技术栈的 H5 应用，开箱即用，帮助研发快速开发用户界面，提升开发效率，改善开发体验。", + "segmentation_method": "sections" + }, + { + "id": "body-region-2", + "text": "特 性 80 高 质 量 组 件 覆 盖 移 动 端 主 流 场 景 基 于 京 东 app 10 0 视 觉 规 范 支 持 按 需 引 用 详 尽 的 文 档 和 示 例 支 持 typescript 支 持 webstorm vscode 组 件 属 性 高 亮 支 持 动 态 定 制 主 题 支 持 暗 黑 模 式 支 持 国 际 化 单 元 测 试 覆 盖 率 超 过 80 保 障 稳 定 性 提 供 sketch 设 计 资 源", + "source_text": "特性\n\n🚀 80+ 高质量组件，覆盖移动端主流场景\n\n📖 基于京东 APP 10.0 视觉规范\n\n🍭 支持按需引用\n\n📖 详尽的文档和示例\n\n💪 支持 TypeScript\n\n💡 支持 webstorm，vscode 组件属性高亮\n\n💪 支持动态定制主题\n\n🍭 支持暗黑模式\n\n🌍 支持国际化\n\n🍭 单元测试覆盖率超过 80%，保障稳定性\n\n📖 提供 Sketch 设计资源", + "segmentation_method": "sections" + }, + { + "id": "body-region-4", + "text": "官 方 生 态 由 nutui 官 方 团 队 维 护 的 项 目 如 下 项 目 描 述 nutui nutui nutui 移 动 端 h5 版 本 nutui nutui taro nutui taro 多 端 版 本 nutui icons vue nutui nutui 使 用 的 图 标 库 nutui icons vue taro nutui nutui taro 使 用 的 图 标 库 nutui touch emulator 在 桌 面 端 使 用 nutui 的 辅 助 库 nutui playground nutui 在 线 playground nutui demo nutui 官 方 示 例 合 集", + "source_text": "官方生态\n\n由 NutUI 官方团队维护的项目如下：\n\n项目 描述\n\n@nutui/nutui NutUI 移动端 H5 版本\n\n@nutui/nutui-taro NutUI Taro 多端版本\n\n@nutui/icons-vue @nutui/nutui 使用的图标库\n\n@nutui/icons-vue-taro @nutui/nutui-taro 使用的图标库\n\n@nutui/touch-emulator 在桌面端使用 NutUI 的辅助库\n\n@nutui/playground NutUI 在线 Playground\n\nnutui-demo NutUI 官方示例合集", + "segmentation_method": "sections" + }, + { + "id": "body-region-5", + "text": "社 区 生 态 由 社 区 维 护 的 项 目 如 下 欢 迎 补 充 项 目 描 述 yang1206 uniapp nutui 基 于 nutui 4 x 版 本 开 发 的 uni app 组 件 库 jwaterwater uni nutui 基 于 nutui 3 x 版 本 开 发 的 uni app 组 件 库 贡 献 核 心 人 员 以 下 是 nutui 的 核 心 贡 献 者 们 richard1015 drjingfubo love forever yangxiaolu1993 snandy eiinu ymm ailululu lkjh3214 yangjinjun3 贡 献 者 们 感 谢 以 下 小 伙 伴 们 为 nutui 发 展 做 出 的 贡 献 共 建 指 南 贡 献 指 南 pr 开 发 者 开 源 协 议 mit", + "source_text": "社区生态\n\n由社区维护的项目如下，欢迎补充：\n\n项目 描述\n\nyang1206/uniapp-nutui 基于 NutUI 4.x 版本开发的 uni-app 组件库\n\njwaterwater/uni-nutui 基于 NutUI 3.x 版本开发的 uni-app 组件库\n\n贡献\n\n核心人员\n\n以下是 NutUI 的核心贡献者们：\n\nrichard1015\n\nDrjingfubo\n\nlove_forever\n\nyangxiaolu1993\n\nsnandy\n\nEiinu\n\nYmm\n\nailululu\n\nlkjh3214\n\nyangjinjun3\n\n贡献者们\n\n感谢以下小伙伴们为 NutUI 发展做出的贡献：\n\n共建指南\n\n贡献指南\n\nPR\n\n开发者\n\n开源协议\n\nMIT", + "segmentation_method": "sections" } - }, - "artifact": "[](/filme/new-movie)\n[Another Movie](/filme/another-movie)", - "ok": true, - "expected_task_pass": false - }, - { - "id": "joyn-movie-ownership-html-placeholders", - "category": "structured-extraction", - "root_cause": "movie_records_not_genre_navigation_or_placeholder", - "failure_mode": "false-positive", - "case": { - "url": "https://www.joyn.de/filme", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 2, - "hosts": [ - "www.joyn.de" - ], - "entry_path": "/filme/(?!genre(?:/|$))[^/]+", - "stable_text_anchors": [], - "name_validation": "nonempty", - "entry_scope": { - "html_selector": "main div[class*=\"CardStandard_StandardCardWrapper\"]", - "title_selector": "h3", - "link_selector": "a[data-testid=\"LL--movie\"]" - } + ] + } + }, + "artifact": "## 扫码体验\n\n![NutUI](https://example.org/qr.png)\n\n## 链接", + "ok": true, + "expected_task_pass": false + }, + { + "id": "detail-url-unreserved-encoding-unreserved-encoded", + "category": "format-normalization", + "root_cause": "unreserved_url_encoding_preserves_detail_identity", + "failure_mode": "false-negative", + "case": { + "url": "https://www.allocine.fr/film/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "www.allocine.fr" + ], + "entry_path": "/film/fichefilm_gen_cfilm=[0-9]+\\.html", + "stable_text_anchors": [], + "entry_scope": { + "html_selector": "main a.meta-title-link, main a.list-entity-item-link, main a.light, main a.show-name" + }, + "name_validation": "nonempty" + } + }, + "artifact": "[Film Alpha](/film/fichefilm%5Fgen%5Fcfilm=123.html)\n[Film Beta](/film/fichefilm_gen_cfilm=456.html)", + "ok": true, + "expected_task_pass": true + }, + { + "id": "detail-url-unreserved-encoding-encoded-duplicate", + "category": "format-normalization", + "root_cause": "unreserved_url_encoding_preserves_detail_identity", + "failure_mode": "false-positive", + "case": { + "url": "https://www.allocine.fr/film/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "www.allocine.fr" + ], + "entry_path": "/film/fichefilm_gen_cfilm=[0-9]+\\.html", + "stable_text_anchors": [], + "entry_scope": { + "html_selector": "main a.meta-title-link, main a.list-entity-item-link, main a.light, main a.show-name" + }, + "name_validation": "nonempty" + } + }, + "artifact": "[Film Alpha](/film/fichefilm%5Fgen%5Fcfilm=123.html)\n[Film Alpha](/film/fichefilm_gen_cfilm=123.html)", + "ok": true, + "expected_task_pass": false + }, + { + "id": "detail-url-unreserved-encoding-reserved-separator", + "category": "format-normalization", + "root_cause": "unreserved_url_encoding_preserves_detail_identity", + "failure_mode": "false-positive", + "case": { + "url": "https://www.allocine.fr/film/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "www.allocine.fr" + ], + "entry_path": "/film/fichefilm_gen_cfilm=[0-9]+\\.html", + "stable_text_anchors": [], + "entry_scope": { + "html_selector": "main a.meta-title-link, main a.list-entity-item-link, main a.light, main a.show-name" + }, + "name_validation": "nonempty" + } + }, + "artifact": "[Film Alpha](/film/fichefilm%5Fgen%5Fcfilm=123.html)\n[Film Beta](/film%2Ffichefilm_gen_cfilm=456.html)", + "ok": true, + "expected_task_pass": false + }, + { + "id": "detail-url-unreserved-encoding-empty-name", + "category": "format-normalization", + "root_cause": "unreserved_url_encoding_preserves_detail_identity", + "failure_mode": "false-positive", + "case": { + "url": "https://www.allocine.fr/film/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "www.allocine.fr" + ], + "entry_path": "/film/fichefilm_gen_cfilm=[0-9]+\\.html", + "stable_text_anchors": [], + "entry_scope": { + "html_selector": "main a.meta-title-link, main a.list-entity-item-link, main a.light, main a.show-name" + }, + "name_validation": "nonempty" + } + }, + "artifact": "[Film Alpha](/film/fichefilm%5Fgen%5Fcfilm=123.html)\n[](/film/fichefilm_gen_cfilm=456.html)", + "ok": true, + "expected_task_pass": false + }, + { + "id": "detail-url-unreserved-encoding-wrong-detail-path", + "category": "format-normalization", + "root_cause": "unreserved_url_encoding_preserves_detail_identity", + "failure_mode": "false-positive", + "case": { + "url": "https://www.allocine.fr/film/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "www.allocine.fr" + ], + "entry_path": "/film/fichefilm_gen_cfilm=[0-9]+\\.html", + "stable_text_anchors": [], + "entry_scope": { + "html_selector": "main a.meta-title-link, main a.list-entity-item-link, main a.light, main a.show-name" + }, + "name_validation": "nonempty" + } + }, + "artifact": "[Film Alpha](/film/fichefilm%5Fgen%5Fcfilm=123.html)\n[Film Beta](/film/fichefilm_gen_cfilm=456.json)", + "ok": true, + "expected_task_pass": false + }, + { + "id": "named-link-separated-list-comma-list", + "category": "format-normalization", + "root_cause": "punctuation_separated_explicit_named_links", + "failure_mode": "false-negative", + "case": { + "url": "https://www.allocine.fr/film/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "www.allocine.fr" + ], + "entry_path": "/film/fichefilm_gen_cfilm=[0-9]+\\.html", + "stable_text_anchors": [], + "entry_scope": { + "html_selector": "main a.meta-title-link, main a.list-entity-item-link, main a.light, main a.show-name" + }, + "name_validation": "nonempty" + } + }, + "artifact": "[Film Alpha](/film/fichefilm_gen_cfilm=123.html), [Film Beta](/film/fichefilm_gen_cfilm=456.html)", + "ok": true, + "expected_task_pass": true + }, + { + "id": "named-link-separated-list-semicolon-list", + "category": "format-normalization", + "root_cause": "punctuation_separated_explicit_named_links", + "failure_mode": "false-negative", + "case": { + "url": "https://www.allocine.fr/film/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "www.allocine.fr" + ], + "entry_path": "/film/fichefilm_gen_cfilm=[0-9]+\\.html", + "stable_text_anchors": [], + "entry_scope": { + "html_selector": "main a.meta-title-link, main a.list-entity-item-link, main a.light, main a.show-name" + }, + "name_validation": "nonempty" + } + }, + "artifact": "[Film Alpha](/film/fichefilm_gen_cfilm=123.html); [Film Beta](/film/fichefilm_gen_cfilm=456.html)", + "ok": true, + "expected_task_pass": true + }, + { + "id": "named-link-separated-list-prose-recommendation", + "category": "format-normalization", + "root_cause": "punctuation_separated_explicit_named_links", + "failure_mode": "false-positive", + "case": { + "url": "https://www.allocine.fr/film/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "www.allocine.fr" + ], + "entry_path": "/film/fichefilm_gen_cfilm=[0-9]+\\.html", + "stable_text_anchors": [], + "entry_scope": { + "html_selector": "main a.meta-title-link, main a.list-entity-item-link, main a.light, main a.show-name" + }, + "name_validation": "nonempty" + } + }, + "artifact": "Watch [Film Alpha](/film/fichefilm_gen_cfilm=123.html) or [Film Beta](/film/fichefilm_gen_cfilm=456.html)", + "ok": true, + "expected_task_pass": false + }, + { + "id": "named-link-separated-list-prose-between-links", + "category": "format-normalization", + "root_cause": "punctuation_separated_explicit_named_links", + "failure_mode": "false-positive", + "case": { + "url": "https://www.allocine.fr/film/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "www.allocine.fr" + ], + "entry_path": "/film/fichefilm_gen_cfilm=[0-9]+\\.html", + "stable_text_anchors": [], + "entry_scope": { + "html_selector": "main a.meta-title-link, main a.list-entity-item-link, main a.light, main a.show-name" + }, + "name_validation": "nonempty" + } + }, + "artifact": "[Film Alpha](/film/fichefilm_gen_cfilm=123.html) is better than [Film Beta](/film/fichefilm_gen_cfilm=456.html)", + "ok": true, + "expected_task_pass": false + }, + { + "id": "joyn-movie-ownership-fresh-movies", + "category": "structured-extraction", + "root_cause": "movie_records_not_genre_navigation_or_placeholder", + "failure_mode": "false-negative", + "case": { + "url": "https://www.joyn.de/filme", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "www.joyn.de" + ], + "entry_path": "/filme/(?!genre(?:/|$))[^/]+", + "stable_text_anchors": [], + "name_validation": "nonempty", + "entry_scope": { + "html_selector": "main div[class*=\"CardStandard_StandardCardWrapper\"]", + "title_selector": "h3", + "link_selector": "a[data-testid=\"LL--movie\"]" + } + } + }, + "artifact": "[New Movie](/filme/new-movie)\n[Another Movie](/filme/another-movie)", + "ok": true, + "expected_task_pass": true + }, + { + "id": "joyn-movie-ownership-genres-only", + "category": "structured-extraction", + "root_cause": "movie_records_not_genre_navigation_or_placeholder", + "failure_mode": "false-positive", + "case": { + "url": "https://www.joyn.de/filme", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "www.joyn.de" + ], + "entry_path": "/filme/(?!genre(?:/|$))[^/]+", + "stable_text_anchors": [], + "name_validation": "nonempty", + "entry_scope": { + "html_selector": "main div[class*=\"CardStandard_StandardCardWrapper\"]", + "title_selector": "h3", + "link_selector": "a[data-testid=\"LL--movie\"]" + } + } + }, + "artifact": "[Drama](/filme/genre/drama)\n[Comedy](/filme/genre/comedy)", + "ok": true, + "expected_task_pass": false + }, + { + "id": "joyn-movie-ownership-duplicated-movie", + "category": "structured-extraction", + "root_cause": "movie_records_not_genre_navigation_or_placeholder", + "failure_mode": "false-positive", + "case": { + "url": "https://www.joyn.de/filme", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "www.joyn.de" + ], + "entry_path": "/filme/(?!genre(?:/|$))[^/]+", + "stable_text_anchors": [], + "name_validation": "nonempty", + "entry_scope": { + "html_selector": "main div[class*=\"CardStandard_StandardCardWrapper\"]", + "title_selector": "h3", + "link_selector": "a[data-testid=\"LL--movie\"]" + } + } + }, + "artifact": "[New Movie](/filme/new-movie)\n[New Movie](/filme/new-movie)", + "ok": true, + "expected_task_pass": false + }, + { + "id": "joyn-movie-ownership-empty-name", + "category": "structured-extraction", + "root_cause": "movie_records_not_genre_navigation_or_placeholder", + "failure_mode": "false-positive", + "case": { + "url": "https://www.joyn.de/filme", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "www.joyn.de" + ], + "entry_path": "/filme/(?!genre(?:/|$))[^/]+", + "stable_text_anchors": [], + "name_validation": "nonempty", + "entry_scope": { + "html_selector": "main div[class*=\"CardStandard_StandardCardWrapper\"]", + "title_selector": "h3", + "link_selector": "a[data-testid=\"LL--movie\"]" + } + } + }, + "artifact": "[](/filme/new-movie)\n[Another Movie](/filme/another-movie)", + "ok": true, + "expected_task_pass": false + }, + { + "id": "joyn-movie-ownership-html-placeholders", + "category": "structured-extraction", + "root_cause": "movie_records_not_genre_navigation_or_placeholder", + "failure_mode": "false-positive", + "case": { + "url": "https://www.joyn.de/filme", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "www.joyn.de" + ], + "entry_path": "/filme/(?!genre(?:/|$))[^/]+", + "stable_text_anchors": [], + "name_validation": "nonempty", + "entry_scope": { + "html_selector": "main div[class*=\"CardStandard_StandardCardWrapper\"]", + "title_selector": "h3", + "link_selector": "a[data-testid=\"LL--movie\"]" + } + } + }, + "artifact": "
Hidden titleHidden title2
", + "ok": true, + "expected_task_pass": false + }, + { + "id": "prosieben-owned-content-fresh-content", + "category": "structured-extraction", + "root_cause": "content_cards_not_placeholder_or_editorial_links", + "failure_mode": "false-negative", + "case": { + "url": "https://www.joyn.de/mediatheken/prosieben", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "www.joyn.de" + ], + "entry_path": "/(?:serien|filme|shows)/(?!genre(?:/|$))[^/]+", + "stable_text_anchors": [], + "name_validation": "nonempty", + "entry_scope": { + "html_selector": "main div[class*=\"CardStandard_StandardCardWrapper\"]", + "title_selector": "h3", + "link_selector": "a[data-testid=\"LL--movie\"],a[data-testid=\"LL--series\"]", + "start_pattern": "Meistgesehene Sendungen", + "end_pattern": "ProSieben Mediathek bei Joyn: Alle Shows, Serien und Filme online streamen" + } + } + }, + "artifact": "Meistgesehene Sendungen\n[New Show](/serien/new-show)\n[New Movie](/filme/new-movie)\nProSieben Mediathek bei Joyn: Alle Shows, Serien und Filme online streamen", + "ok": true, + "expected_task_pass": true + }, + { + "id": "prosieben-owned-content-editorial-only", + "category": "structured-extraction", + "root_cause": "content_cards_not_placeholder_or_editorial_links", + "failure_mode": "false-positive", + "case": { + "url": "https://www.joyn.de/mediatheken/prosieben", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "www.joyn.de" + ], + "entry_path": "/(?:serien|filme|shows)/(?!genre(?:/|$))[^/]+", + "stable_text_anchors": [], + "name_validation": "nonempty", + "entry_scope": { + "html_selector": "main div[class*=\"CardStandard_StandardCardWrapper\"]", + "title_selector": "h3", + "link_selector": "a[data-testid=\"LL--movie\"],a[data-testid=\"LL--series\"]", + "start_pattern": "Meistgesehene Sendungen", + "end_pattern": "ProSieben Mediathek bei Joyn: Alle Shows, Serien und Filme online streamen" + } + } + }, + "artifact": "Meistgesehene Sendungen\nProSieben Mediathek bei Joyn: Alle Shows, Serien und Filme online streamen\n[New Show](/serien/new-show)\n[New Movie](/filme/new-movie)", + "ok": true, + "expected_task_pass": false + }, + { + "id": "prosieben-owned-content-missing-list-region", + "category": "structured-extraction", + "root_cause": "content_cards_not_placeholder_or_editorial_links", + "failure_mode": "false-positive", + "case": { + "url": "https://www.joyn.de/mediatheken/prosieben", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "www.joyn.de" + ], + "entry_path": "/(?:serien|filme|shows)/(?!genre(?:/|$))[^/]+", + "stable_text_anchors": [], + "name_validation": "nonempty", + "entry_scope": { + "html_selector": "main div[class*=\"CardStandard_StandardCardWrapper\"]", + "title_selector": "h3", + "link_selector": "a[data-testid=\"LL--movie\"],a[data-testid=\"LL--series\"]", + "start_pattern": "Meistgesehene Sendungen", + "end_pattern": "ProSieben Mediathek bei Joyn: Alle Shows, Serien und Filme online streamen" + } + } + }, + "artifact": "ProSieben Mediathek bei Joyn: Alle Shows, Serien und Filme online streamen\n[New Show](/serien/new-show)\n[New Movie](/filme/new-movie)", + "ok": true, + "expected_task_pass": false + }, + { + "id": "prosieben-owned-content-genres", + "category": "structured-extraction", + "root_cause": "content_cards_not_placeholder_or_editorial_links", + "failure_mode": "false-positive", + "case": { + "url": "https://www.joyn.de/mediatheken/prosieben", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "www.joyn.de" + ], + "entry_path": "/(?:serien|filme|shows)/(?!genre(?:/|$))[^/]+", + "stable_text_anchors": [], + "name_validation": "nonempty", + "entry_scope": { + "html_selector": "main div[class*=\"CardStandard_StandardCardWrapper\"]", + "title_selector": "h3", + "link_selector": "a[data-testid=\"LL--movie\"],a[data-testid=\"LL--series\"]", + "start_pattern": "Meistgesehene Sendungen", + "end_pattern": "ProSieben Mediathek bei Joyn: Alle Shows, Serien und Filme online streamen" + } + } + }, + "artifact": "Meistgesehene Sendungen\n[Drama](/filme/genre/drama)\n[Comedy](/serien/genre/comedy)", + "ok": true, + "expected_task_pass": false + }, + { + "id": "prosieben-owned-content-placeholder-html", + "category": "structured-extraction", + "root_cause": "content_cards_not_placeholder_or_editorial_links", + "failure_mode": "false-positive", + "case": { + "url": "https://www.joyn.de/mediatheken/prosieben", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "www.joyn.de" + ], + "entry_path": "/(?:serien|filme|shows)/(?!genre(?:/|$))[^/]+", + "stable_text_anchors": [], + "name_validation": "nonempty", + "entry_scope": { + "html_selector": "main div[class*=\"CardStandard_StandardCardWrapper\"]", + "title_selector": "h3", + "link_selector": "a[data-testid=\"LL--movie\"],a[data-testid=\"LL--series\"]", + "start_pattern": "Meistgesehene Sendungen", + "end_pattern": "ProSieben Mediathek bei Joyn: Alle Shows, Serien und Filme online streamen" + } + } + }, + "artifact": "
New ShowNew Movie
", + "ok": true, + "expected_task_pass": false + }, + { + "id": "francetv-content-cards-real-content-paths", + "category": "structured-extraction", + "root_cause": "content_card_routes_not_partial_channel_allowlist", + "failure_mode": "false-negative", + "case": { + "url": "https://www.france.tv/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "www.france.tv" + ], + "entry_path": "/(?:arte|collection|documentaire|documentaires|films|france\\-2|france\\-24|france\\-3|france\\-4|france\\-5|franceinfo|ina|info|la1ere|lcp|lcp\\-public\\-senat|mieux|public\\-senat|serie|series\\-et\\-fictions|slash|societe|spectacles\\-et\\-culture|sport|tv5\\-monde)/.+", + "stable_text_anchors": [], + "name_validation": "nonempty", + "entry_scope": { + "html_selector": "main li:has(> a[data-card-link=\"true\"])", + "title_selector": ":scope > a[data-card-link] > div:first-child > span:first-child, :scope > a[data-card-link] > div:first-child > div:first-child > span", + "link_selector": ":scope > a[data-card-link=\"true\"]" + } + } + }, + "artifact": "[Programme Nouveau](/arte/programme-nouveau/)\n[Collection Nouvelle](/collection/collection-nouvelle/)", + "ok": true, + "expected_task_pass": true + }, + { + "id": "francetv-content-cards-root-navigation", + "category": "structured-extraction", + "root_cause": "content_card_routes_not_partial_channel_allowlist", + "failure_mode": "false-positive", + "case": { + "url": "https://www.france.tv/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "www.france.tv" + ], + "entry_path": "/(?:arte|collection|documentaire|documentaires|films|france\\-2|france\\-24|france\\-3|france\\-4|france\\-5|franceinfo|ina|info|la1ere|lcp|lcp\\-public\\-senat|mieux|public\\-senat|serie|series\\-et\\-fictions|slash|societe|spectacles\\-et\\-culture|sport|tv5\\-monde)/.+", + "stable_text_anchors": [], + "name_validation": "nonempty", + "entry_scope": { + "html_selector": "main li:has(> a[data-card-link=\"true\"])", + "title_selector": ":scope > a[data-card-link] > div:first-child > span:first-child, :scope > a[data-card-link] > div:first-child > div:first-child > span", + "link_selector": ":scope > a[data-card-link=\"true\"]" + } + } + }, + "artifact": "[Films](/films/)\n[Arte](/arte/)", + "ok": true, + "expected_task_pass": false + }, + { + "id": "francetv-content-cards-empty-name", + "category": "structured-extraction", + "root_cause": "content_card_routes_not_partial_channel_allowlist", + "failure_mode": "false-positive", + "case": { + "url": "https://www.france.tv/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "www.france.tv" + ], + "entry_path": "/(?:arte|collection|documentaire|documentaires|films|france\\-2|france\\-24|france\\-3|france\\-4|france\\-5|franceinfo|ina|info|la1ere|lcp|lcp\\-public\\-senat|mieux|public\\-senat|serie|series\\-et\\-fictions|slash|societe|spectacles\\-et\\-culture|sport|tv5\\-monde)/.+", + "stable_text_anchors": [], + "name_validation": "nonempty", + "entry_scope": { + "html_selector": "main li:has(> a[data-card-link=\"true\"])", + "title_selector": ":scope > a[data-card-link] > div:first-child > span:first-child, :scope > a[data-card-link] > div:first-child > div:first-child > span", + "link_selector": ":scope > a[data-card-link=\"true\"]" + } + } + }, + "artifact": "[](/arte/programme-nouveau/)\n[Collection Nouvelle](/collection/collection-nouvelle/)", + "ok": true, + "expected_task_pass": false + }, + { + "id": "francetv-content-cards-foreign-host", + "category": "structured-extraction", + "root_cause": "content_card_routes_not_partial_channel_allowlist", + "failure_mode": "false-positive", + "case": { + "url": "https://www.france.tv/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "www.france.tv" + ], + "entry_path": "/(?:arte|collection|documentaire|documentaires|films|france\\-2|france\\-24|france\\-3|france\\-4|france\\-5|franceinfo|ina|info|la1ere|lcp|lcp\\-public\\-senat|mieux|public\\-senat|serie|series\\-et\\-fictions|slash|societe|spectacles\\-et\\-culture|sport|tv5\\-monde)/.+", + "stable_text_anchors": [], + "name_validation": "nonempty", + "entry_scope": { + "html_selector": "main li:has(> a[data-card-link=\"true\"])", + "title_selector": ":scope > a[data-card-link] > div:first-child > span:first-child, :scope > a[data-card-link] > div:first-child > div:first-child > span", + "link_selector": ":scope > a[data-card-link=\"true\"]" + } + } + }, + "artifact": "[Programme](https://example.org/arte/programme/)\n[Collection](https://example.org/collection/nouvelle/)", + "ok": true, + "expected_task_pass": false + }, + { + "id": "francetv-content-cards-same-id-twice", + "category": "structured-extraction", + "root_cause": "content_card_routes_not_partial_channel_allowlist", + "failure_mode": "false-positive", + "case": { + "url": "https://www.france.tv/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "www.france.tv" + ], + "entry_path": "/(?:arte|collection|documentaire|documentaires|films|france\\-2|france\\-24|france\\-3|france\\-4|france\\-5|franceinfo|ina|info|la1ere|lcp|lcp\\-public\\-senat|mieux|public\\-senat|serie|series\\-et\\-fictions|slash|societe|spectacles\\-et\\-culture|sport|tv5\\-monde)/.+", + "stable_text_anchors": [], + "name_validation": "nonempty", + "entry_scope": { + "html_selector": "main li:has(> a[data-card-link=\"true\"])", + "title_selector": ":scope > a[data-card-link] > div:first-child > span:first-child, :scope > a[data-card-link] > div:first-child > div:first-child > span", + "link_selector": ":scope > a[data-card-link=\"true\"]" + } + } + }, + "artifact": "[Programme](/arte/programme/)\n[Programme](/arte/programme/)", + "ok": true, + "expected_task_pass": false + }, + { + "id": "rtve-title-scope-new-programme-and-episode", + "category": "structured-extraction", + "root_cause": "programme_title_links_not_action_buttons", + "failure_mode": "false-negative", + "case": { + "url": "https://www.rtve.es/play/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "www.rtve.es" + ], + "entry_path": "/play/videos/.+", + "stable_text_anchors": [], + "name_validation": "nonempty", + "entry_scope": { + "html_selector": "div.content strong > a:has(.maintitle),div.content strong > a:has(.pretitle)" + }, + "title_exclusion_patterns": [ + "(?:\\\\?\\+\\s*info|En directo|Directos)" + ] + } + }, + "artifact": "[Programa Nuevo](/play/videos/programa-nuevo/)\n[Episodio Nuevo](/play/videos/programa-nuevo/episodio/123/)", + "ok": true, + "expected_task_pass": true + }, + { + "id": "rtve-title-scope-buttons-only", + "category": "structured-extraction", + "root_cause": "programme_title_links_not_action_buttons", + "failure_mode": "false-positive", + "case": { + "url": "https://www.rtve.es/play/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "www.rtve.es" + ], + "entry_path": "/play/videos/.+", + "stable_text_anchors": [], + "name_validation": "nonempty", + "entry_scope": { + "html_selector": "div.content strong > a:has(.maintitle),div.content strong > a:has(.pretitle)" + }, + "title_exclusion_patterns": [ + "(?:\\\\?\\+\\s*info|En directo|Directos)" + ] + } + }, + "artifact": "[+ info](/play/videos/a/)\n[\\+ info](/play/videos/b/)\n[Directos](/play/videos/directo/)", + "ok": true, + "expected_task_pass": false + }, + { + "id": "rtve-title-scope-duplicate-id", + "category": "structured-extraction", + "root_cause": "programme_title_links_not_action_buttons", + "failure_mode": "false-positive", + "case": { + "url": "https://www.rtve.es/play/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "www.rtve.es" + ], + "entry_path": "/play/videos/.+", + "stable_text_anchors": [], + "name_validation": "nonempty", + "entry_scope": { + "html_selector": "div.content strong > a:has(.maintitle),div.content strong > a:has(.pretitle)" + }, + "title_exclusion_patterns": [ + "(?:\\\\?\\+\\s*info|En directo|Directos)" + ] + } + }, + "artifact": "[Nuevo](/play/videos/nuevo/)\n[Nuevo otro texto](/play/videos/nuevo/)", + "ok": true, + "expected_task_pass": false + }, + { + "id": "rtve-title-scope-empty-title", + "category": "structured-extraction", + "root_cause": "programme_title_links_not_action_buttons", + "failure_mode": "false-positive", + "case": { + "url": "https://www.rtve.es/play/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "www.rtve.es" + ], + "entry_path": "/play/videos/.+", + "stable_text_anchors": [], + "name_validation": "nonempty", + "entry_scope": { + "html_selector": "div.content strong > a:has(.maintitle),div.content strong > a:has(.pretitle)" + }, + "title_exclusion_patterns": [ + "(?:\\\\?\\+\\s*info|En directo|Directos)" + ] + } + }, + "artifact": "[](/play/videos/a/)\n[Real](/play/videos/b/)", + "ok": true, + "expected_task_pass": false + }, + { + "id": "rtve-title-scope-audio-not-video", + "category": "structured-extraction", + "root_cause": "programme_title_links_not_action_buttons", + "failure_mode": "false-positive", + "case": { + "url": "https://www.rtve.es/play/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "www.rtve.es" + ], + "entry_path": "/play/videos/.+", + "stable_text_anchors": [], + "name_validation": "nonempty", + "entry_scope": { + "html_selector": "div.content strong > a:has(.maintitle),div.content strong > a:has(.pretitle)" + }, + "title_exclusion_patterns": [ + "(?:\\\\?\\+\\s*info|En directo|Directos)" + ] + } + }, + "artifact": "[Audio](/play/audios/programa/)\n[Real](/play/videos/b/)", + "ok": true, + "expected_task_pass": false + }, + { + "id": "rtve-title-scope-html-owned-programme-and-episode", + "category": "structured-extraction", + "root_cause": "programme_title_links_not_action_buttons", + "failure_mode": "false-negative", + "case": { + "url": "https://www.rtve.es/play/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "www.rtve.es" + ], + "entry_path": "/play/videos/.+", + "stable_text_anchors": [], + "name_validation": "nonempty", + "entry_scope": { + "html_selector": "div.content strong > a:has(.maintitle),div.content strong > a:has(.pretitle)" + }, + "title_exclusion_patterns": [ + "(?:\\\\?\\+\\s*info|En directo|Directos)" + ] + } + }, + "artifact": "
NuevoEpisodio+ info
", + "ok": true, + "expected_task_pass": true + }, + { + "id": "anchor-owned-links-md-independent-nav", + "category": "format-normalization", + "root_cause": "record_title_masking_erases_independent_navigation", + "failure_mode": "false-negative", + "case": { + "url": "https://example.org/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 1, + "hosts": [ + "example.org" + ], + "entry_path": "/items/[^/]+", + "name_validation": "nonempty", + "stable_text_anchors": [ + { + "id": "nav", + "alternatives": [ + "Library" + ], + "match_scope": "non_record_lines" } - }, - "artifact": "
Hidden titleHidden title2
", - "ok": true, - "expected_task_pass": false - }, - { - "id": "prosieben-owned-content-fresh-content", - "category": "structured-extraction", - "root_cause": "content_cards_not_placeholder_or_editorial_links", - "failure_mode": "false-negative", - "case": { - "url": "https://www.joyn.de/mediatheken/prosieben", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 2, - "hosts": [ - "www.joyn.de" - ], - "entry_path": "/(?:serien|filme|shows)/(?!genre(?:/|$))[^/]+", - "stable_text_anchors": [], - "name_validation": "nonempty", - "entry_scope": { - "html_selector": "main div[class*=\"CardStandard_StandardCardWrapper\"]", - "title_selector": "h3", - "link_selector": "a[data-testid=\"LL--movie\"],a[data-testid=\"LL--series\"]", - "start_pattern": "Meistgesehene Sendungen", - "end_pattern": "ProSieben Mediathek bei Joyn: Alle Shows, Serien und Filme online streamen" - } + ] + } + }, + "artifact": "[Library](/library/)\n[Library](/items/new)", + "ok": true, + "expected_task_pass": true + }, + { + "id": "anchor-owned-links-md-record-only", + "category": "format-normalization", + "root_cause": "record_title_masking_erases_independent_navigation", + "failure_mode": "false-positive", + "case": { + "url": "https://example.org/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 1, + "hosts": [ + "example.org" + ], + "entry_path": "/items/[^/]+", + "name_validation": "nonempty", + "stable_text_anchors": [ + { + "id": "nav", + "alternatives": [ + "Library" + ], + "match_scope": "non_record_lines" } - }, - "artifact": "Meistgesehene Sendungen\n[New Show](/serien/new-show)\n[New Movie](/filme/new-movie)\nProSieben Mediathek bei Joyn: Alle Shows, Serien und Filme online streamen", - "ok": true, - "expected_task_pass": true - }, - { - "id": "prosieben-owned-content-editorial-only", - "category": "structured-extraction", - "root_cause": "content_cards_not_placeholder_or_editorial_links", - "failure_mode": "false-positive", - "case": { - "url": "https://www.joyn.de/mediatheken/prosieben", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 2, - "hosts": [ - "www.joyn.de" - ], - "entry_path": "/(?:serien|filme|shows)/(?!genre(?:/|$))[^/]+", - "stable_text_anchors": [], - "name_validation": "nonempty", - "entry_scope": { - "html_selector": "main div[class*=\"CardStandard_StandardCardWrapper\"]", - "title_selector": "h3", - "link_selector": "a[data-testid=\"LL--movie\"],a[data-testid=\"LL--series\"]", - "start_pattern": "Meistgesehene Sendungen", - "end_pattern": "ProSieben Mediathek bei Joyn: Alle Shows, Serien und Filme online streamen" - } + ] + } + }, + "artifact": "[Library](/items/new)", + "ok": true, + "expected_task_pass": false + }, + { + "id": "anchor-owned-links-md-repeated-record", + "category": "format-normalization", + "root_cause": "record_title_masking_erases_independent_navigation", + "failure_mode": "false-positive", + "case": { + "url": "https://example.org/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 1, + "hosts": [ + "example.org" + ], + "entry_path": "/items/[^/]+", + "name_validation": "nonempty", + "stable_text_anchors": [ + { + "id": "nav", + "alternatives": [ + "Library" + ], + "match_scope": "non_record_lines" } - }, - "artifact": "Meistgesehene Sendungen\nProSieben Mediathek bei Joyn: Alle Shows, Serien und Filme online streamen\n[New Show](/serien/new-show)\n[New Movie](/filme/new-movie)", - "ok": true, - "expected_task_pass": false - }, - { - "id": "prosieben-owned-content-missing-list-region", - "category": "structured-extraction", - "root_cause": "content_cards_not_placeholder_or_editorial_links", - "failure_mode": "false-positive", - "case": { - "url": "https://www.joyn.de/mediatheken/prosieben", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 2, - "hosts": [ - "www.joyn.de" - ], - "entry_path": "/(?:serien|filme|shows)/(?!genre(?:/|$))[^/]+", - "stable_text_anchors": [], - "name_validation": "nonempty", - "entry_scope": { - "html_selector": "main div[class*=\"CardStandard_StandardCardWrapper\"]", - "title_selector": "h3", - "link_selector": "a[data-testid=\"LL--movie\"],a[data-testid=\"LL--series\"]", - "start_pattern": "Meistgesehene Sendungen", - "end_pattern": "ProSieben Mediathek bei Joyn: Alle Shows, Serien und Filme online streamen" - } + ] + } + }, + "artifact": "[Library](/items/new)\n[Library](/items/new)", + "ok": true, + "expected_task_pass": false + }, + { + "id": "anchor-owned-links-html-independent-nav", + "category": "format-normalization", + "root_cause": "record_title_masking_erases_independent_navigation", + "failure_mode": "false-negative", + "case": { + "url": "https://example.org/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 1, + "hosts": [ + "example.org" + ], + "entry_path": "/items/[^/]+", + "name_validation": "nonempty", + "stable_text_anchors": [ + { + "id": "nav", + "alternatives": [ + "Library" + ], + "match_scope": "non_record_lines" } - }, - "artifact": "ProSieben Mediathek bei Joyn: Alle Shows, Serien und Filme online streamen\n[New Show](/serien/new-show)\n[New Movie](/filme/new-movie)", - "ok": true, - "expected_task_pass": false - }, - { - "id": "prosieben-owned-content-genres", - "category": "structured-extraction", - "root_cause": "content_cards_not_placeholder_or_editorial_links", - "failure_mode": "false-positive", - "case": { - "url": "https://www.joyn.de/mediatheken/prosieben", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 2, - "hosts": [ - "www.joyn.de" - ], - "entry_path": "/(?:serien|filme|shows)/(?!genre(?:/|$))[^/]+", - "stable_text_anchors": [], - "name_validation": "nonempty", - "entry_scope": { - "html_selector": "main div[class*=\"CardStandard_StandardCardWrapper\"]", - "title_selector": "h3", - "link_selector": "a[data-testid=\"LL--movie\"],a[data-testid=\"LL--series\"]", - "start_pattern": "Meistgesehene Sendungen", - "end_pattern": "ProSieben Mediathek bei Joyn: Alle Shows, Serien und Filme online streamen" - } + ] + } + }, + "artifact": "
Library
", + "ok": true, + "expected_task_pass": true + }, + { + "id": "anchor-owned-links-html-record-only", + "category": "format-normalization", + "root_cause": "record_title_masking_erases_independent_navigation", + "failure_mode": "false-positive", + "case": { + "url": "https://example.org/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 1, + "hosts": [ + "example.org" + ], + "entry_path": "/items/[^/]+", + "name_validation": "nonempty", + "stable_text_anchors": [ + { + "id": "nav", + "alternatives": [ + "Library" + ], + "match_scope": "non_record_lines" } - }, - "artifact": "Meistgesehene Sendungen\n[Drama](/filme/genre/drama)\n[Comedy](/serien/genre/comedy)", - "ok": true, - "expected_task_pass": false - }, - { - "id": "prosieben-owned-content-placeholder-html", - "category": "structured-extraction", - "root_cause": "content_cards_not_placeholder_or_editorial_links", - "failure_mode": "false-positive", - "case": { - "url": "https://www.joyn.de/mediatheken/prosieben", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 2, - "hosts": [ - "www.joyn.de" - ], - "entry_path": "/(?:serien|filme|shows)/(?!genre(?:/|$))[^/]+", - "stable_text_anchors": [], - "name_validation": "nonempty", - "entry_scope": { - "html_selector": "main div[class*=\"CardStandard_StandardCardWrapper\"]", - "title_selector": "h3", - "link_selector": "a[data-testid=\"LL--movie\"],a[data-testid=\"LL--series\"]", - "start_pattern": "Meistgesehene Sendungen", - "end_pattern": "ProSieben Mediathek bei Joyn: Alle Shows, Serien und Filme online streamen" - } + ] + } + }, + "artifact": "
Library
", + "ok": true, + "expected_task_pass": false + }, + { + "id": "anchor-owned-links-html-hidden-nav", + "category": "format-normalization", + "root_cause": "record_title_masking_erases_independent_navigation", + "failure_mode": "false-positive", + "case": { + "url": "https://example.org/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 1, + "hosts": [ + "example.org" + ], + "entry_path": "/items/[^/]+", + "name_validation": "nonempty", + "stable_text_anchors": [ + { + "id": "nav", + "alternatives": [ + "Library" + ], + "match_scope": "non_record_lines" } - }, - "artifact": "
New ShowNew Movie
", - "ok": true, - "expected_task_pass": false - }, - { - "id": "francetv-content-cards-real-content-paths", - "category": "structured-extraction", - "root_cause": "content_card_routes_not_partial_channel_allowlist", - "failure_mode": "false-negative", - "case": { - "url": "https://www.france.tv/", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 2, - "hosts": [ - "www.france.tv" - ], - "entry_path": "/(?:arte|collection|documentaire|documentaires|films|france\\-2|france\\-24|france\\-3|france\\-4|france\\-5|franceinfo|ina|info|la1ere|lcp|lcp\\-public\\-senat|mieux|public\\-senat|serie|series\\-et\\-fictions|slash|societe|spectacles\\-et\\-culture|sport|tv5\\-monde)/.+", - "stable_text_anchors": [], - "name_validation": "nonempty", - "entry_scope": { - "html_selector": "main li:has(> a[data-card-link=\"true\"])", - "title_selector": ":scope > a[data-card-link] > div:first-child > span:first-child, :scope > a[data-card-link] > div:first-child > div:first-child > span", - "link_selector": ":scope > a[data-card-link=\"true\"]" - } + ] + } + }, + "artifact": "
Library
", + "ok": true, + "expected_task_pass": false + }, + { + "id": "anchor-owned-links-md-same-line-nav", + "category": "format-normalization", + "root_cause": "record_title_masking_erases_independent_navigation", + "failure_mode": "false-negative", + "case": { + "url": "https://example.org/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 1, + "hosts": [ + "example.org" + ], + "entry_path": "/items/[^/]+", + "name_validation": "nonempty", + "stable_text_anchors": [ + { + "id": "nav", + "alternatives": [ + "Library" + ], + "match_scope": "non_record_lines" } - }, - "artifact": "[Programme Nouveau](/arte/programme-nouveau/)\n[Collection Nouvelle](/collection/collection-nouvelle/)", - "ok": true, - "expected_task_pass": true - }, - { - "id": "francetv-content-cards-root-navigation", - "category": "structured-extraction", - "root_cause": "content_card_routes_not_partial_channel_allowlist", - "failure_mode": "false-positive", - "case": { - "url": "https://www.france.tv/", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 2, - "hosts": [ - "www.france.tv" - ], - "entry_path": "/(?:arte|collection|documentaire|documentaires|films|france\\-2|france\\-24|france\\-3|france\\-4|france\\-5|franceinfo|ina|info|la1ere|lcp|lcp\\-public\\-senat|mieux|public\\-senat|serie|series\\-et\\-fictions|slash|societe|spectacles\\-et\\-culture|sport|tv5\\-monde)/.+", - "stable_text_anchors": [], - "name_validation": "nonempty", - "entry_scope": { - "html_selector": "main li:has(> a[data-card-link=\"true\"])", - "title_selector": ":scope > a[data-card-link] > div:first-child > span:first-child, :scope > a[data-card-link] > div:first-child > div:first-child > span", - "link_selector": ":scope > a[data-card-link=\"true\"]" - } + ] + } + }, + "artifact": "[Library](/library/) [Library](/items/new)", + "ok": true, + "expected_task_pass": true + }, + { + "id": "anchor-owned-links-non-http-navigation", + "category": "format-normalization", + "root_cause": "record_title_masking_erases_independent_navigation", + "failure_mode": "false-negative", + "case": { + "url": "https://example.org/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 1, + "hosts": [ + "example.org" + ], + "entry_path": "/items/[^/]+", + "name_validation": "nonempty", + "stable_text_anchors": [ + { + "id": "nav", + "alternatives": [ + "Library" + ], + "match_scope": "non_record_lines" } - }, - "artifact": "[Films](/films/)\n[Arte](/arte/)", - "ok": true, - "expected_task_pass": false - }, - { - "id": "francetv-content-cards-empty-name", - "category": "structured-extraction", - "root_cause": "content_card_routes_not_partial_channel_allowlist", - "failure_mode": "false-positive", - "case": { - "url": "https://www.france.tv/", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 2, - "hosts": [ - "www.france.tv" - ], - "entry_path": "/(?:arte|collection|documentaire|documentaires|films|france\\-2|france\\-24|france\\-3|france\\-4|france\\-5|franceinfo|ina|info|la1ere|lcp|lcp\\-public\\-senat|mieux|public\\-senat|serie|series\\-et\\-fictions|slash|societe|spectacles\\-et\\-culture|sport|tv5\\-monde)/.+", - "stable_text_anchors": [], - "name_validation": "nonempty", - "entry_scope": { - "html_selector": "main li:has(> a[data-card-link=\"true\"])", - "title_selector": ":scope > a[data-card-link] > div:first-child > span:first-child, :scope > a[data-card-link] > div:first-child > div:first-child > span", - "link_selector": ":scope > a[data-card-link=\"true\"]" - } + ] + } + }, + "artifact": "[Library](mailto:help@example.org)\n[Library](/items/new)", + "ok": true, + "expected_task_pass": true + }, + { + "id": "nullable-table-faithful-empty", + "category": "structured-extraction", + "root_cause": "declared_empty_table_cells_are_real_values", + "failure_mode": "false-negative", + "case": { + "url": "https://mui.com/x/react-data-grid/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 5, + "stable_text_anchors": [], + "record_identity_fields": [ + "id" + ], + "table_columns": [ + { + "name": "id", + "label": "ID", + "type": "integer", + "minimum": 1 + }, + { + "name": "first", + "label": "First name", + "type": "text" + }, + { + "name": "last", + "label": "Last name", + "type": "text" + }, + { + "name": "age", + "label": "Age", + "type": "integer", + "minimum": 0, + "allow_empty": true } - }, - "artifact": "[](/arte/programme-nouveau/)\n[Collection Nouvelle](/collection/collection-nouvelle/)", - "ok": true, - "expected_task_pass": false - }, - { - "id": "francetv-content-cards-foreign-host", - "category": "structured-extraction", - "root_cause": "content_card_routes_not_partial_channel_allowlist", - "failure_mode": "false-positive", - "case": { - "url": "https://www.france.tv/", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 2, - "hosts": [ - "www.france.tv" - ], - "entry_path": "/(?:arte|collection|documentaire|documentaires|films|france\\-2|france\\-24|france\\-3|france\\-4|france\\-5|franceinfo|ina|info|la1ere|lcp|lcp\\-public\\-senat|mieux|public\\-senat|serie|series\\-et\\-fictions|slash|societe|spectacles\\-et\\-culture|sport|tv5\\-monde)/.+", - "stable_text_anchors": [], - "name_validation": "nonempty", - "entry_scope": { - "html_selector": "main li:has(> a[data-card-link=\"true\"])", - "title_selector": ":scope > a[data-card-link] > div:first-child > span:first-child, :scope > a[data-card-link] > div:first-child > div:first-child > span", - "link_selector": ":scope > a[data-card-link=\"true\"]" - } + ], + "table_scope": { + "root_selector": "[role=grid]", + "root_index": 0, + "row_selector": "[role=row][data-id]", + "cell_selector": "[role=gridcell]:not([data-field=\"__check__\"])", + "start_pattern": "Community version \\(free forever\\)", + "end_pattern": "(?m)^[ \\t#]*Pro version[ \\t]*$", + "plain_row_pattern": "(?P\\d+)\\s+(?P[A-Za-z]+)\\s+(?P[A-Za-z]+)(?:[ \\t]+(?P\\d*))", + "plain_multiline_row_pattern": "(?m)^[ \\t]*(?P\\d+)\\n[ \\t\\n]*(?P[A-Za-z]+)\\n[ \\t\\n]*(?P[A-Za-z]+)\\n[ \\t\\n]*(?P\\d+)[ \\t]*$", + "reference_kind": "fixed_demo", + "reference_rows": [ + { + "id": "1", + "first": "Jon", + "last": "Snow", + "age": "14" + }, + { + "id": "2", + "first": "Cersei", + "last": "Lannister", + "age": "31" + }, + { + "id": "3", + "first": "Jaime", + "last": "Lannister", + "age": "31" + }, + { + "id": "4", + "first": "Arya", + "last": "Stark", + "age": "11" + }, + { + "id": "5", + "first": "Daenerys", + "last": "Targaryen", + "age": "" + } + ] + } + } + }, + "artifact": "Community version (free forever)\n|ID|First name|Last name|Age|\n|---|---|---|---|\n|1|Jon|Snow|14|\n|2|Cersei|Lannister|31|\n|3|Jaime|Lannister|31|\n|4|Arya|Stark|11|\n|5|Daenerys|Targaryen||\nPro version", + "ok": true, + "expected_task_pass": true + }, + { + "id": "nullable-table-wrong-empty", + "category": "structured-extraction", + "root_cause": "declared_empty_table_cells_are_real_values", + "failure_mode": "false-positive", + "case": { + "url": "https://mui.com/x/react-data-grid/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 5, + "stable_text_anchors": [], + "record_identity_fields": [ + "id" + ], + "table_columns": [ + { + "name": "id", + "label": "ID", + "type": "integer", + "minimum": 1 + }, + { + "name": "first", + "label": "First name", + "type": "text" + }, + { + "name": "last", + "label": "Last name", + "type": "text" + }, + { + "name": "age", + "label": "Age", + "type": "integer", + "minimum": 0, + "allow_empty": true } - }, - "artifact": "[Programme](https://example.org/arte/programme/)\n[Collection](https://example.org/collection/nouvelle/)", - "ok": true, - "expected_task_pass": false - }, - { - "id": "francetv-content-cards-same-id-twice", - "category": "structured-extraction", - "root_cause": "content_card_routes_not_partial_channel_allowlist", - "failure_mode": "false-positive", - "case": { - "url": "https://www.france.tv/", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 2, - "hosts": [ - "www.france.tv" - ], - "entry_path": "/(?:arte|collection|documentaire|documentaires|films|france\\-2|france\\-24|france\\-3|france\\-4|france\\-5|franceinfo|ina|info|la1ere|lcp|lcp\\-public\\-senat|mieux|public\\-senat|serie|series\\-et\\-fictions|slash|societe|spectacles\\-et\\-culture|sport|tv5\\-monde)/.+", - "stable_text_anchors": [], - "name_validation": "nonempty", - "entry_scope": { - "html_selector": "main li:has(> a[data-card-link=\"true\"])", - "title_selector": ":scope > a[data-card-link] > div:first-child > span:first-child, :scope > a[data-card-link] > div:first-child > div:first-child > span", - "link_selector": ":scope > a[data-card-link=\"true\"]" - } + ], + "table_scope": { + "root_selector": "[role=grid]", + "root_index": 0, + "row_selector": "[role=row][data-id]", + "cell_selector": "[role=gridcell]:not([data-field=\"__check__\"])", + "start_pattern": "Community version \\(free forever\\)", + "end_pattern": "(?m)^[ \\t#]*Pro version[ \\t]*$", + "plain_row_pattern": "(?P\\d+)\\s+(?P[A-Za-z]+)\\s+(?P[A-Za-z]+)(?:[ \\t]+(?P\\d*))", + "plain_multiline_row_pattern": "(?m)^[ \\t]*(?P\\d+)\\n[ \\t\\n]*(?P[A-Za-z]+)\\n[ \\t\\n]*(?P[A-Za-z]+)\\n[ \\t\\n]*(?P\\d+)[ \\t]*$", + "reference_kind": "fixed_demo", + "reference_rows": [ + { + "id": "1", + "first": "Jon", + "last": "Snow", + "age": "14" + }, + { + "id": "2", + "first": "Cersei", + "last": "Lannister", + "age": "31" + }, + { + "id": "3", + "first": "Jaime", + "last": "Lannister", + "age": "31" + }, + { + "id": "4", + "first": "Arya", + "last": "Stark", + "age": "11" + }, + { + "id": "5", + "first": "Daenerys", + "last": "Targaryen", + "age": "" + } + ] + } + } + }, + "artifact": "Community version (free forever)\n|ID|First name|Last name|Age|\n|---|---|---|---|\n|1|Jon|Snow||\n|2|Cersei|Lannister|31|\n|3|Jaime|Lannister|31|\n|4|Arya|Stark|11|\n|5|Daenerys|Targaryen||\nPro version", + "ok": true, + "expected_task_pass": false + }, + { + "id": "nullable-table-invented-age", + "category": "structured-extraction", + "root_cause": "declared_empty_table_cells_are_real_values", + "failure_mode": "false-positive", + "case": { + "url": "https://mui.com/x/react-data-grid/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 5, + "stable_text_anchors": [], + "record_identity_fields": [ + "id" + ], + "table_columns": [ + { + "name": "id", + "label": "ID", + "type": "integer", + "minimum": 1 + }, + { + "name": "first", + "label": "First name", + "type": "text" + }, + { + "name": "last", + "label": "Last name", + "type": "text" + }, + { + "name": "age", + "label": "Age", + "type": "integer", + "minimum": 0, + "allow_empty": true } - }, - "artifact": "[Programme](/arte/programme/)\n[Programme](/arte/programme/)", - "ok": true, - "expected_task_pass": false - }, - { - "id": "rtve-title-scope-new-programme-and-episode", - "category": "structured-extraction", - "root_cause": "programme_title_links_not_action_buttons", - "failure_mode": "false-negative", - "case": { - "url": "https://www.rtve.es/play/", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 2, - "hosts": [ - "www.rtve.es" - ], - "entry_path": "/play/videos/.+", - "stable_text_anchors": [], - "name_validation": "nonempty", - "entry_scope": { - "html_selector": "div.content strong > a:has(.maintitle),div.content strong > a:has(.pretitle)" - }, - "title_exclusion_patterns": [ - "(?:\\\\?\\+\\s*info|En directo|Directos)" - ] + ], + "table_scope": { + "root_selector": "[role=grid]", + "root_index": 0, + "row_selector": "[role=row][data-id]", + "cell_selector": "[role=gridcell]:not([data-field=\"__check__\"])", + "start_pattern": "Community version \\(free forever\\)", + "end_pattern": "(?m)^[ \\t#]*Pro version[ \\t]*$", + "plain_row_pattern": "(?P\\d+)\\s+(?P[A-Za-z]+)\\s+(?P[A-Za-z]+)(?:[ \\t]+(?P\\d*))", + "plain_multiline_row_pattern": "(?m)^[ \\t]*(?P\\d+)\\n[ \\t\\n]*(?P[A-Za-z]+)\\n[ \\t\\n]*(?P[A-Za-z]+)\\n[ \\t\\n]*(?P\\d+)[ \\t]*$", + "reference_kind": "fixed_demo", + "reference_rows": [ + { + "id": "1", + "first": "Jon", + "last": "Snow", + "age": "14" + }, + { + "id": "2", + "first": "Cersei", + "last": "Lannister", + "age": "31" + }, + { + "id": "3", + "first": "Jaime", + "last": "Lannister", + "age": "31" + }, + { + "id": "4", + "first": "Arya", + "last": "Stark", + "age": "11" + }, + { + "id": "5", + "first": "Daenerys", + "last": "Targaryen", + "age": "" + } + ] + } + } + }, + "artifact": "Community version (free forever)\n|ID|First name|Last name|Age|\n|---|---|---|---|\n|1|Jon|Snow|14|\n|2|Cersei|Lannister|31|\n|3|Jaime|Lannister|31|\n|4|Arya|Stark|11|\n|5|Daenerys|Targaryen|25|\nPro version", + "ok": true, + "expected_task_pass": false + }, + { + "id": "nullable-table-missing-cell", + "category": "structured-extraction", + "root_cause": "declared_empty_table_cells_are_real_values", + "failure_mode": "false-positive", + "case": { + "url": "https://mui.com/x/react-data-grid/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 5, + "stable_text_anchors": [], + "record_identity_fields": [ + "id" + ], + "table_columns": [ + { + "name": "id", + "label": "ID", + "type": "integer", + "minimum": 1 + }, + { + "name": "first", + "label": "First name", + "type": "text" + }, + { + "name": "last", + "label": "Last name", + "type": "text" + }, + { + "name": "age", + "label": "Age", + "type": "integer", + "minimum": 0, + "allow_empty": true } - }, - "artifact": "[Programa Nuevo](/play/videos/programa-nuevo/)\n[Episodio Nuevo](/play/videos/programa-nuevo/episodio/123/)", - "ok": true, - "expected_task_pass": true - }, - { - "id": "rtve-title-scope-buttons-only", - "category": "structured-extraction", - "root_cause": "programme_title_links_not_action_buttons", - "failure_mode": "false-positive", - "case": { - "url": "https://www.rtve.es/play/", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 2, - "hosts": [ - "www.rtve.es" - ], - "entry_path": "/play/videos/.+", - "stable_text_anchors": [], - "name_validation": "nonempty", - "entry_scope": { - "html_selector": "div.content strong > a:has(.maintitle),div.content strong > a:has(.pretitle)" - }, - "title_exclusion_patterns": [ - "(?:\\\\?\\+\\s*info|En directo|Directos)" - ] + ], + "table_scope": { + "root_selector": "[role=grid]", + "root_index": 0, + "row_selector": "[role=row][data-id]", + "cell_selector": "[role=gridcell]:not([data-field=\"__check__\"])", + "start_pattern": "Community version \\(free forever\\)", + "end_pattern": "(?m)^[ \\t#]*Pro version[ \\t]*$", + "plain_row_pattern": "(?P\\d+)\\s+(?P[A-Za-z]+)\\s+(?P[A-Za-z]+)(?:[ \\t]+(?P\\d*))", + "plain_multiline_row_pattern": "(?m)^[ \\t]*(?P\\d+)\\n[ \\t\\n]*(?P[A-Za-z]+)\\n[ \\t\\n]*(?P[A-Za-z]+)\\n[ \\t\\n]*(?P\\d+)[ \\t]*$", + "reference_kind": "fixed_demo", + "reference_rows": [ + { + "id": "1", + "first": "Jon", + "last": "Snow", + "age": "14" + }, + { + "id": "2", + "first": "Cersei", + "last": "Lannister", + "age": "31" + }, + { + "id": "3", + "first": "Jaime", + "last": "Lannister", + "age": "31" + }, + { + "id": "4", + "first": "Arya", + "last": "Stark", + "age": "11" + }, + { + "id": "5", + "first": "Daenerys", + "last": "Targaryen", + "age": "" + } + ] + } + } + }, + "artifact": "Community version (free forever)\n|ID|First name|Last name|Age|\n|---|---|---|---|\n|1|Jon|Snow|14|\n|2|Cersei|Lannister|31|\n|3|Jaime|Lannister|31|\n|4|Arya|Stark|11|\n|5|Daenerys|Targaryen|\nPro version", + "ok": true, + "expected_task_pass": false + }, + { + "id": "nullable-table-mixed-row", + "category": "structured-extraction", + "root_cause": "declared_empty_table_cells_are_real_values", + "failure_mode": "false-positive", + "case": { + "url": "https://mui.com/x/react-data-grid/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 5, + "stable_text_anchors": [], + "record_identity_fields": [ + "id" + ], + "table_columns": [ + { + "name": "id", + "label": "ID", + "type": "integer", + "minimum": 1 + }, + { + "name": "first", + "label": "First name", + "type": "text" + }, + { + "name": "last", + "label": "Last name", + "type": "text" + }, + { + "name": "age", + "label": "Age", + "type": "integer", + "minimum": 0, + "allow_empty": true } - }, - "artifact": "[+ info](/play/videos/a/)\n[\\+ info](/play/videos/b/)\n[Directos](/play/videos/directo/)", - "ok": true, - "expected_task_pass": false - }, - { - "id": "rtve-title-scope-duplicate-id", - "category": "structured-extraction", - "root_cause": "programme_title_links_not_action_buttons", - "failure_mode": "false-positive", - "case": { - "url": "https://www.rtve.es/play/", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 2, - "hosts": [ - "www.rtve.es" - ], - "entry_path": "/play/videos/.+", - "stable_text_anchors": [], - "name_validation": "nonempty", - "entry_scope": { - "html_selector": "div.content strong > a:has(.maintitle),div.content strong > a:has(.pretitle)" - }, - "title_exclusion_patterns": [ - "(?:\\\\?\\+\\s*info|En directo|Directos)" - ] + ], + "table_scope": { + "root_selector": "[role=grid]", + "root_index": 0, + "row_selector": "[role=row][data-id]", + "cell_selector": "[role=gridcell]:not([data-field=\"__check__\"])", + "start_pattern": "Community version \\(free forever\\)", + "end_pattern": "(?m)^[ \\t#]*Pro version[ \\t]*$", + "plain_row_pattern": "(?P\\d+)\\s+(?P[A-Za-z]+)\\s+(?P[A-Za-z]+)(?:[ \\t]+(?P\\d*))", + "plain_multiline_row_pattern": "(?m)^[ \\t]*(?P\\d+)\\n[ \\t\\n]*(?P[A-Za-z]+)\\n[ \\t\\n]*(?P[A-Za-z]+)\\n[ \\t\\n]*(?P\\d+)[ \\t]*$", + "reference_kind": "fixed_demo", + "reference_rows": [ + { + "id": "1", + "first": "Jon", + "last": "Snow", + "age": "14" + }, + { + "id": "2", + "first": "Cersei", + "last": "Lannister", + "age": "31" + }, + { + "id": "3", + "first": "Jaime", + "last": "Lannister", + "age": "31" + }, + { + "id": "4", + "first": "Arya", + "last": "Stark", + "age": "11" + }, + { + "id": "5", + "first": "Daenerys", + "last": "Targaryen", + "age": "" + } + ] + } + } + }, + "artifact": "Community version (free forever)\n|ID|First name|Last name|Age|\n|---|---|---|---|\n|1|Jon|Lannister|14|\n|2|Cersei|Lannister|31|\n|3|Jaime|Lannister|31|\n|4|Arya|Stark|11|\n|5|Daenerys|Targaryen||\nPro version", + "ok": true, + "expected_task_pass": false + }, + { + "id": "documentation-prose-and-grid", + "category": "structured-extraction", + "root_cause": "documentation_prose_and_grid_are_separate_requirements", + "failure_mode": "false-negative", + "case": { + "url": "https://mui.com/x/react-data-grid/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 5, + "stable_text_anchors": [], + "record_identity_fields": [ + "id" + ], + "table_columns": [ + { + "name": "id", + "label": "ID", + "type": "integer", + "minimum": 1 + }, + { + "name": "first", + "label": "First name", + "type": "text" + }, + { + "name": "last", + "label": "Last name", + "type": "text" + }, + { + "name": "age", + "label": "Age", + "type": "integer", + "minimum": 0, + "allow_empty": true } + ], + "table_scope": { + "root_selector": "[role=grid]", + "root_index": 0, + "row_selector": "[role=row][data-id]", + "cell_selector": "[role=gridcell]:not([data-field=\"__check__\"])", + "start_pattern": "Community version \\(free forever\\)", + "end_pattern": "(?m)^[ \\t#]*Pro version[ \\t]*$", + "plain_row_pattern": "(?P\\d+)\\s+(?P[A-Za-z]+)\\s+(?P[A-Za-z]+)(?:[ \\t]+(?P\\d*))", + "plain_multiline_row_pattern": "(?m)^[ \\t]*(?P\\d+)\\n[ \\t\\n]*(?P[A-Za-z]+)\\n[ \\t\\n]*(?P[A-Za-z]+)\\n[ \\t\\n]*(?P\\d+)[ \\t]*$", + "reference_kind": "fixed_demo", + "reference_rows": [ + { + "id": "1", + "first": "Jon", + "last": "Snow", + "age": "14" + }, + { + "id": "2", + "first": "Cersei", + "last": "Lannister", + "age": "31" + }, + { + "id": "3", + "first": "Jaime", + "last": "Lannister", + "age": "31" + }, + { + "id": "4", + "first": "Arya", + "last": "Stark", + "age": "11" + }, + { + "id": "5", + "first": "Daenerys", + "last": "Targaryen", + "age": "" + } + ] + } }, - "artifact": "[Nuevo](/play/videos/nuevo/)\n[Nuevo otro texto](/play/videos/nuevo/)", - "ok": true, - "expected_task_pass": false - }, - { - "id": "rtve-title-scope-empty-title", - "category": "structured-extraction", - "root_cause": "programme_title_links_not_action_buttons", - "failure_mode": "false-positive", - "case": { - "url": "https://www.rtve.es/play/", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 2, - "hosts": [ - "www.rtve.es" - ], - "entry_path": "/play/videos/.+", - "stable_text_anchors": [], - "name_validation": "nonempty", - "entry_scope": { - "html_selector": "div.content strong > a:has(.maintitle),div.content strong > a:has(.pretitle)" - }, - "title_exclusion_patterns": [ - "(?:\\\\?\\+\\s*info|En directo|Directos)" - ] + "body_requirements": { + "version": 3, + "unit": "region_compact_hexagram_recall", + "minimum_coverage": 0.8, + "minimum_floor": 0.6, + "minimum_region_passes": 1, + "segmentation_method": "sentences", + "regions": [ + { + "id": "body-region-1", + "text": "the grid displays structured rows and columns each row preserves the fields of one record", + "source_text": "The grid displays structured rows and columns.\n\nEach row preserves the fields of one record.", + "segmentation_method": "sentences" } + ] }, - "artifact": "[](/play/videos/a/)\n[Real](/play/videos/b/)", - "ok": true, - "expected_task_pass": false - }, - { - "id": "rtve-title-scope-audio-not-video", - "category": "structured-extraction", - "root_cause": "programme_title_links_not_action_buttons", - "failure_mode": "false-positive", - "case": { - "url": "https://www.rtve.es/play/", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 2, - "hosts": [ - "www.rtve.es" - ], - "entry_path": "/play/videos/.+", - "stable_text_anchors": [], - "name_validation": "nonempty", - "entry_scope": { - "html_selector": "div.content strong > a:has(.maintitle),div.content strong > a:has(.pretitle)" - }, - "title_exclusion_patterns": [ - "(?:\\\\?\\+\\s*info|En directo|Directos)" - ] + "forbidden_any": [ + "Access denied", + "CAPTCHA" + ] + }, + "artifact": "The grid displays structured rows and columns. Each row preserves the fields of one record.\nCommunity version (free forever)\n|ID|First name|Last name|Age|\n|---|---|---|---|\n|1|Jon|Snow|14|\n|2|Cersei|Lannister|31|\n|3|Jaime|Lannister|31|\n|4|Arya|Stark|11|\n|5|Daenerys|Targaryen||\nPro version", + "ok": true, + "expected_task_pass": true + }, + { + "id": "documentation-prose-without-grid", + "category": "structured-extraction", + "root_cause": "documentation_prose_and_grid_are_separate_requirements", + "failure_mode": "false-positive", + "case": { + "url": "https://mui.com/x/react-data-grid/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 5, + "stable_text_anchors": [], + "record_identity_fields": [ + "id" + ], + "table_columns": [ + { + "name": "id", + "label": "ID", + "type": "integer", + "minimum": 1 + }, + { + "name": "first", + "label": "First name", + "type": "text" + }, + { + "name": "last", + "label": "Last name", + "type": "text" + }, + { + "name": "age", + "label": "Age", + "type": "integer", + "minimum": 0, + "allow_empty": true } + ], + "table_scope": { + "root_selector": "[role=grid]", + "root_index": 0, + "row_selector": "[role=row][data-id]", + "cell_selector": "[role=gridcell]:not([data-field=\"__check__\"])", + "start_pattern": "Community version \\(free forever\\)", + "end_pattern": "(?m)^[ \\t#]*Pro version[ \\t]*$", + "plain_row_pattern": "(?P\\d+)\\s+(?P[A-Za-z]+)\\s+(?P[A-Za-z]+)(?:[ \\t]+(?P\\d*))", + "plain_multiline_row_pattern": "(?m)^[ \\t]*(?P\\d+)\\n[ \\t\\n]*(?P[A-Za-z]+)\\n[ \\t\\n]*(?P[A-Za-z]+)\\n[ \\t\\n]*(?P\\d+)[ \\t]*$", + "reference_kind": "fixed_demo", + "reference_rows": [ + { + "id": "1", + "first": "Jon", + "last": "Snow", + "age": "14" + }, + { + "id": "2", + "first": "Cersei", + "last": "Lannister", + "age": "31" + }, + { + "id": "3", + "first": "Jaime", + "last": "Lannister", + "age": "31" + }, + { + "id": "4", + "first": "Arya", + "last": "Stark", + "age": "11" + }, + { + "id": "5", + "first": "Daenerys", + "last": "Targaryen", + "age": "" + } + ] + } }, - "artifact": "[Audio](/play/audios/programa/)\n[Real](/play/videos/b/)", - "ok": true, - "expected_task_pass": false - }, - { - "id": "rtve-title-scope-html-owned-programme-and-episode", - "category": "structured-extraction", - "root_cause": "programme_title_links_not_action_buttons", - "failure_mode": "false-negative", - "case": { - "url": "https://www.rtve.es/play/", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 2, - "hosts": [ - "www.rtve.es" - ], - "entry_path": "/play/videos/.+", - "stable_text_anchors": [], - "name_validation": "nonempty", - "entry_scope": { - "html_selector": "div.content strong > a:has(.maintitle),div.content strong > a:has(.pretitle)" - }, - "title_exclusion_patterns": [ - "(?:\\\\?\\+\\s*info|En directo|Directos)" - ] + "body_requirements": { + "version": 3, + "unit": "region_compact_hexagram_recall", + "minimum_coverage": 0.8, + "minimum_floor": 0.6, + "minimum_region_passes": 1, + "segmentation_method": "sentences", + "regions": [ + { + "id": "body-region-1", + "text": "the grid displays structured rows and columns each row preserves the fields of one record", + "source_text": "The grid displays structured rows and columns.\n\nEach row preserves the fields of one record.", + "segmentation_method": "sentences" } + ] }, - "artifact": "
NuevoEpisodio+ info
", - "ok": true, - "expected_task_pass": true - }, - { - "id": "anchor-owned-links-md-independent-nav", - "category": "format-normalization", - "root_cause": "record_title_masking_erases_independent_navigation", - "failure_mode": "false-negative", - "case": { - "url": "https://example.org/", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 1, - "hosts": [ - "example.org" - ], - "entry_path": "/items/[^/]+", - "name_validation": "nonempty", - "stable_text_anchors": [ - { - "id": "nav", - "alternatives": [ - "Library" - ], - "match_scope": "non_record_lines" - } - ] + "forbidden_any": [ + "Access denied", + "CAPTCHA" + ] + }, + "artifact": "The grid displays structured rows and columns. Each row preserves the fields of one record.", + "ok": true, + "expected_task_pass": false + }, + { + "id": "documentation-grid-without-prose", + "category": "structured-extraction", + "root_cause": "documentation_prose_and_grid_are_separate_requirements", + "failure_mode": "false-positive", + "case": { + "url": "https://mui.com/x/react-data-grid/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 5, + "stable_text_anchors": [], + "record_identity_fields": [ + "id" + ], + "table_columns": [ + { + "name": "id", + "label": "ID", + "type": "integer", + "minimum": 1 + }, + { + "name": "first", + "label": "First name", + "type": "text" + }, + { + "name": "last", + "label": "Last name", + "type": "text" + }, + { + "name": "age", + "label": "Age", + "type": "integer", + "minimum": 0, + "allow_empty": true } + ], + "table_scope": { + "root_selector": "[role=grid]", + "root_index": 0, + "row_selector": "[role=row][data-id]", + "cell_selector": "[role=gridcell]:not([data-field=\"__check__\"])", + "start_pattern": "Community version \\(free forever\\)", + "end_pattern": "(?m)^[ \\t#]*Pro version[ \\t]*$", + "plain_row_pattern": "(?P\\d+)\\s+(?P[A-Za-z]+)\\s+(?P[A-Za-z]+)(?:[ \\t]+(?P\\d*))", + "plain_multiline_row_pattern": "(?m)^[ \\t]*(?P\\d+)\\n[ \\t\\n]*(?P[A-Za-z]+)\\n[ \\t\\n]*(?P[A-Za-z]+)\\n[ \\t\\n]*(?P\\d+)[ \\t]*$", + "reference_kind": "fixed_demo", + "reference_rows": [ + { + "id": "1", + "first": "Jon", + "last": "Snow", + "age": "14" + }, + { + "id": "2", + "first": "Cersei", + "last": "Lannister", + "age": "31" + }, + { + "id": "3", + "first": "Jaime", + "last": "Lannister", + "age": "31" + }, + { + "id": "4", + "first": "Arya", + "last": "Stark", + "age": "11" + }, + { + "id": "5", + "first": "Daenerys", + "last": "Targaryen", + "age": "" + } + ] + } }, - "artifact": "[Library](/library/)\n[Library](/items/new)", - "ok": true, - "expected_task_pass": true - }, - { - "id": "anchor-owned-links-md-record-only", - "category": "format-normalization", - "root_cause": "record_title_masking_erases_independent_navigation", - "failure_mode": "false-positive", - "case": { - "url": "https://example.org/", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 1, - "hosts": [ - "example.org" - ], - "entry_path": "/items/[^/]+", - "name_validation": "nonempty", - "stable_text_anchors": [ - { - "id": "nav", - "alternatives": [ - "Library" - ], - "match_scope": "non_record_lines" - } - ] + "body_requirements": { + "version": 3, + "unit": "region_compact_hexagram_recall", + "minimum_coverage": 0.8, + "minimum_floor": 0.6, + "minimum_region_passes": 1, + "segmentation_method": "sentences", + "regions": [ + { + "id": "body-region-1", + "text": "the grid displays structured rows and columns each row preserves the fields of one record", + "source_text": "The grid displays structured rows and columns.\n\nEach row preserves the fields of one record.", + "segmentation_method": "sentences" } + ] }, - "artifact": "[Library](/items/new)", - "ok": true, - "expected_task_pass": false - }, - { - "id": "anchor-owned-links-md-repeated-record", - "category": "format-normalization", - "root_cause": "record_title_masking_erases_independent_navigation", - "failure_mode": "false-positive", - "case": { - "url": "https://example.org/", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 1, - "hosts": [ - "example.org" - ], - "entry_path": "/items/[^/]+", - "name_validation": "nonempty", - "stable_text_anchors": [ - { - "id": "nav", - "alternatives": [ - "Library" - ], - "match_scope": "non_record_lines" - } - ] + "forbidden_any": [ + "Access denied", + "CAPTCHA" + ] + }, + "artifact": "Community version (free forever)\n|ID|First name|Last name|Age|\n|---|---|---|---|\n|1|Jon|Snow|14|\n|2|Cersei|Lannister|31|\n|3|Jaime|Lannister|31|\n|4|Arya|Stark|11|\n|5|Daenerys|Targaryen||\nPro version", + "ok": true, + "expected_task_pass": false + }, + { + "id": "consent-alongside-real-documentation", + "category": "structured-extraction", + "root_cause": "cookie_preferences_not_blocked_documentation", + "failure_mode": "false-negative", + "case": { + "url": "https://mui.com/x/react-data-grid/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 5, + "stable_text_anchors": [], + "record_identity_fields": [ + "id" + ], + "table_columns": [ + { + "name": "id", + "label": "ID", + "type": "integer", + "minimum": 1 + }, + { + "name": "first", + "label": "First name", + "type": "text" + }, + { + "name": "last", + "label": "Last name", + "type": "text" + }, + { + "name": "age", + "label": "Age", + "type": "integer", + "minimum": 0, + "allow_empty": true } + ], + "table_scope": { + "root_selector": "[role=grid]", + "root_index": 0, + "row_selector": "[role=row][data-id]", + "cell_selector": "[role=gridcell]:not([data-field=\"__check__\"])", + "start_pattern": "Community version \\(free forever\\)", + "end_pattern": "(?m)^[ \\t#]*Pro version[ \\t]*$", + "plain_row_pattern": "(?P\\d+)\\s+(?P[A-Za-z]+)\\s+(?P[A-Za-z]+)(?:[ \\t]+(?P\\d*))", + "plain_multiline_row_pattern": "(?m)^[ \\t]*(?P\\d+)\\n[ \\t\\n]*(?P[A-Za-z]+)\\n[ \\t\\n]*(?P[A-Za-z]+)\\n[ \\t\\n]*(?P\\d+)[ \\t]*$", + "reference_kind": "fixed_demo", + "reference_rows": [ + { + "id": "1", + "first": "Jon", + "last": "Snow", + "age": "14" + }, + { + "id": "2", + "first": "Cersei", + "last": "Lannister", + "age": "31" + }, + { + "id": "3", + "first": "Jaime", + "last": "Lannister", + "age": "31" + }, + { + "id": "4", + "first": "Arya", + "last": "Stark", + "age": "11" + }, + { + "id": "5", + "first": "Daenerys", + "last": "Targaryen", + "age": "" + } + ] + } }, - "artifact": "[Library](/items/new)\n[Library](/items/new)", - "ok": true, - "expected_task_pass": false - }, - { - "id": "anchor-owned-links-html-independent-nav", - "category": "format-normalization", - "root_cause": "record_title_masking_erases_independent_navigation", - "failure_mode": "false-negative", - "case": { - "url": "https://example.org/", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 1, - "hosts": [ - "example.org" - ], - "entry_path": "/items/[^/]+", - "name_validation": "nonempty", - "stable_text_anchors": [ - { - "id": "nav", - "alternatives": [ - "Library" - ], - "match_scope": "non_record_lines" - } - ] + "body_requirements": { + "version": 3, + "unit": "region_compact_hexagram_recall", + "minimum_coverage": 0.8, + "minimum_floor": 0.6, + "minimum_region_passes": 1, + "segmentation_method": "sentences", + "regions": [ + { + "id": "body-region-1", + "text": "the grid displays structured rows and columns each row preserves the fields of one record", + "source_text": "The grid displays structured rows and columns.\n\nEach row preserves the fields of one record.", + "segmentation_method": "sentences" } + ] }, - "artifact": "
Library
", - "ok": true, - "expected_task_pass": true - }, - { - "id": "anchor-owned-links-html-record-only", - "category": "format-normalization", - "root_cause": "record_title_masking_erases_independent_navigation", - "failure_mode": "false-positive", - "case": { - "url": "https://example.org/", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 1, - "hosts": [ - "example.org" - ], - "entry_path": "/items/[^/]+", - "name_validation": "nonempty", - "stable_text_anchors": [ - { - "id": "nav", - "alternatives": [ - "Library" - ], - "match_scope": "non_record_lines" - } - ] + "forbidden_any": [ + "Access denied", + "CAPTCHA" + ] + }, + "artifact": "Cookie preferences\nThe grid displays structured rows and columns. Each row preserves the fields of one record.\nCommunity version (free forever)\n|ID|First name|Last name|Age|\n|---|---|---|---|\n|1|Jon|Snow|14|\n|2|Cersei|Lannister|31|\n|3|Jaime|Lannister|31|\n|4|Arya|Stark|11|\n|5|Daenerys|Targaryen||\nPro version", + "ok": true, + "expected_task_pass": true + }, + { + "id": "consent-only-is-not-documentation", + "category": "structured-extraction", + "root_cause": "cookie_preferences_not_blocked_documentation", + "failure_mode": "false-positive", + "case": { + "url": "https://mui.com/x/react-data-grid/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 5, + "stable_text_anchors": [], + "record_identity_fields": [ + "id" + ], + "table_columns": [ + { + "name": "id", + "label": "ID", + "type": "integer", + "minimum": 1 + }, + { + "name": "first", + "label": "First name", + "type": "text" + }, + { + "name": "last", + "label": "Last name", + "type": "text" + }, + { + "name": "age", + "label": "Age", + "type": "integer", + "minimum": 0, + "allow_empty": true } + ], + "table_scope": { + "root_selector": "[role=grid]", + "root_index": 0, + "row_selector": "[role=row][data-id]", + "cell_selector": "[role=gridcell]:not([data-field=\"__check__\"])", + "start_pattern": "Community version \\(free forever\\)", + "end_pattern": "(?m)^[ \\t#]*Pro version[ \\t]*$", + "plain_row_pattern": "(?P\\d+)\\s+(?P[A-Za-z]+)\\s+(?P[A-Za-z]+)(?:[ \\t]+(?P\\d*))", + "plain_multiline_row_pattern": "(?m)^[ \\t]*(?P\\d+)\\n[ \\t\\n]*(?P[A-Za-z]+)\\n[ \\t\\n]*(?P[A-Za-z]+)\\n[ \\t\\n]*(?P\\d+)[ \\t]*$", + "reference_kind": "fixed_demo", + "reference_rows": [ + { + "id": "1", + "first": "Jon", + "last": "Snow", + "age": "14" + }, + { + "id": "2", + "first": "Cersei", + "last": "Lannister", + "age": "31" + }, + { + "id": "3", + "first": "Jaime", + "last": "Lannister", + "age": "31" + }, + { + "id": "4", + "first": "Arya", + "last": "Stark", + "age": "11" + }, + { + "id": "5", + "first": "Daenerys", + "last": "Targaryen", + "age": "" + } + ] + } }, - "artifact": "
Library
", - "ok": true, - "expected_task_pass": false - }, - { - "id": "anchor-owned-links-html-hidden-nav", - "category": "format-normalization", - "root_cause": "record_title_masking_erases_independent_navigation", - "failure_mode": "false-positive", - "case": { - "url": "https://example.org/", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 1, - "hosts": [ - "example.org" - ], - "entry_path": "/items/[^/]+", - "name_validation": "nonempty", - "stable_text_anchors": [ - { - "id": "nav", - "alternatives": [ - "Library" - ], - "match_scope": "non_record_lines" - } - ] + "body_requirements": { + "version": 3, + "unit": "region_compact_hexagram_recall", + "minimum_coverage": 0.8, + "minimum_floor": 0.6, + "minimum_region_passes": 1, + "segmentation_method": "sentences", + "regions": [ + { + "id": "body-region-1", + "text": "the grid displays structured rows and columns each row preserves the fields of one record", + "source_text": "The grid displays structured rows and columns.\n\nEach row preserves the fields of one record.", + "segmentation_method": "sentences" } + ] }, - "artifact": "
Library
", - "ok": true, - "expected_task_pass": false - }, - { - "id": "anchor-owned-links-md-same-line-nav", - "category": "format-normalization", - "root_cause": "record_title_masking_erases_independent_navigation", - "failure_mode": "false-negative", - "case": { - "url": "https://example.org/", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 1, - "hosts": [ - "example.org" - ], - "entry_path": "/items/[^/]+", - "name_validation": "nonempty", - "stable_text_anchors": [ - { - "id": "nav", - "alternatives": [ - "Library" - ], - "match_scope": "non_record_lines" - } - ] + "forbidden_any": [ + "Access denied", + "CAPTCHA" + ] + }, + "artifact": "Cookie preferences", + "ok": true, + "expected_task_pass": false + }, + { + "id": "consent-does-not-hide-access-denied", + "category": "structured-extraction", + "root_cause": "cookie_preferences_not_blocked_documentation", + "failure_mode": "false-positive", + "case": { + "url": "https://mui.com/x/react-data-grid/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 5, + "stable_text_anchors": [], + "record_identity_fields": [ + "id" + ], + "table_columns": [ + { + "name": "id", + "label": "ID", + "type": "integer", + "minimum": 1 + }, + { + "name": "first", + "label": "First name", + "type": "text" + }, + { + "name": "last", + "label": "Last name", + "type": "text" + }, + { + "name": "age", + "label": "Age", + "type": "integer", + "minimum": 0, + "allow_empty": true } + ], + "table_scope": { + "root_selector": "[role=grid]", + "root_index": 0, + "row_selector": "[role=row][data-id]", + "cell_selector": "[role=gridcell]:not([data-field=\"__check__\"])", + "start_pattern": "Community version \\(free forever\\)", + "end_pattern": "(?m)^[ \\t#]*Pro version[ \\t]*$", + "plain_row_pattern": "(?P\\d+)\\s+(?P[A-Za-z]+)\\s+(?P[A-Za-z]+)(?:[ \\t]+(?P\\d*))", + "plain_multiline_row_pattern": "(?m)^[ \\t]*(?P\\d+)\\n[ \\t\\n]*(?P[A-Za-z]+)\\n[ \\t\\n]*(?P[A-Za-z]+)\\n[ \\t\\n]*(?P\\d+)[ \\t]*$", + "reference_kind": "fixed_demo", + "reference_rows": [ + { + "id": "1", + "first": "Jon", + "last": "Snow", + "age": "14" + }, + { + "id": "2", + "first": "Cersei", + "last": "Lannister", + "age": "31" + }, + { + "id": "3", + "first": "Jaime", + "last": "Lannister", + "age": "31" + }, + { + "id": "4", + "first": "Arya", + "last": "Stark", + "age": "11" + }, + { + "id": "5", + "first": "Daenerys", + "last": "Targaryen", + "age": "" + } + ] + } }, - "artifact": "[Library](/library/) [Library](/items/new)", - "ok": true, - "expected_task_pass": true - }, - { - "id": "anchor-owned-links-non-http-navigation", - "category": "format-normalization", - "root_cause": "record_title_masking_erases_independent_navigation", - "failure_mode": "false-negative", - "case": { - "url": "https://example.org/", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 1, - "hosts": [ - "example.org" - ], - "entry_path": "/items/[^/]+", - "name_validation": "nonempty", - "stable_text_anchors": [ - { - "id": "nav", - "alternatives": [ - "Library" - ], - "match_scope": "non_record_lines" - } - ] + "body_requirements": { + "version": 3, + "unit": "region_compact_hexagram_recall", + "minimum_coverage": 0.8, + "minimum_floor": 0.6, + "minimum_region_passes": 1, + "segmentation_method": "sentences", + "regions": [ + { + "id": "body-region-1", + "text": "the grid displays structured rows and columns each row preserves the fields of one record", + "source_text": "The grid displays structured rows and columns.\n\nEach row preserves the fields of one record.", + "segmentation_method": "sentences" } + ] }, - "artifact": "[Library](mailto:help@example.org)\n[Library](/items/new)", - "ok": true, - "expected_task_pass": true - }, - { - "id": "nullable-table-faithful-empty", - "category": "structured-extraction", - "root_cause": "declared_empty_table_cells_are_real_values", - "failure_mode": "false-negative", - "case": { - "url": "https://mui.com/x/react-data-grid/", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "table_row", - "minimum_records": 5, - "stable_text_anchors": [], - "record_identity_fields": [ - "id" - ], - "table_columns": [ - { - "name": "id", - "label": "ID", - "type": "integer", - "minimum": 1 - }, - { - "name": "first", - "label": "First name", - "type": "text" - }, - { - "name": "last", - "label": "Last name", - "type": "text" - }, - { - "name": "age", - "label": "Age", - "type": "integer", - "minimum": 0, - "allow_empty": true - } - ], - "table_scope": { - "root_selector": "[role=grid]", - "root_index": 0, - "row_selector": "[role=row][data-id]", - "cell_selector": "[role=gridcell]:not([data-field=\"__check__\"])", - "start_pattern": "Community version \\(free forever\\)", - "end_pattern": "(?m)^[ \\t#]*Pro version[ \\t]*$", - "plain_row_pattern": "(?P\\d+)\\s+(?P[A-Za-z]+)\\s+(?P[A-Za-z]+)(?:[ \\t]+(?P\\d*))", - "plain_multiline_row_pattern": "(?m)^[ \\t]*(?P\\d+)\\n[ \\t\\n]*(?P[A-Za-z]+)\\n[ \\t\\n]*(?P[A-Za-z]+)\\n[ \\t\\n]*(?P\\d+)[ \\t]*$", - "reference_kind": "fixed_demo", - "reference_rows": [ - { - "id": "1", - "first": "Jon", - "last": "Snow", - "age": "14" - }, - { - "id": "2", - "first": "Cersei", - "last": "Lannister", - "age": "31" - }, - { - "id": "3", - "first": "Jaime", - "last": "Lannister", - "age": "31" - }, - { - "id": "4", - "first": "Arya", - "last": "Stark", - "age": "11" - }, - { - "id": "5", - "first": "Daenerys", - "last": "Targaryen", - "age": "" - } - ] - } + "forbidden_any": [ + "Access denied", + "CAPTCHA" + ] + }, + "artifact": "Access denied\nThe grid displays structured rows and columns. Each row preserves the fields of one record.\nCommunity version (free forever)\n|ID|First name|Last name|Age|\n|---|---|---|---|\n|1|Jon|Snow|14|\n|2|Cersei|Lannister|31|\n|3|Jaime|Lannister|31|\n|4|Arya|Stark|11|\n|5|Daenerys|Targaryen||\nPro version", + "ok": true, + "expected_task_pass": false + }, + { + "id": "mui-rowheader-full-name-counts", + "category": "structured-extraction", + "root_cause": "documentation_prose_and_grid_are_separate_requirements", + "failure_mode": "false-negative", + "case": { + "url": "https://mui.com/x/react-data-grid/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 5, + "stable_text_anchors": [], + "record_identity_fields": [ + "id" + ], + "table_columns": [ + { + "name": "id", + "label": "ID", + "type": "integer", + "minimum": 1 + }, + { + "name": "first", + "label": "First name", + "type": "text" + }, + { + "name": "last", + "label": "Last name", + "type": "text" + }, + { + "name": "age", + "label": "Age", + "type": "integer", + "minimum": 0, + "allow_empty": true + }, + { + "name": "full", + "label": "Full name", + "type": "text" } - }, - "artifact": "Community version (free forever)\n|ID|First name|Last name|Age|\n|---|---|---|---|\n|1|Jon|Snow|14|\n|2|Cersei|Lannister|31|\n|3|Jaime|Lannister|31|\n|4|Arya|Stark|11|\n|5|Daenerys|Targaryen||\nPro version", - "ok": true, - "expected_task_pass": true - }, - { - "id": "nullable-table-wrong-empty", - "category": "structured-extraction", - "root_cause": "declared_empty_table_cells_are_real_values", - "failure_mode": "false-positive", - "case": { - "url": "https://mui.com/x/react-data-grid/", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "table_row", - "minimum_records": 5, - "stable_text_anchors": [], - "record_identity_fields": [ - "id" - ], - "table_columns": [ - { - "name": "id", - "label": "ID", - "type": "integer", - "minimum": 1 - }, - { - "name": "first", - "label": "First name", - "type": "text" - }, - { - "name": "last", - "label": "Last name", - "type": "text" - }, - { - "name": "age", - "label": "Age", - "type": "integer", - "minimum": 0, - "allow_empty": true - } - ], - "table_scope": { - "root_selector": "[role=grid]", - "root_index": 0, - "row_selector": "[role=row][data-id]", - "cell_selector": "[role=gridcell]:not([data-field=\"__check__\"])", - "start_pattern": "Community version \\(free forever\\)", - "end_pattern": "(?m)^[ \\t#]*Pro version[ \\t]*$", - "plain_row_pattern": "(?P\\d+)\\s+(?P[A-Za-z]+)\\s+(?P[A-Za-z]+)(?:[ \\t]+(?P\\d*))", - "plain_multiline_row_pattern": "(?m)^[ \\t]*(?P\\d+)\\n[ \\t\\n]*(?P[A-Za-z]+)\\n[ \\t\\n]*(?P[A-Za-z]+)\\n[ \\t\\n]*(?P\\d+)[ \\t]*$", - "reference_kind": "fixed_demo", - "reference_rows": [ - { - "id": "1", - "first": "Jon", - "last": "Snow", - "age": "14" - }, - { - "id": "2", - "first": "Cersei", - "last": "Lannister", - "age": "31" - }, - { - "id": "3", - "first": "Jaime", - "last": "Lannister", - "age": "31" - }, - { - "id": "4", - "first": "Arya", - "last": "Stark", - "age": "11" - }, - { - "id": "5", - "first": "Daenerys", - "last": "Targaryen", - "age": "" - } - ] - } + ], + "table_scope": { + "root_selector": "[role=grid]", + "root_index": 0, + "row_selector": "[role=row][data-id]", + "cell_selector": "[role=gridcell]:not([data-field=\"__check__\"]),[role=rowheader]", + "start_pattern": "Community version \\(free forever\\)", + "end_pattern": "(?m)^[ \\t#]*Pro version[ \\t]*$", + "plain_row_pattern": "(?P\\d+)\\s+(?P[A-Za-z]+)\\s+(?P[A-Za-z]+)[ \\t]+(?P\\d*)[ \\t]+(?P[A-Za-z]+ [A-Za-z]+)", + "plain_multiline_row_pattern": "(?m)^[ \\t]*(?P\\d+)\\n[ \\t\\n]*(?P[A-Za-z]+)\\n[ \\t\\n]*(?P[A-Za-z]+)\\n[ \\t\\n]*(?P\\d+)\\n[ \\t\\n]*(?P[A-Za-z]+ [A-Za-z]+)[ \\t]*$", + "reference_kind": "fixed_demo", + "reference_rows": [ + { + "id": "1", + "first": "Jon", + "last": "Snow", + "age": "14", + "full": "Jon Snow" + }, + { + "id": "2", + "first": "Cersei", + "last": "Lannister", + "age": "31", + "full": "Cersei Lannister" + }, + { + "id": "3", + "first": "Jaime", + "last": "Lannister", + "age": "31", + "full": "Jaime Lannister" + }, + { + "id": "4", + "first": "Arya", + "last": "Stark", + "age": "11", + "full": "Arya Stark" + }, + { + "id": "5", + "first": "Daenerys", + "last": "Targaryen", + "age": "", + "full": "Daenerys Targaryen" + } + ] + } + } + }, + "artifact": "
1
Jon
Snow
14
Jon Snow
2
Cersei
Lannister
31
Cersei Lannister
3
Jaime
Lannister
31
Jaime Lannister
4
Arya
Stark
11
Arya Stark
5
Daenerys
Targaryen
Daenerys Targaryen
", + "ok": true, + "expected_task_pass": true + }, + { + "id": "mui-missing-rowheader-full-name", + "category": "structured-extraction", + "root_cause": "documentation_prose_and_grid_are_separate_requirements", + "failure_mode": "false-positive", + "case": { + "url": "https://mui.com/x/react-data-grid/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 5, + "stable_text_anchors": [], + "record_identity_fields": [ + "id" + ], + "table_columns": [ + { + "name": "id", + "label": "ID", + "type": "integer", + "minimum": 1 + }, + { + "name": "first", + "label": "First name", + "type": "text" + }, + { + "name": "last", + "label": "Last name", + "type": "text" + }, + { + "name": "age", + "label": "Age", + "type": "integer", + "minimum": 0, + "allow_empty": true + }, + { + "name": "full", + "label": "Full name", + "type": "text" } - }, - "artifact": "Community version (free forever)\n|ID|First name|Last name|Age|\n|---|---|---|---|\n|1|Jon|Snow||\n|2|Cersei|Lannister|31|\n|3|Jaime|Lannister|31|\n|4|Arya|Stark|11|\n|5|Daenerys|Targaryen||\nPro version", - "ok": true, - "expected_task_pass": false - }, - { - "id": "nullable-table-invented-age", - "category": "structured-extraction", - "root_cause": "declared_empty_table_cells_are_real_values", - "failure_mode": "false-positive", - "case": { - "url": "https://mui.com/x/react-data-grid/", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "table_row", - "minimum_records": 5, - "stable_text_anchors": [], - "record_identity_fields": [ - "id" - ], - "table_columns": [ - { - "name": "id", - "label": "ID", - "type": "integer", - "minimum": 1 - }, - { - "name": "first", - "label": "First name", - "type": "text" - }, - { - "name": "last", - "label": "Last name", - "type": "text" - }, - { - "name": "age", - "label": "Age", - "type": "integer", - "minimum": 0, - "allow_empty": true - } - ], - "table_scope": { - "root_selector": "[role=grid]", - "root_index": 0, - "row_selector": "[role=row][data-id]", - "cell_selector": "[role=gridcell]:not([data-field=\"__check__\"])", - "start_pattern": "Community version \\(free forever\\)", - "end_pattern": "(?m)^[ \\t#]*Pro version[ \\t]*$", - "plain_row_pattern": "(?P\\d+)\\s+(?P[A-Za-z]+)\\s+(?P[A-Za-z]+)(?:[ \\t]+(?P\\d*))", - "plain_multiline_row_pattern": "(?m)^[ \\t]*(?P\\d+)\\n[ \\t\\n]*(?P[A-Za-z]+)\\n[ \\t\\n]*(?P[A-Za-z]+)\\n[ \\t\\n]*(?P\\d+)[ \\t]*$", - "reference_kind": "fixed_demo", - "reference_rows": [ - { - "id": "1", - "first": "Jon", - "last": "Snow", - "age": "14" - }, - { - "id": "2", - "first": "Cersei", - "last": "Lannister", - "age": "31" - }, - { - "id": "3", - "first": "Jaime", - "last": "Lannister", - "age": "31" - }, - { - "id": "4", - "first": "Arya", - "last": "Stark", - "age": "11" - }, - { - "id": "5", - "first": "Daenerys", - "last": "Targaryen", - "age": "" - } - ] - } + ], + "table_scope": { + "root_selector": "[role=grid]", + "root_index": 0, + "row_selector": "[role=row][data-id]", + "cell_selector": "[role=gridcell]:not([data-field=\"__check__\"]),[role=rowheader]", + "start_pattern": "Community version \\(free forever\\)", + "end_pattern": "(?m)^[ \\t#]*Pro version[ \\t]*$", + "plain_row_pattern": "(?P\\d+)\\s+(?P[A-Za-z]+)\\s+(?P[A-Za-z]+)[ \\t]+(?P\\d*)[ \\t]+(?P[A-Za-z]+ [A-Za-z]+)", + "plain_multiline_row_pattern": "(?m)^[ \\t]*(?P\\d+)\\n[ \\t\\n]*(?P[A-Za-z]+)\\n[ \\t\\n]*(?P[A-Za-z]+)\\n[ \\t\\n]*(?P\\d+)\\n[ \\t\\n]*(?P[A-Za-z]+ [A-Za-z]+)[ \\t]*$", + "reference_kind": "fixed_demo", + "reference_rows": [ + { + "id": "1", + "first": "Jon", + "last": "Snow", + "age": "14", + "full": "Jon Snow" + }, + { + "id": "2", + "first": "Cersei", + "last": "Lannister", + "age": "31", + "full": "Cersei Lannister" + }, + { + "id": "3", + "first": "Jaime", + "last": "Lannister", + "age": "31", + "full": "Jaime Lannister" + }, + { + "id": "4", + "first": "Arya", + "last": "Stark", + "age": "11", + "full": "Arya Stark" + }, + { + "id": "5", + "first": "Daenerys", + "last": "Targaryen", + "age": "", + "full": "Daenerys Targaryen" + } + ] + } + } + }, + "artifact": "
1
Jon
Snow
14
2
Cersei
Lannister
31
Cersei Lannister
3
Jaime
Lannister
31
Jaime Lannister
4
Arya
Stark
11
Arya Stark
5
Daenerys
Targaryen
Daenerys Targaryen
", + "ok": true, + "expected_task_pass": false + }, + { + "id": "html-anchor-entity-named-space", + "category": "format-normalization", + "root_cause": "html_entity_space_is_visible_whitespace", + "failure_mode": "false-negative", + "case": { + "url": "https://example.org/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "body_text", + "minimum_records": 0, + "stable_text_anchors": [ + { + "id": "heading", + "alternatives": [ + "MUI X Data Grid" + ] } - }, - "artifact": "Community version (free forever)\n|ID|First name|Last name|Age|\n|---|---|---|---|\n|1|Jon|Snow|14|\n|2|Cersei|Lannister|31|\n|3|Jaime|Lannister|31|\n|4|Arya|Stark|11|\n|5|Daenerys|Targaryen|25|\nPro version", - "ok": true, - "expected_task_pass": false - }, - { - "id": "nullable-table-missing-cell", - "category": "structured-extraction", - "root_cause": "declared_empty_table_cells_are_real_values", - "failure_mode": "false-positive", - "case": { - "url": "https://mui.com/x/react-data-grid/", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "table_row", - "minimum_records": 5, - "stable_text_anchors": [], - "record_identity_fields": [ - "id" - ], - "table_columns": [ - { - "name": "id", - "label": "ID", - "type": "integer", - "minimum": 1 - }, - { - "name": "first", - "label": "First name", - "type": "text" - }, - { - "name": "last", - "label": "Last name", - "type": "text" - }, - { - "name": "age", - "label": "Age", - "type": "integer", - "minimum": 0, - "allow_empty": true - } - ], - "table_scope": { - "root_selector": "[role=grid]", - "root_index": 0, - "row_selector": "[role=row][data-id]", - "cell_selector": "[role=gridcell]:not([data-field=\"__check__\"])", - "start_pattern": "Community version \\(free forever\\)", - "end_pattern": "(?m)^[ \\t#]*Pro version[ \\t]*$", - "plain_row_pattern": "(?P\\d+)\\s+(?P[A-Za-z]+)\\s+(?P[A-Za-z]+)(?:[ \\t]+(?P\\d*))", - "plain_multiline_row_pattern": "(?m)^[ \\t]*(?P\\d+)\\n[ \\t\\n]*(?P[A-Za-z]+)\\n[ \\t\\n]*(?P[A-Za-z]+)\\n[ \\t\\n]*(?P\\d+)[ \\t]*$", - "reference_kind": "fixed_demo", - "reference_rows": [ - { - "id": "1", - "first": "Jon", - "last": "Snow", - "age": "14" - }, - { - "id": "2", - "first": "Cersei", - "last": "Lannister", - "age": "31" - }, - { - "id": "3", - "first": "Jaime", - "last": "Lannister", - "age": "31" - }, - { - "id": "4", - "first": "Arya", - "last": "Stark", - "age": "11" - }, - { - "id": "5", - "first": "Daenerys", - "last": "Targaryen", - "age": "" - } - ] - } + ] + } + }, + "artifact": "

MUI X Data Grid

", + "ok": true, + "expected_task_pass": true + }, + { + "id": "html-anchor-entity-numeric-space", + "category": "format-normalization", + "root_cause": "html_entity_space_is_visible_whitespace", + "failure_mode": "false-negative", + "case": { + "url": "https://example.org/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "body_text", + "minimum_records": 0, + "stable_text_anchors": [ + { + "id": "heading", + "alternatives": [ + "MUI X Data Grid" + ] + } + ] + } + }, + "artifact": "

MUI X Data Grid

", + "ok": true, + "expected_task_pass": true + }, + { + "id": "html-anchor-entity-wrong-heading", + "category": "format-normalization", + "root_cause": "html_entity_space_is_visible_whitespace", + "failure_mode": "false-positive", + "case": { + "url": "https://example.org/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "body_text", + "minimum_records": 0, + "stable_text_anchors": [ + { + "id": "heading", + "alternatives": [ + "MUI X Data Grid" + ] } - }, - "artifact": "Community version (free forever)\n|ID|First name|Last name|Age|\n|---|---|---|---|\n|1|Jon|Snow|14|\n|2|Cersei|Lannister|31|\n|3|Jaime|Lannister|31|\n|4|Arya|Stark|11|\n|5|Daenerys|Targaryen|\nPro version", - "ok": true, - "expected_task_pass": false - }, - { - "id": "nullable-table-mixed-row", - "category": "structured-extraction", - "root_cause": "declared_empty_table_cells_are_real_values", - "failure_mode": "false-positive", - "case": { - "url": "https://mui.com/x/react-data-grid/", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "table_row", - "minimum_records": 5, - "stable_text_anchors": [], - "record_identity_fields": [ - "id" - ], - "table_columns": [ - { - "name": "id", - "label": "ID", - "type": "integer", - "minimum": 1 - }, - { - "name": "first", - "label": "First name", - "type": "text" - }, - { - "name": "last", - "label": "Last name", - "type": "text" - }, - { - "name": "age", - "label": "Age", - "type": "integer", - "minimum": 0, - "allow_empty": true - } - ], - "table_scope": { - "root_selector": "[role=grid]", - "root_index": 0, - "row_selector": "[role=row][data-id]", - "cell_selector": "[role=gridcell]:not([data-field=\"__check__\"])", - "start_pattern": "Community version \\(free forever\\)", - "end_pattern": "(?m)^[ \\t#]*Pro version[ \\t]*$", - "plain_row_pattern": "(?P\\d+)\\s+(?P[A-Za-z]+)\\s+(?P[A-Za-z]+)(?:[ \\t]+(?P\\d*))", - "plain_multiline_row_pattern": "(?m)^[ \\t]*(?P\\d+)\\n[ \\t\\n]*(?P[A-Za-z]+)\\n[ \\t\\n]*(?P[A-Za-z]+)\\n[ \\t\\n]*(?P\\d+)[ \\t]*$", - "reference_kind": "fixed_demo", - "reference_rows": [ - { - "id": "1", - "first": "Jon", - "last": "Snow", - "age": "14" - }, - { - "id": "2", - "first": "Cersei", - "last": "Lannister", - "age": "31" - }, - { - "id": "3", - "first": "Jaime", - "last": "Lannister", - "age": "31" - }, - { - "id": "4", - "first": "Arya", - "last": "Stark", - "age": "11" - }, - { - "id": "5", - "first": "Daenerys", - "last": "Targaryen", - "age": "" - } - ] - } + ] + } + }, + "artifact": "

MUI X Date Picker

", + "ok": true, + "expected_task_pass": false + }, + { + "id": "html-anchor-entity-literal-entity", + "category": "format-normalization", + "root_cause": "html_entity_space_is_visible_whitespace", + "failure_mode": "false-positive", + "case": { + "url": "https://example.org/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "body_text", + "minimum_records": 0, + "stable_text_anchors": [ + { + "id": "heading", + "alternatives": [ + "MUI X Data Grid" + ] } - }, - "artifact": "Community version (free forever)\n|ID|First name|Last name|Age|\n|---|---|---|---|\n|1|Jon|Lannister|14|\n|2|Cersei|Lannister|31|\n|3|Jaime|Lannister|31|\n|4|Arya|Stark|11|\n|5|Daenerys|Targaryen||\nPro version", - "ok": true, - "expected_task_pass": false - }, - { - "id": "documentation-prose-and-grid", - "category": "structured-extraction", - "root_cause": "documentation_prose_and_grid_are_separate_requirements", - "failure_mode": "false-negative", - "case": { - "url": "https://mui.com/x/react-data-grid/", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "table_row", - "minimum_records": 5, - "stable_text_anchors": [], - "record_identity_fields": [ - "id" - ], - "table_columns": [ - { - "name": "id", - "label": "ID", - "type": "integer", - "minimum": 1 - }, - { - "name": "first", - "label": "First name", - "type": "text" - }, - { - "name": "last", - "label": "Last name", - "type": "text" - }, - { - "name": "age", - "label": "Age", - "type": "integer", - "minimum": 0, - "allow_empty": true - } - ], - "table_scope": { - "root_selector": "[role=grid]", - "root_index": 0, - "row_selector": "[role=row][data-id]", - "cell_selector": "[role=gridcell]:not([data-field=\"__check__\"])", - "start_pattern": "Community version \\(free forever\\)", - "end_pattern": "(?m)^[ \\t#]*Pro version[ \\t]*$", - "plain_row_pattern": "(?P\\d+)\\s+(?P[A-Za-z]+)\\s+(?P[A-Za-z]+)(?:[ \\t]+(?P\\d*))", - "plain_multiline_row_pattern": "(?m)^[ \\t]*(?P\\d+)\\n[ \\t\\n]*(?P[A-Za-z]+)\\n[ \\t\\n]*(?P[A-Za-z]+)\\n[ \\t\\n]*(?P\\d+)[ \\t]*$", - "reference_kind": "fixed_demo", - "reference_rows": [ - { - "id": "1", - "first": "Jon", - "last": "Snow", - "age": "14" - }, - { - "id": "2", - "first": "Cersei", - "last": "Lannister", - "age": "31" - }, - { - "id": "3", - "first": "Jaime", - "last": "Lannister", - "age": "31" - }, - { - "id": "4", - "first": "Arya", - "last": "Stark", - "age": "11" - }, - { - "id": "5", - "first": "Daenerys", - "last": "Targaryen", - "age": "" - } - ] - } + ] + } + }, + "artifact": "

MUI&nbsp;X Data Grid

", + "ok": true, + "expected_task_pass": false + }, + { + "id": "table-source-duplicate-faithful", + "category": "structured-extraction", + "root_cause": "source_identical_table_duplicates_count_once", + "failure_mode": "false-negative", + "case": { + "url": "https://vuetifyjs.com/en/components/data-tables/basics/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 2, + "stable_text_anchors": [], + "record_identity_fields": [ + "name", + "species" + ], + "duplicate_policy": "deduplicate_identical", + "table_columns": [ + { + "name": "name", + "label": "Name", + "type": "text" }, - "body_requirements": { - "version": 3, - "unit": "region_compact_hexagram_recall", - "minimum_coverage": 0.8, - "minimum_floor": 0.6, - "minimum_region_passes": 1, - "segmentation_method": "sentences", - "regions": [ - { - "id": "body-region-1", - "text": "the grid displays structured rows and columns each row preserves the fields of one record", - "source_text": "The grid displays structured rows and columns.\n\nEach row preserves the fields of one record.", - "segmentation_method": "sentences" - } - ] - }, - "forbidden_any": [ - "Access denied", - "CAPTCHA" - ] - }, - "artifact": "The grid displays structured rows and columns. Each row preserves the fields of one record.\nCommunity version (free forever)\n|ID|First name|Last name|Age|\n|---|---|---|---|\n|1|Jon|Snow|14|\n|2|Cersei|Lannister|31|\n|3|Jaime|Lannister|31|\n|4|Arya|Stark|11|\n|5|Daenerys|Targaryen||\nPro version", - "ok": true, - "expected_task_pass": true - }, - { - "id": "documentation-prose-without-grid", - "category": "structured-extraction", - "root_cause": "documentation_prose_and_grid_are_separate_requirements", - "failure_mode": "false-positive", - "case": { - "url": "https://mui.com/x/react-data-grid/", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "table_row", - "minimum_records": 5, - "stable_text_anchors": [], - "record_identity_fields": [ - "id" - ], - "table_columns": [ - { - "name": "id", - "label": "ID", - "type": "integer", - "minimum": 1 - }, - { - "name": "first", - "label": "First name", - "type": "text" - }, - { - "name": "last", - "label": "Last name", - "type": "text" - }, - { - "name": "age", - "label": "Age", - "type": "integer", - "minimum": 0, - "allow_empty": true - } - ], - "table_scope": { - "root_selector": "[role=grid]", - "root_index": 0, - "row_selector": "[role=row][data-id]", - "cell_selector": "[role=gridcell]:not([data-field=\"__check__\"])", - "start_pattern": "Community version \\(free forever\\)", - "end_pattern": "(?m)^[ \\t#]*Pro version[ \\t]*$", - "plain_row_pattern": "(?P\\d+)\\s+(?P[A-Za-z]+)\\s+(?P[A-Za-z]+)(?:[ \\t]+(?P\\d*))", - "plain_multiline_row_pattern": "(?m)^[ \\t]*(?P\\d+)\\n[ \\t\\n]*(?P[A-Za-z]+)\\n[ \\t\\n]*(?P[A-Za-z]+)\\n[ \\t\\n]*(?P\\d+)[ \\t]*$", - "reference_kind": "fixed_demo", - "reference_rows": [ - { - "id": "1", - "first": "Jon", - "last": "Snow", - "age": "14" - }, - { - "id": "2", - "first": "Cersei", - "last": "Lannister", - "age": "31" - }, - { - "id": "3", - "first": "Jaime", - "last": "Lannister", - "age": "31" - }, - { - "id": "4", - "first": "Arya", - "last": "Stark", - "age": "11" - }, - { - "id": "5", - "first": "Daenerys", - "last": "Targaryen", - "age": "" - } - ] - } + { + "name": "species", + "label": "Species", + "type": "text" }, - "body_requirements": { - "version": 3, - "unit": "region_compact_hexagram_recall", - "minimum_coverage": 0.8, - "minimum_floor": 0.6, - "minimum_region_passes": 1, - "segmentation_method": "sentences", - "regions": [ - { - "id": "body-region-1", - "text": "the grid displays structured rows and columns each row preserves the fields of one record", - "source_text": "The grid displays structured rows and columns.\n\nEach row preserves the fields of one record.", - "segmentation_method": "sentences" - } - ] - }, - "forbidden_any": [ - "Access denied", - "CAPTCHA" - ] - }, - "artifact": "The grid displays structured rows and columns. Each row preserves the fields of one record.", - "ok": true, - "expected_task_pass": false - }, - { - "id": "documentation-grid-without-prose", - "category": "structured-extraction", - "root_cause": "documentation_prose_and_grid_are_separate_requirements", - "failure_mode": "false-positive", - "case": { - "url": "https://mui.com/x/react-data-grid/", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "table_row", - "minimum_records": 5, - "stable_text_anchors": [], - "record_identity_fields": [ - "id" - ], - "table_columns": [ - { - "name": "id", - "label": "ID", - "type": "integer", - "minimum": 1 - }, - { - "name": "first", - "label": "First name", - "type": "text" - }, - { - "name": "last", - "label": "Last name", - "type": "text" - }, - { - "name": "age", - "label": "Age", - "type": "integer", - "minimum": 0, - "allow_empty": true - } - ], - "table_scope": { - "root_selector": "[role=grid]", - "root_index": 0, - "row_selector": "[role=row][data-id]", - "cell_selector": "[role=gridcell]:not([data-field=\"__check__\"])", - "start_pattern": "Community version \\(free forever\\)", - "end_pattern": "(?m)^[ \\t#]*Pro version[ \\t]*$", - "plain_row_pattern": "(?P\\d+)\\s+(?P[A-Za-z]+)\\s+(?P[A-Za-z]+)(?:[ \\t]+(?P\\d*))", - "plain_multiline_row_pattern": "(?m)^[ \\t]*(?P\\d+)\\n[ \\t\\n]*(?P[A-Za-z]+)\\n[ \\t\\n]*(?P[A-Za-z]+)\\n[ \\t\\n]*(?P\\d+)[ \\t]*$", - "reference_kind": "fixed_demo", - "reference_rows": [ - { - "id": "1", - "first": "Jon", - "last": "Snow", - "age": "14" - }, - { - "id": "2", - "first": "Cersei", - "last": "Lannister", - "age": "31" - }, - { - "id": "3", - "first": "Jaime", - "last": "Lannister", - "age": "31" - }, - { - "id": "4", - "first": "Arya", - "last": "Stark", - "age": "11" - }, - { - "id": "5", - "first": "Daenerys", - "last": "Targaryen", - "age": "" - } - ] - } + { + "name": "diet", + "label": "Diet", + "type": "text" }, - "body_requirements": { - "version": 3, - "unit": "region_compact_hexagram_recall", - "minimum_coverage": 0.8, - "minimum_floor": 0.6, - "minimum_region_passes": 1, - "segmentation_method": "sentences", - "regions": [ - { - "id": "body-region-1", - "text": "the grid displays structured rows and columns each row preserves the fields of one record", - "source_text": "The grid displays structured rows and columns.\n\nEach row preserves the fields of one record.", - "segmentation_method": "sentences" - } - ] - }, - "forbidden_any": [ - "Access denied", - "CAPTCHA" + { + "name": "habitat", + "label": "Habitat", + "type": "text" + } + ], + "table_scope": { + "root_selector": "section#usage", + "root_index": 0, + "row_selector": "table tbody tr", + "cell_selector": "td", + "start_pattern": "The standard data table presumes that the entire data set is available locally\\.", + "end_pattern": "(?m)^[ \\t#]*API[ \\t]*$", + "plain_row_pattern": "(?P[A-Z][A-Za-z]+(?: [A-Z][A-Za-z]+)*)\\s+(?P[A-Z][a-z]+ [a-z]+)\\s+(?PHerbivore|Carnivore|Omnivore)\\s+(?P[A-Za-z, ]+)", + "plain_multiline_row_pattern": "(?m)^[ \\t]*(?P[A-Z][A-Za-z]+(?: [A-Z][A-Za-z]+)*)\\n[ \\t\\n]*(?P[A-Z][a-z]+ [a-z]+)\\n[ \\t\\n]*(?PHerbivore|Carnivore|Omnivore)\\n[ \\t\\n]*(?P[A-Za-z, ]+)[ \\t]*$", + "reference_kind": "fixed_demo", + "reference_rows": [ + { + "name": "African Elephant", + "species": "Loxodonta africana", + "diet": "Herbivore", + "habitat": "Savanna, Forests" + }, + { + "name": "Lion", + "species": "Panthera leo", + "diet": "Carnivore", + "habitat": "Savanna, Grassland" + } ] - }, - "artifact": "Community version (free forever)\n|ID|First name|Last name|Age|\n|---|---|---|---|\n|1|Jon|Snow|14|\n|2|Cersei|Lannister|31|\n|3|Jaime|Lannister|31|\n|4|Arya|Stark|11|\n|5|Daenerys|Targaryen||\nPro version", - "ok": true, - "expected_task_pass": false - }, - { - "id": "consent-alongside-real-documentation", - "category": "structured-extraction", - "root_cause": "cookie_preferences_not_blocked_documentation", - "failure_mode": "false-negative", - "case": { - "url": "https://mui.com/x/react-data-grid/", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "table_row", - "minimum_records": 5, - "stable_text_anchors": [], - "record_identity_fields": [ - "id" - ], - "table_columns": [ - { - "name": "id", - "label": "ID", - "type": "integer", - "minimum": 1 - }, - { - "name": "first", - "label": "First name", - "type": "text" - }, - { - "name": "last", - "label": "Last name", - "type": "text" - }, - { - "name": "age", - "label": "Age", - "type": "integer", - "minimum": 0, - "allow_empty": true - } - ], - "table_scope": { - "root_selector": "[role=grid]", - "root_index": 0, - "row_selector": "[role=row][data-id]", - "cell_selector": "[role=gridcell]:not([data-field=\"__check__\"])", - "start_pattern": "Community version \\(free forever\\)", - "end_pattern": "(?m)^[ \\t#]*Pro version[ \\t]*$", - "plain_row_pattern": "(?P\\d+)\\s+(?P[A-Za-z]+)\\s+(?P[A-Za-z]+)(?:[ \\t]+(?P\\d*))", - "plain_multiline_row_pattern": "(?m)^[ \\t]*(?P\\d+)\\n[ \\t\\n]*(?P[A-Za-z]+)\\n[ \\t\\n]*(?P[A-Za-z]+)\\n[ \\t\\n]*(?P\\d+)[ \\t]*$", - "reference_kind": "fixed_demo", - "reference_rows": [ - { - "id": "1", - "first": "Jon", - "last": "Snow", - "age": "14" - }, - { - "id": "2", - "first": "Cersei", - "last": "Lannister", - "age": "31" - }, - { - "id": "3", - "first": "Jaime", - "last": "Lannister", - "age": "31" - }, - { - "id": "4", - "first": "Arya", - "last": "Stark", - "age": "11" - }, - { - "id": "5", - "first": "Daenerys", - "last": "Targaryen", - "age": "" - } - ] - } + } + } + }, + "artifact": "The standard data table presumes that the entire data set is available locally.\n|Name|Species|Diet|Habitat|\n|---|---|---|---|\n|African Elephant|Loxodonta africana|Herbivore|Savanna, Forests|\n|Lion|Panthera leo|Carnivore|Savanna, Grassland|\n|Lion|Panthera leo|Carnivore|Savanna, Grassland|\n", + "ok": true, + "expected_task_pass": true + }, + { + "id": "table-source-duplicate-not-extra-record", + "category": "structured-extraction", + "root_cause": "source_identical_table_duplicates_count_once", + "failure_mode": "false-positive", + "case": { + "url": "https://vuetifyjs.com/en/components/data-tables/basics/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 2, + "stable_text_anchors": [], + "record_identity_fields": [ + "name", + "species" + ], + "duplicate_policy": "deduplicate_identical", + "table_columns": [ + { + "name": "name", + "label": "Name", + "type": "text" }, - "body_requirements": { - "version": 3, - "unit": "region_compact_hexagram_recall", - "minimum_coverage": 0.8, - "minimum_floor": 0.6, - "minimum_region_passes": 1, - "segmentation_method": "sentences", - "regions": [ - { - "id": "body-region-1", - "text": "the grid displays structured rows and columns each row preserves the fields of one record", - "source_text": "The grid displays structured rows and columns.\n\nEach row preserves the fields of one record.", - "segmentation_method": "sentences" - } - ] - }, - "forbidden_any": [ - "Access denied", - "CAPTCHA" - ] - }, - "artifact": "Cookie preferences\nThe grid displays structured rows and columns. Each row preserves the fields of one record.\nCommunity version (free forever)\n|ID|First name|Last name|Age|\n|---|---|---|---|\n|1|Jon|Snow|14|\n|2|Cersei|Lannister|31|\n|3|Jaime|Lannister|31|\n|4|Arya|Stark|11|\n|5|Daenerys|Targaryen||\nPro version", - "ok": true, - "expected_task_pass": true - }, - { - "id": "consent-only-is-not-documentation", - "category": "structured-extraction", - "root_cause": "cookie_preferences_not_blocked_documentation", - "failure_mode": "false-positive", - "case": { - "url": "https://mui.com/x/react-data-grid/", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "table_row", - "minimum_records": 5, - "stable_text_anchors": [], - "record_identity_fields": [ - "id" - ], - "table_columns": [ - { - "name": "id", - "label": "ID", - "type": "integer", - "minimum": 1 - }, - { - "name": "first", - "label": "First name", - "type": "text" - }, - { - "name": "last", - "label": "Last name", - "type": "text" - }, - { - "name": "age", - "label": "Age", - "type": "integer", - "minimum": 0, - "allow_empty": true - } - ], - "table_scope": { - "root_selector": "[role=grid]", - "root_index": 0, - "row_selector": "[role=row][data-id]", - "cell_selector": "[role=gridcell]:not([data-field=\"__check__\"])", - "start_pattern": "Community version \\(free forever\\)", - "end_pattern": "(?m)^[ \\t#]*Pro version[ \\t]*$", - "plain_row_pattern": "(?P\\d+)\\s+(?P[A-Za-z]+)\\s+(?P[A-Za-z]+)(?:[ \\t]+(?P\\d*))", - "plain_multiline_row_pattern": "(?m)^[ \\t]*(?P\\d+)\\n[ \\t\\n]*(?P[A-Za-z]+)\\n[ \\t\\n]*(?P[A-Za-z]+)\\n[ \\t\\n]*(?P\\d+)[ \\t]*$", - "reference_kind": "fixed_demo", - "reference_rows": [ - { - "id": "1", - "first": "Jon", - "last": "Snow", - "age": "14" - }, - { - "id": "2", - "first": "Cersei", - "last": "Lannister", - "age": "31" - }, - { - "id": "3", - "first": "Jaime", - "last": "Lannister", - "age": "31" - }, - { - "id": "4", - "first": "Arya", - "last": "Stark", - "age": "11" - }, - { - "id": "5", - "first": "Daenerys", - "last": "Targaryen", - "age": "" - } - ] - } + { + "name": "species", + "label": "Species", + "type": "text" }, - "body_requirements": { - "version": 3, - "unit": "region_compact_hexagram_recall", - "minimum_coverage": 0.8, - "minimum_floor": 0.6, - "minimum_region_passes": 1, - "segmentation_method": "sentences", - "regions": [ - { - "id": "body-region-1", - "text": "the grid displays structured rows and columns each row preserves the fields of one record", - "source_text": "The grid displays structured rows and columns.\n\nEach row preserves the fields of one record.", - "segmentation_method": "sentences" - } - ] - }, - "forbidden_any": [ - "Access denied", - "CAPTCHA" - ] - }, - "artifact": "Cookie preferences", - "ok": true, - "expected_task_pass": false - }, - { - "id": "consent-does-not-hide-access-denied", - "category": "structured-extraction", - "root_cause": "cookie_preferences_not_blocked_documentation", - "failure_mode": "false-positive", - "case": { - "url": "https://mui.com/x/react-data-grid/", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "table_row", - "minimum_records": 5, - "stable_text_anchors": [], - "record_identity_fields": [ - "id" - ], - "table_columns": [ - { - "name": "id", - "label": "ID", - "type": "integer", - "minimum": 1 - }, - { - "name": "first", - "label": "First name", - "type": "text" - }, - { - "name": "last", - "label": "Last name", - "type": "text" - }, - { - "name": "age", - "label": "Age", - "type": "integer", - "minimum": 0, - "allow_empty": true - } - ], - "table_scope": { - "root_selector": "[role=grid]", - "root_index": 0, - "row_selector": "[role=row][data-id]", - "cell_selector": "[role=gridcell]:not([data-field=\"__check__\"])", - "start_pattern": "Community version \\(free forever\\)", - "end_pattern": "(?m)^[ \\t#]*Pro version[ \\t]*$", - "plain_row_pattern": "(?P\\d+)\\s+(?P[A-Za-z]+)\\s+(?P[A-Za-z]+)(?:[ \\t]+(?P\\d*))", - "plain_multiline_row_pattern": "(?m)^[ \\t]*(?P\\d+)\\n[ \\t\\n]*(?P[A-Za-z]+)\\n[ \\t\\n]*(?P[A-Za-z]+)\\n[ \\t\\n]*(?P\\d+)[ \\t]*$", - "reference_kind": "fixed_demo", - "reference_rows": [ - { - "id": "1", - "first": "Jon", - "last": "Snow", - "age": "14" - }, - { - "id": "2", - "first": "Cersei", - "last": "Lannister", - "age": "31" - }, - { - "id": "3", - "first": "Jaime", - "last": "Lannister", - "age": "31" - }, - { - "id": "4", - "first": "Arya", - "last": "Stark", - "age": "11" - }, - { - "id": "5", - "first": "Daenerys", - "last": "Targaryen", - "age": "" - } - ] - } + { + "name": "diet", + "label": "Diet", + "type": "text" }, - "body_requirements": { - "version": 3, - "unit": "region_compact_hexagram_recall", - "minimum_coverage": 0.8, - "minimum_floor": 0.6, - "minimum_region_passes": 1, - "segmentation_method": "sentences", - "regions": [ - { - "id": "body-region-1", - "text": "the grid displays structured rows and columns each row preserves the fields of one record", - "source_text": "The grid displays structured rows and columns.\n\nEach row preserves the fields of one record.", - "segmentation_method": "sentences" - } - ] - }, - "forbidden_any": [ - "Access denied", - "CAPTCHA" - ] - }, - "artifact": "Access denied\nThe grid displays structured rows and columns. Each row preserves the fields of one record.\nCommunity version (free forever)\n|ID|First name|Last name|Age|\n|---|---|---|---|\n|1|Jon|Snow|14|\n|2|Cersei|Lannister|31|\n|3|Jaime|Lannister|31|\n|4|Arya|Stark|11|\n|5|Daenerys|Targaryen||\nPro version", - "ok": true, - "expected_task_pass": false - }, - { - "id": "mui-rowheader-full-name-counts", - "category": "structured-extraction", - "root_cause": "documentation_prose_and_grid_are_separate_requirements", - "failure_mode": "false-negative", - "case": { - "url": "https://mui.com/x/react-data-grid/", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "table_row", - "minimum_records": 5, - "stable_text_anchors": [], - "record_identity_fields": [ - "id" - ], - "table_columns": [ - { - "name": "id", - "label": "ID", - "type": "integer", - "minimum": 1 - }, - { - "name": "first", - "label": "First name", - "type": "text" - }, - { - "name": "last", - "label": "Last name", - "type": "text" - }, - { - "name": "age", - "label": "Age", - "type": "integer", - "minimum": 0, - "allow_empty": true - }, - { - "name": "full", - "label": "Full name", - "type": "text" - } - ], - "table_scope": { - "root_selector": "[role=grid]", - "root_index": 0, - "row_selector": "[role=row][data-id]", - "cell_selector": "[role=gridcell]:not([data-field=\"__check__\"]),[role=rowheader]", - "start_pattern": "Community version \\(free forever\\)", - "end_pattern": "(?m)^[ \\t#]*Pro version[ \\t]*$", - "plain_row_pattern": "(?P\\d+)\\s+(?P[A-Za-z]+)\\s+(?P[A-Za-z]+)[ \\t]+(?P\\d*)[ \\t]+(?P[A-Za-z]+ [A-Za-z]+)", - "plain_multiline_row_pattern": "(?m)^[ \\t]*(?P\\d+)\\n[ \\t\\n]*(?P[A-Za-z]+)\\n[ \\t\\n]*(?P[A-Za-z]+)\\n[ \\t\\n]*(?P\\d+)\\n[ \\t\\n]*(?P[A-Za-z]+ [A-Za-z]+)[ \\t]*$", - "reference_kind": "fixed_demo", - "reference_rows": [ - { - "id": "1", - "first": "Jon", - "last": "Snow", - "age": "14", - "full": "Jon Snow" - }, - { - "id": "2", - "first": "Cersei", - "last": "Lannister", - "age": "31", - "full": "Cersei Lannister" - }, - { - "id": "3", - "first": "Jaime", - "last": "Lannister", - "age": "31", - "full": "Jaime Lannister" - }, - { - "id": "4", - "first": "Arya", - "last": "Stark", - "age": "11", - "full": "Arya Stark" - }, - { - "id": "5", - "first": "Daenerys", - "last": "Targaryen", - "age": "", - "full": "Daenerys Targaryen" - } - ] - } - } - }, - "artifact": "
1
Jon
Snow
14
Jon Snow
2
Cersei
Lannister
31
Cersei Lannister
3
Jaime
Lannister
31
Jaime Lannister
4
Arya
Stark
11
Arya Stark
5
Daenerys
Targaryen
Daenerys Targaryen
", - "ok": true, - "expected_task_pass": true - }, - { - "id": "mui-missing-rowheader-full-name", - "category": "structured-extraction", - "root_cause": "documentation_prose_and_grid_are_separate_requirements", - "failure_mode": "false-positive", - "case": { - "url": "https://mui.com/x/react-data-grid/", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "table_row", - "minimum_records": 5, - "stable_text_anchors": [], - "record_identity_fields": [ - "id" - ], - "table_columns": [ - { - "name": "id", - "label": "ID", - "type": "integer", - "minimum": 1 - }, - { - "name": "first", - "label": "First name", - "type": "text" - }, - { - "name": "last", - "label": "Last name", - "type": "text" - }, - { - "name": "age", - "label": "Age", - "type": "integer", - "minimum": 0, - "allow_empty": true - }, - { - "name": "full", - "label": "Full name", - "type": "text" - } - ], - "table_scope": { - "root_selector": "[role=grid]", - "root_index": 0, - "row_selector": "[role=row][data-id]", - "cell_selector": "[role=gridcell]:not([data-field=\"__check__\"]),[role=rowheader]", - "start_pattern": "Community version \\(free forever\\)", - "end_pattern": "(?m)^[ \\t#]*Pro version[ \\t]*$", - "plain_row_pattern": "(?P\\d+)\\s+(?P[A-Za-z]+)\\s+(?P[A-Za-z]+)[ \\t]+(?P\\d*)[ \\t]+(?P[A-Za-z]+ [A-Za-z]+)", - "plain_multiline_row_pattern": "(?m)^[ \\t]*(?P\\d+)\\n[ \\t\\n]*(?P[A-Za-z]+)\\n[ \\t\\n]*(?P[A-Za-z]+)\\n[ \\t\\n]*(?P\\d+)\\n[ \\t\\n]*(?P[A-Za-z]+ [A-Za-z]+)[ \\t]*$", - "reference_kind": "fixed_demo", - "reference_rows": [ - { - "id": "1", - "first": "Jon", - "last": "Snow", - "age": "14", - "full": "Jon Snow" - }, - { - "id": "2", - "first": "Cersei", - "last": "Lannister", - "age": "31", - "full": "Cersei Lannister" - }, - { - "id": "3", - "first": "Jaime", - "last": "Lannister", - "age": "31", - "full": "Jaime Lannister" - }, - { - "id": "4", - "first": "Arya", - "last": "Stark", - "age": "11", - "full": "Arya Stark" - }, - { - "id": "5", - "first": "Daenerys", - "last": "Targaryen", - "age": "", - "full": "Daenerys Targaryen" - } - ] - } - } - }, - "artifact": "
1
Jon
Snow
14
2
Cersei
Lannister
31
Cersei Lannister
3
Jaime
Lannister
31
Jaime Lannister
4
Arya
Stark
11
Arya Stark
5
Daenerys
Targaryen
Daenerys Targaryen
", - "ok": true, - "expected_task_pass": false - }, - { - "id": "html-anchor-entity-named-space", - "category": "format-normalization", - "root_cause": "html_entity_space_is_visible_whitespace", - "failure_mode": "false-negative", - "case": { - "url": "https://example.org/", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "body_text", - "minimum_records": 0, - "stable_text_anchors": [ - { - "id": "heading", - "alternatives": [ - "MUI X Data Grid" - ] - } - ] - } - }, - "artifact": "

MUI X Data Grid

", - "ok": true, - "expected_task_pass": true - }, - { - "id": "html-anchor-entity-numeric-space", - "category": "format-normalization", - "root_cause": "html_entity_space_is_visible_whitespace", - "failure_mode": "false-negative", - "case": { - "url": "https://example.org/", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "body_text", - "minimum_records": 0, - "stable_text_anchors": [ - { - "id": "heading", - "alternatives": [ - "MUI X Data Grid" - ] - } - ] - } - }, - "artifact": "

MUI X Data Grid

", - "ok": true, - "expected_task_pass": true - }, - { - "id": "html-anchor-entity-wrong-heading", - "category": "format-normalization", - "root_cause": "html_entity_space_is_visible_whitespace", - "failure_mode": "false-positive", - "case": { - "url": "https://example.org/", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "body_text", - "minimum_records": 0, - "stable_text_anchors": [ - { - "id": "heading", - "alternatives": [ - "MUI X Data Grid" - ] - } - ] - } - }, - "artifact": "

MUI X Date Picker

", - "ok": true, - "expected_task_pass": false - }, - { - "id": "html-anchor-entity-literal-entity", - "category": "format-normalization", - "root_cause": "html_entity_space_is_visible_whitespace", - "failure_mode": "false-positive", - "case": { - "url": "https://example.org/", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "body_text", - "minimum_records": 0, - "stable_text_anchors": [ - { - "id": "heading", - "alternatives": [ - "MUI X Data Grid" - ] - } - ] - } - }, - "artifact": "

MUI&nbsp;X Data Grid

", - "ok": true, - "expected_task_pass": false - }, - { - "id": "table-source-duplicate-faithful", - "category": "structured-extraction", - "root_cause": "source_identical_table_duplicates_count_once", - "failure_mode": "false-negative", - "case": { - "url": "https://vuetifyjs.com/en/components/data-tables/basics/", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "table_row", - "minimum_records": 2, - "stable_text_anchors": [], - "record_identity_fields": [ - "name", - "species" - ], - "duplicate_policy": "deduplicate_identical", - "table_columns": [ - { - "name": "name", - "label": "Name", - "type": "text" - }, - { - "name": "species", - "label": "Species", - "type": "text" - }, - { - "name": "diet", - "label": "Diet", - "type": "text" - }, - { - "name": "habitat", - "label": "Habitat", - "type": "text" - } - ], - "table_scope": { - "root_selector": "section#usage", - "root_index": 0, - "row_selector": "table tbody tr", - "cell_selector": "td", - "start_pattern": "The standard data table presumes that the entire data set is available locally\\.", - "end_pattern": "(?m)^[ \\t#]*API[ \\t]*$", - "plain_row_pattern": "(?P[A-Z][A-Za-z]+(?: [A-Z][A-Za-z]+)*)\\s+(?P[A-Z][a-z]+ [a-z]+)\\s+(?PHerbivore|Carnivore|Omnivore)\\s+(?P[A-Za-z, ]+)", - "plain_multiline_row_pattern": "(?m)^[ \\t]*(?P[A-Z][A-Za-z]+(?: [A-Z][A-Za-z]+)*)\\n[ \\t\\n]*(?P[A-Z][a-z]+ [a-z]+)\\n[ \\t\\n]*(?PHerbivore|Carnivore|Omnivore)\\n[ \\t\\n]*(?P[A-Za-z, ]+)[ \\t]*$", - "reference_kind": "fixed_demo", - "reference_rows": [ - { - "name": "African Elephant", - "species": "Loxodonta africana", - "diet": "Herbivore", - "habitat": "Savanna, Forests" - }, - { - "name": "Lion", - "species": "Panthera leo", - "diet": "Carnivore", - "habitat": "Savanna, Grassland" - } - ] - } - } - }, - "artifact": "The standard data table presumes that the entire data set is available locally.\n|Name|Species|Diet|Habitat|\n|---|---|---|---|\n|African Elephant|Loxodonta africana|Herbivore|Savanna, Forests|\n|Lion|Panthera leo|Carnivore|Savanna, Grassland|\n|Lion|Panthera leo|Carnivore|Savanna, Grassland|\n", - "ok": true, - "expected_task_pass": true - }, - { - "id": "table-source-duplicate-not-extra-record", - "category": "structured-extraction", - "root_cause": "source_identical_table_duplicates_count_once", - "failure_mode": "false-positive", - "case": { - "url": "https://vuetifyjs.com/en/components/data-tables/basics/", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "table_row", - "minimum_records": 2, - "stable_text_anchors": [], - "record_identity_fields": [ - "name", - "species" - ], - "duplicate_policy": "deduplicate_identical", - "table_columns": [ - { - "name": "name", - "label": "Name", - "type": "text" - }, - { - "name": "species", - "label": "Species", - "type": "text" - }, - { - "name": "diet", - "label": "Diet", - "type": "text" - }, - { - "name": "habitat", - "label": "Habitat", - "type": "text" - } - ], - "table_scope": { - "root_selector": "section#usage", - "root_index": 0, - "row_selector": "table tbody tr", - "cell_selector": "td", - "start_pattern": "The standard data table presumes that the entire data set is available locally\\.", - "end_pattern": "(?m)^[ \\t#]*API[ \\t]*$", - "plain_row_pattern": "(?P[A-Z][A-Za-z]+(?: [A-Z][A-Za-z]+)*)\\s+(?P[A-Z][a-z]+ [a-z]+)\\s+(?PHerbivore|Carnivore|Omnivore)\\s+(?P[A-Za-z, ]+)", - "plain_multiline_row_pattern": "(?m)^[ \\t]*(?P[A-Z][A-Za-z]+(?: [A-Z][A-Za-z]+)*)\\n[ \\t\\n]*(?P[A-Z][a-z]+ [a-z]+)\\n[ \\t\\n]*(?PHerbivore|Carnivore|Omnivore)\\n[ \\t\\n]*(?P[A-Za-z, ]+)[ \\t]*$", - "reference_kind": "fixed_demo", - "reference_rows": [ - { - "name": "African Elephant", - "species": "Loxodonta africana", - "diet": "Herbivore", - "habitat": "Savanna, Forests" - }, - { - "name": "Lion", - "species": "Panthera leo", - "diet": "Carnivore", - "habitat": "Savanna, Grassland" - } - ] - } - } - }, - "artifact": "The standard data table presumes that the entire data set is available locally.\n|Name|Species|Diet|Habitat|\n|---|---|---|---|\n|African Elephant|Loxodonta africana|Herbivore|Savanna, Forests|\n|African Elephant|Loxodonta africana|Herbivore|Savanna, Forests|\n|African Elephant|Loxodonta africana|Herbivore|Savanna, Forests|\n|African Elephant|Loxodonta africana|Herbivore|Savanna, Forests|\n", - "ok": true, - "expected_task_pass": false - }, - { - "id": "table-conflicting-identity-not-deduplicated", - "category": "structured-extraction", - "root_cause": "source_identical_table_duplicates_count_once", - "failure_mode": "false-positive", - "case": { - "url": "https://vuetifyjs.com/en/components/data-tables/basics/", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "table_row", - "minimum_records": 2, - "stable_text_anchors": [], - "record_identity_fields": [ - "name", - "species" - ], - "duplicate_policy": "deduplicate_identical", - "table_columns": [ - { - "name": "name", - "label": "Name", - "type": "text" - }, - { - "name": "species", - "label": "Species", - "type": "text" - }, - { - "name": "diet", - "label": "Diet", - "type": "text" - }, - { - "name": "habitat", - "label": "Habitat", - "type": "text" - } - ], - "table_scope": { - "root_selector": "section#usage", - "root_index": 0, - "row_selector": "table tbody tr", - "cell_selector": "td", - "start_pattern": "The standard data table presumes that the entire data set is available locally\\.", - "end_pattern": "(?m)^[ \\t#]*API[ \\t]*$", - "plain_row_pattern": "(?P[A-Z][A-Za-z]+(?: [A-Z][A-Za-z]+)*)\\s+(?P[A-Z][a-z]+ [a-z]+)\\s+(?PHerbivore|Carnivore|Omnivore)\\s+(?P[A-Za-z, ]+)", - "plain_multiline_row_pattern": "(?m)^[ \\t]*(?P[A-Z][A-Za-z]+(?: [A-Z][A-Za-z]+)*)\\n[ \\t\\n]*(?P[A-Z][a-z]+ [a-z]+)\\n[ \\t\\n]*(?PHerbivore|Carnivore|Omnivore)\\n[ \\t\\n]*(?P[A-Za-z, ]+)[ \\t]*$" - } + { + "name": "habitat", + "label": "Habitat", + "type": "text" } - }, - "artifact": "The standard data table presumes that the entire data set is available locally.\n|Name|Species|Diet|Habitat|\n|---|---|---|---|\n|African Elephant|Loxodonta africana|Herbivore|Savanna, Forests|\n|Lion|Panthera leo|Carnivore|Savanna, Grassland|\n|Lion|Panthera leo|Herbivore|Savanna, Grassland|\n", - "ok": true, - "expected_task_pass": false - }, - { - "id": "usage-demo-body-and-owned-rows", - "category": "structured-extraction", - "root_cause": "usage_demo_rows_not_api_component_table", - "failure_mode": "false-negative", - "case": { - "url": "https://vuetifyjs.com/en/components/data-tables/basics/", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "table_row", - "minimum_records": 2, - "stable_text_anchors": [], - "record_identity_fields": [ - "name", - "species" - ], - "duplicate_policy": "deduplicate_identical", - "table_columns": [ - { - "name": "name", - "label": "Name", - "type": "text" - }, - { - "name": "species", - "label": "Species", - "type": "text" - }, - { - "name": "diet", - "label": "Diet", - "type": "text" - }, - { - "name": "habitat", - "label": "Habitat", - "type": "text" - } - ], - "table_scope": { - "root_selector": "section#usage", - "root_index": 0, - "row_selector": "table tbody tr", - "cell_selector": "td", - "start_pattern": "The standard data table presumes that the entire data set is available locally\\.", - "end_pattern": "(?m)^[ \\t#]*API[ \\t]*$", - "plain_row_pattern": "(?P[A-Z][A-Za-z]+(?: [A-Z][A-Za-z]+)*)\\s+(?P[A-Z][a-z]+ [a-z]+)\\s+(?PHerbivore|Carnivore|Omnivore)\\s+(?P[A-Za-z, ]+)", - "plain_multiline_row_pattern": "(?m)^[ \\t]*(?P[A-Z][A-Za-z]+(?: [A-Z][A-Za-z]+)*)\\n[ \\t\\n]*(?P[A-Z][a-z]+ [a-z]+)\\n[ \\t\\n]*(?PHerbivore|Carnivore|Omnivore)\\n[ \\t\\n]*(?P[A-Za-z, ]+)[ \\t]*$", - "reference_kind": "fixed_demo", - "reference_rows": [ - { - "name": "African Elephant", - "species": "Loxodonta africana", - "diet": "Herbivore", - "habitat": "Savanna, Forests" - }, - { - "name": "Lion", - "species": "Panthera leo", - "diet": "Carnivore", - "habitat": "Savanna, Grassland" - } - ] - } + ], + "table_scope": { + "root_selector": "section#usage", + "root_index": 0, + "row_selector": "table tbody tr", + "cell_selector": "td", + "start_pattern": "The standard data table presumes that the entire data set is available locally\\.", + "end_pattern": "(?m)^[ \\t#]*API[ \\t]*$", + "plain_row_pattern": "(?P[A-Z][A-Za-z]+(?: [A-Z][A-Za-z]+)*)\\s+(?P[A-Z][a-z]+ [a-z]+)\\s+(?PHerbivore|Carnivore|Omnivore)\\s+(?P[A-Za-z, ]+)", + "plain_multiline_row_pattern": "(?m)^[ \\t]*(?P[A-Z][A-Za-z]+(?: [A-Z][A-Za-z]+)*)\\n[ \\t\\n]*(?P[A-Z][a-z]+ [a-z]+)\\n[ \\t\\n]*(?PHerbivore|Carnivore|Omnivore)\\n[ \\t\\n]*(?P[A-Za-z, ]+)[ \\t]*$", + "reference_kind": "fixed_demo", + "reference_rows": [ + { + "name": "African Elephant", + "species": "Loxodonta africana", + "diet": "Herbivore", + "habitat": "Savanna, Forests" + }, + { + "name": "Lion", + "species": "Panthera leo", + "diet": "Carnivore", + "habitat": "Savanna, Grassland" + } + ] + } + } + }, + "artifact": "The standard data table presumes that the entire data set is available locally.\n|Name|Species|Diet|Habitat|\n|---|---|---|---|\n|African Elephant|Loxodonta africana|Herbivore|Savanna, Forests|\n|African Elephant|Loxodonta africana|Herbivore|Savanna, Forests|\n|African Elephant|Loxodonta africana|Herbivore|Savanna, Forests|\n|African Elephant|Loxodonta africana|Herbivore|Savanna, Forests|\n", + "ok": true, + "expected_task_pass": false + }, + { + "id": "table-conflicting-identity-not-deduplicated", + "category": "structured-extraction", + "root_cause": "source_identical_table_duplicates_count_once", + "failure_mode": "false-positive", + "case": { + "url": "https://vuetifyjs.com/en/components/data-tables/basics/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 2, + "stable_text_anchors": [], + "record_identity_fields": [ + "name", + "species" + ], + "duplicate_policy": "deduplicate_identical", + "table_columns": [ + { + "name": "name", + "label": "Name", + "type": "text" }, - "body_requirements": { - "version": 3, - "unit": "region_compact_hexagram_recall", - "minimum_coverage": 0.8, - "minimum_floor": 0.6, - "minimum_region_passes": 1, - "segmentation_method": "balanced_tokens_fallback", - "regions": [ - { - "id": "body-region-1", - "text": "sorting pagination and filtering is supported and done internally by the component itself", - "source_text": "sorting pagination and filtering is supported and done internally by the component itself", - "segmentation_method": "balanced_tokens_fallback" - } - ] - } - }, - "artifact": "Sorting, pagination, and filtering is supported and done internally by the component itself.\nThe standard data table presumes that the entire data set is available locally.\n|Name|Species|Diet|Habitat|\n|---|---|---|---|\n|African Elephant|Loxodonta africana|Herbivore|Savanna, Forests|\n|Lion|Panthera leo|Carnivore|Savanna, Grassland|\n", - "ok": true, - "expected_task_pass": true - }, - { - "id": "usage-prose-alone-not-enough", - "category": "structured-extraction", - "root_cause": "usage_demo_rows_not_api_component_table", - "failure_mode": "false-positive", - "case": { - "url": "https://vuetifyjs.com/en/components/data-tables/basics/", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "table_row", - "minimum_records": 2, - "stable_text_anchors": [], - "record_identity_fields": [ - "name", - "species" - ], - "duplicate_policy": "deduplicate_identical", - "table_columns": [ - { - "name": "name", - "label": "Name", - "type": "text" - }, - { - "name": "species", - "label": "Species", - "type": "text" - }, - { - "name": "diet", - "label": "Diet", - "type": "text" - }, - { - "name": "habitat", - "label": "Habitat", - "type": "text" - } - ], - "table_scope": { - "root_selector": "section#usage", - "root_index": 0, - "row_selector": "table tbody tr", - "cell_selector": "td", - "start_pattern": "The standard data table presumes that the entire data set is available locally\\.", - "end_pattern": "(?m)^[ \\t#]*API[ \\t]*$", - "plain_row_pattern": "(?P[A-Z][A-Za-z]+(?: [A-Z][A-Za-z]+)*)\\s+(?P[A-Z][a-z]+ [a-z]+)\\s+(?PHerbivore|Carnivore|Omnivore)\\s+(?P[A-Za-z, ]+)", - "plain_multiline_row_pattern": "(?m)^[ \\t]*(?P[A-Z][A-Za-z]+(?: [A-Z][A-Za-z]+)*)\\n[ \\t\\n]*(?P[A-Z][a-z]+ [a-z]+)\\n[ \\t\\n]*(?PHerbivore|Carnivore|Omnivore)\\n[ \\t\\n]*(?P[A-Za-z, ]+)[ \\t]*$", - "reference_kind": "fixed_demo", - "reference_rows": [ - { - "name": "African Elephant", - "species": "Loxodonta africana", - "diet": "Herbivore", - "habitat": "Savanna, Forests" - }, - { - "name": "Lion", - "species": "Panthera leo", - "diet": "Carnivore", - "habitat": "Savanna, Grassland" - } - ] - } + { + "name": "species", + "label": "Species", + "type": "text" }, - "body_requirements": { - "version": 3, - "unit": "region_compact_hexagram_recall", - "minimum_coverage": 0.8, - "minimum_floor": 0.6, - "minimum_region_passes": 1, - "segmentation_method": "balanced_tokens_fallback", - "regions": [ - { - "id": "body-region-1", - "text": "sorting pagination and filtering is supported and done internally by the component itself", - "source_text": "sorting pagination and filtering is supported and done internally by the component itself", - "segmentation_method": "balanced_tokens_fallback" - } - ] - } - }, - "artifact": "Sorting, pagination, and filtering is supported and done internally by the component itself.\nThe standard data table presumes that the entire data set is available locally.", - "ok": true, - "expected_task_pass": false - }, - { - "id": "usage-api-table-not-demo", - "category": "structured-extraction", - "root_cause": "usage_demo_rows_not_api_component_table", - "failure_mode": "false-positive", - "case": { - "url": "https://vuetifyjs.com/en/components/data-tables/basics/", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "table_row", - "minimum_records": 2, - "stable_text_anchors": [], - "record_identity_fields": [ - "name", - "species" - ], - "duplicate_policy": "deduplicate_identical", - "table_columns": [ - { - "name": "name", - "label": "Name", - "type": "text" - }, - { - "name": "species", - "label": "Species", - "type": "text" - }, - { - "name": "diet", - "label": "Diet", - "type": "text" - }, - { - "name": "habitat", - "label": "Habitat", - "type": "text" - } - ], - "table_scope": { - "root_selector": "section#usage", - "root_index": 0, - "row_selector": "table tbody tr", - "cell_selector": "td", - "start_pattern": "The standard data table presumes that the entire data set is available locally\\.", - "end_pattern": "(?m)^[ \\t#]*API[ \\t]*$", - "plain_row_pattern": "(?P[A-Z][A-Za-z]+(?: [A-Z][A-Za-z]+)*)\\s+(?P[A-Z][a-z]+ [a-z]+)\\s+(?PHerbivore|Carnivore|Omnivore)\\s+(?P[A-Za-z, ]+)", - "plain_multiline_row_pattern": "(?m)^[ \\t]*(?P[A-Z][A-Za-z]+(?: [A-Z][A-Za-z]+)*)\\n[ \\t\\n]*(?P[A-Z][a-z]+ [a-z]+)\\n[ \\t\\n]*(?PHerbivore|Carnivore|Omnivore)\\n[ \\t\\n]*(?P[A-Za-z, ]+)[ \\t]*$", - "reference_kind": "fixed_demo", - "reference_rows": [ - { - "name": "African Elephant", - "species": "Loxodonta africana", - "diet": "Herbivore", - "habitat": "Savanna, Forests" - }, - { - "name": "Lion", - "species": "Panthera leo", - "diet": "Carnivore", - "habitat": "Savanna, Grassland" - } - ] - } + { + "name": "diet", + "label": "Diet", + "type": "text" }, - "body_requirements": { - "version": 3, - "unit": "region_compact_hexagram_recall", - "minimum_coverage": 0.8, - "minimum_floor": 0.6, - "minimum_region_passes": 1, - "segmentation_method": "balanced_tokens_fallback", - "regions": [ - { - "id": "body-region-1", - "text": "sorting pagination and filtering is supported and done internally by the component itself", - "source_text": "sorting pagination and filtering is supported and done internally by the component itself", - "segmentation_method": "balanced_tokens_fallback" - } - ] + { + "name": "habitat", + "label": "Habitat", + "type": "text" } - }, - "artifact": "Sorting, pagination, and filtering is supported and done internally by the component itself.\nThe standard data table presumes that the entire data set is available locally.\nAPI\n\n|Name|Species|Diet|Habitat|\n|---|---|---|---|\n|African Elephant|Loxodonta africana|Herbivore|Savanna, Forests|\n|Lion|Panthera leo|Carnivore|Savanna, Grassland|\n", - "ok": true, - "expected_task_pass": false - }, - { - "id": "usage-mixed-fields-rejected", - "category": "structured-extraction", - "root_cause": "usage_demo_rows_not_api_component_table", - "failure_mode": "false-positive", - "case": { - "url": "https://vuetifyjs.com/en/components/data-tables/basics/", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "table_row", - "minimum_records": 2, - "stable_text_anchors": [], - "record_identity_fields": [ - "name", - "species" - ], - "duplicate_policy": "deduplicate_identical", - "table_columns": [ - { - "name": "name", - "label": "Name", - "type": "text" - }, - { - "name": "species", - "label": "Species", - "type": "text" - }, - { - "name": "diet", - "label": "Diet", - "type": "text" - }, - { - "name": "habitat", - "label": "Habitat", - "type": "text" - } - ], - "table_scope": { - "root_selector": "section#usage", - "root_index": 0, - "row_selector": "table tbody tr", - "cell_selector": "td", - "start_pattern": "The standard data table presumes that the entire data set is available locally\\.", - "end_pattern": "(?m)^[ \\t#]*API[ \\t]*$", - "plain_row_pattern": "(?P[A-Z][A-Za-z]+(?: [A-Z][A-Za-z]+)*)\\s+(?P[A-Z][a-z]+ [a-z]+)\\s+(?PHerbivore|Carnivore|Omnivore)\\s+(?P[A-Za-z, ]+)", - "plain_multiline_row_pattern": "(?m)^[ \\t]*(?P[A-Z][A-Za-z]+(?: [A-Z][A-Za-z]+)*)\\n[ \\t\\n]*(?P[A-Z][a-z]+ [a-z]+)\\n[ \\t\\n]*(?PHerbivore|Carnivore|Omnivore)\\n[ \\t\\n]*(?P[A-Za-z, ]+)[ \\t]*$", - "reference_kind": "fixed_demo", - "reference_rows": [ - { - "name": "African Elephant", - "species": "Loxodonta africana", - "diet": "Herbivore", - "habitat": "Savanna, Forests" - }, - { - "name": "Lion", - "species": "Panthera leo", - "diet": "Carnivore", - "habitat": "Savanna, Grassland" - } - ] - } + ], + "table_scope": { + "root_selector": "section#usage", + "root_index": 0, + "row_selector": "table tbody tr", + "cell_selector": "td", + "start_pattern": "The standard data table presumes that the entire data set is available locally\\.", + "end_pattern": "(?m)^[ \\t#]*API[ \\t]*$", + "plain_row_pattern": "(?P[A-Z][A-Za-z]+(?: [A-Z][A-Za-z]+)*)\\s+(?P[A-Z][a-z]+ [a-z]+)\\s+(?PHerbivore|Carnivore|Omnivore)\\s+(?P[A-Za-z, ]+)", + "plain_multiline_row_pattern": "(?m)^[ \\t]*(?P[A-Z][A-Za-z]+(?: [A-Z][A-Za-z]+)*)\\n[ \\t\\n]*(?P[A-Z][a-z]+ [a-z]+)\\n[ \\t\\n]*(?PHerbivore|Carnivore|Omnivore)\\n[ \\t\\n]*(?P[A-Za-z, ]+)[ \\t]*$" + } + } + }, + "artifact": "The standard data table presumes that the entire data set is available locally.\n|Name|Species|Diet|Habitat|\n|---|---|---|---|\n|African Elephant|Loxodonta africana|Herbivore|Savanna, Forests|\n|Lion|Panthera leo|Carnivore|Savanna, Grassland|\n|Lion|Panthera leo|Herbivore|Savanna, Grassland|\n", + "ok": true, + "expected_task_pass": false + }, + { + "id": "usage-demo-body-and-owned-rows", + "category": "structured-extraction", + "root_cause": "usage_demo_rows_not_api_component_table", + "failure_mode": "false-negative", + "case": { + "url": "https://vuetifyjs.com/en/components/data-tables/basics/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 2, + "stable_text_anchors": [], + "record_identity_fields": [ + "name", + "species" + ], + "duplicate_policy": "deduplicate_identical", + "table_columns": [ + { + "name": "name", + "label": "Name", + "type": "text" }, - "body_requirements": { - "version": 3, - "unit": "region_compact_hexagram_recall", - "minimum_coverage": 0.8, - "minimum_floor": 0.6, - "minimum_region_passes": 1, - "segmentation_method": "balanced_tokens_fallback", - "regions": [ - { - "id": "body-region-1", - "text": "sorting pagination and filtering is supported and done internally by the component itself", - "source_text": "sorting pagination and filtering is supported and done internally by the component itself", - "segmentation_method": "balanced_tokens_fallback" - } - ] - } - }, - "artifact": "Sorting, pagination, and filtering is supported and done internally by the component itself.\nThe standard data table presumes that the entire data set is available locally.\n|Name|Species|Diet|Habitat|\n|---|---|---|---|\n|African Elephant|Loxodonta africana|Herbivore|Savanna, Forests|\n|Lion|Loxodonta africana|Carnivore|Savanna, Grassland|\n", - "ok": true, - "expected_task_pass": false - }, - { - "id": "amazonmx-owned-card-fresh-products", - "category": "structured-extraction", - "root_cause": "same_card_title_detail_pairing", - "failure_mode": "false-negative", - "case": { - "url": "https://www.amazon.com.mx/gp/bestsellers/electronics/", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 2, - "hosts": [ - "www.amazon.com.mx" - ], - "entry_path": "(?:/[^/]+)?/dp/[A-Z0-9]{10}(?:/[^?#]*)?", - "stable_text_anchors": [], - "entry_identity_pattern": "/dp/([A-Z0-9]{10})(?:/|$)", - "name_validation": "nonempty", - "entry_scope": { - "html_selector": "#gridItemRoot", - "title_selector": "a[role=link] [class*=\"line-clamp\"]", - "link_selector": "a[role=link]:has([class*=\"line-clamp\"])" - } - } - }, - "artifact": "", - "ok": true, - "expected_task_pass": true - }, - { - "id": "amazonmx-owned-card-image-only", - "category": "structured-extraction", - "root_cause": "same_card_title_detail_pairing", - "failure_mode": "false-positive", - "case": { - "url": "https://www.amazon.com.mx/gp/bestsellers/electronics/", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 2, - "hosts": [ - "www.amazon.com.mx" - ], - "entry_path": "(?:/[^/]+)?/dp/[A-Z0-9]{10}(?:/[^?#]*)?", - "stable_text_anchors": [], - "entry_identity_pattern": "/dp/([A-Z0-9]{10})(?:/|$)", - "name_validation": "nonempty", - "entry_scope": { - "html_selector": "#gridItemRoot", - "title_selector": "a[role=link] [class*=\"line-clamp\"]", - "link_selector": "a[role=link]:has([class*=\"line-clamp\"])" - } - } - }, - "artifact": "
\"Producto
", - "ok": true, - "expected_task_pass": false - }, - { - "id": "amazonmx-owned-card-duplicate-asin", - "category": "structured-extraction", - "root_cause": "same_card_title_detail_pairing", - "failure_mode": "false-positive", - "case": { - "url": "https://www.amazon.com.mx/gp/bestsellers/electronics/", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 2, - "hosts": [ - "www.amazon.com.mx" - ], - "entry_path": "(?:/[^/]+)?/dp/[A-Z0-9]{10}(?:/[^?#]*)?", - "stable_text_anchors": [], - "entry_identity_pattern": "/dp/([A-Z0-9]{10})(?:/|$)", - "name_validation": "nonempty", - "entry_scope": { - "html_selector": "#gridItemRoot", - "title_selector": "a[role=link] [class*=\"line-clamp\"]", - "link_selector": "a[role=link]:has([class*=\"line-clamp\"])" - } - } - }, - "artifact": "", - "ok": true, - "expected_task_pass": false - }, - { - "id": "amazonmx-owned-card-wrong-host", - "category": "structured-extraction", - "root_cause": "same_card_title_detail_pairing", - "failure_mode": "false-positive", - "case": { - "url": "https://www.amazon.com.mx/gp/bestsellers/electronics/", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 2, - "hosts": [ - "www.amazon.com.mx" - ], - "entry_path": "(?:/[^/]+)?/dp/[A-Z0-9]{10}(?:/[^?#]*)?", - "stable_text_anchors": [], - "entry_identity_pattern": "/dp/([A-Z0-9]{10})(?:/|$)", - "name_validation": "nonempty", - "entry_scope": { - "html_selector": "#gridItemRoot", - "title_selector": "a[role=link] [class*=\"line-clamp\"]", - "link_selector": "a[role=link]:has([class*=\"line-clamp\"])" - } - } - }, - "artifact": "", - "ok": true, - "expected_task_pass": false - }, - { - "id": "falabella-product-scope-owned-title-detail", - "category": "structured-extraction", - "root_cause": "product_records_not_shop_navigation_or_empty_destinations", - "failure_mode": "false-negative", - "case": { - "url": "https://www.falabella.com/falabella-cl/category/cat3091/Equipos-de-musica-y-karaokes", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 1, - "hosts": [ - "www.falabella.com" - ], - "entry_path": "/falabella-cl/product/[0-9]+/.+", - "stable_text_anchors": [], - "name_validation": "nonempty", - "entry_identity_pattern": "/product/([0-9]+)(?:/|$)", - "entry_scope": { - "html_selector": "#testId-searchResults-products .grid-pod", - "title_selector": ".pod-subTitle", - "link_selector": "a[data-pod]" - } - } - }, - "artifact": "", - "ok": true, - "expected_task_pass": true - }, - { - "id": "falabella-product-scope-no-valid-product-destination", - "category": "structured-extraction", - "root_cause": "product_records_not_shop_navigation_or_empty_destinations", - "failure_mode": "false-positive", - "case": { - "url": "https://www.falabella.com/falabella-cl/category/cat3091/Equipos-de-musica-y-karaokes", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 1, - "hosts": [ - "www.falabella.com" - ], - "entry_path": "/falabella-cl/product/[0-9]+/.+", - "stable_text_anchors": [], - "name_validation": "nonempty", - "entry_identity_pattern": "/product/([0-9]+)(?:/|$)", - "entry_scope": { - "html_selector": "#testId-searchResults-products .grid-pod", - "title_selector": ".pod-subTitle", - "link_selector": "a[data-pod]" - } - } - }, - "artifact": "", - "ok": true, - "expected_task_pass": false - }, - { - "id": "falabella-product-scope-outside-product-card", - "category": "structured-extraction", - "root_cause": "product_records_not_shop_navigation_or_empty_destinations", - "failure_mode": "false-positive", - "case": { - "url": "https://www.falabella.com/falabella-cl/category/cat3091/Equipos-de-musica-y-karaokes", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 1, - "hosts": [ - "www.falabella.com" - ], - "entry_path": "/falabella-cl/product/[0-9]+/.+", - "stable_text_anchors": [], - "name_validation": "nonempty", - "entry_identity_pattern": "/product/([0-9]+)(?:/|$)", - "entry_scope": { - "html_selector": "#testId-searchResults-products .grid-pod", - "title_selector": ".pod-subTitle", - "link_selector": "a[data-pod]" - } - } - }, - "artifact": "", - "ok": true, - "expected_task_pass": false - }, - { - "id": "falabella-product-scope-missing-title", - "category": "structured-extraction", - "root_cause": "product_records_not_shop_navigation_or_empty_destinations", - "failure_mode": "false-positive", - "case": { - "url": "https://www.falabella.com/falabella-cl/category/cat3091/Equipos-de-musica-y-karaokes", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 1, - "hosts": [ - "www.falabella.com" - ], - "entry_path": "/falabella-cl/product/[0-9]+/.+", - "stable_text_anchors": [], - "name_validation": "nonempty", - "entry_identity_pattern": "/product/([0-9]+)(?:/|$)", - "entry_scope": { - "html_selector": "#testId-searchResults-products .grid-pod", - "title_selector": ".pod-subTitle", - "link_selector": "a[data-pod]" - } - } - }, - "artifact": "
", - "ok": true, - "expected_task_pass": false - }, - { - "id": "lotteon-product-scope-owned-title-detail", - "category": "structured-extraction", - "root_cause": "product_records_not_shop_navigation_or_empty_destinations", - "failure_mode": "false-negative", - "case": { - "url": "https://www.lotteon.com/p/display/main/lotteon", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 1, - "hosts": [ - "www.lotteon.com" - ], - "entry_path": "/p/product/[A-Za-z0-9]+", - "stable_text_anchors": [], - "name_validation": "nonempty", - "entry_identity_pattern": "/p/product/([A-Za-z0-9]+)(?:/|$)", - "entry_scope": { - "html_selector": ".c-product-card", - "title_selector": ".c-product-title__title", - "link_selector": "a.c-product-card__url" - } - } - }, - "artifact": "

새 상품 이름

", - "ok": true, - "expected_task_pass": true - }, - { - "id": "lotteon-product-scope-no-valid-product-destination", - "category": "structured-extraction", - "root_cause": "product_records_not_shop_navigation_or_empty_destinations", - "failure_mode": "false-positive", - "case": { - "url": "https://www.lotteon.com/p/display/main/lotteon", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 1, - "hosts": [ - "www.lotteon.com" - ], - "entry_path": "/p/product/[A-Za-z0-9]+", - "stable_text_anchors": [], - "name_validation": "nonempty", - "entry_identity_pattern": "/p/product/([A-Za-z0-9]+)(?:/|$)", - "entry_scope": { - "html_selector": ".c-product-card", - "title_selector": ".c-product-title__title", - "link_selector": "a.c-product-card__url" - } - } - }, - "artifact": "

새 상품 이름

", - "ok": true, - "expected_task_pass": false - }, - { - "id": "lotteon-product-scope-outside-product-card", - "category": "structured-extraction", - "root_cause": "product_records_not_shop_navigation_or_empty_destinations", - "failure_mode": "false-positive", - "case": { - "url": "https://www.lotteon.com/p/display/main/lotteon", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 1, - "hosts": [ - "www.lotteon.com" - ], - "entry_path": "/p/product/[A-Za-z0-9]+", - "stable_text_anchors": [], - "name_validation": "nonempty", - "entry_identity_pattern": "/p/product/([A-Za-z0-9]+)(?:/|$)", - "entry_scope": { - "html_selector": ".c-product-card", - "title_selector": ".c-product-title__title", - "link_selector": "a.c-product-card__url" - } - } - }, - "artifact": "

새 상품 이름

", - "ok": true, - "expected_task_pass": false - }, - { - "id": "lotteon-product-scope-missing-title", - "category": "structured-extraction", - "root_cause": "product_records_not_shop_navigation_or_empty_destinations", - "failure_mode": "false-positive", - "case": { - "url": "https://www.lotteon.com/p/display/main/lotteon", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 1, - "hosts": [ - "www.lotteon.com" - ], - "entry_path": "/p/product/[A-Za-z0-9]+", - "stable_text_anchors": [], - "name_validation": "nonempty", - "entry_identity_pattern": "/p/product/([A-Za-z0-9]+)(?:/|$)", - "entry_scope": { - "html_selector": ".c-product-card", - "title_selector": ".c-product-title__title", - "link_selector": "a.c-product-card__url" - } - } - }, - "artifact": "

", - "ok": true, - "expected_task_pass": false - }, - { - "id": "dynamic-088-terra-home-new-stories", - "category": "structured-extraction", - "root_cause": "news_detail_routes_not_category_navigation", - "failure_mode": "false-negative", - "case": { - "url": "https://www.terra.com.br/", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 2, - "hosts": [ - "www.terra.com.br" - ], - "entry_path": "/.+,[A-Za-z0-9]+\\.html", - "stable_text_anchors": [], - "name_validation": "nonempty", - "entry_scope": { - "html_selector": "a.card-news__text--title,a.card-news__list-item--link,a.t360-carousel-news__card-body-title,a.card-v-mt--embed" - }, - "entry_identity_pattern": ",([A-Za-z0-9]+)\\.html$" - } - }, - "artifact": "[Notícia nova A](https://www.terra.com.br/noticias/mundo/noticia-a,abc123.html)\n[Notícia nova B](https://www.terra.com.br/esportes/noticia-b,def456.html)", - "ok": true, - "expected_task_pass": true - }, - { - "id": "dynamic-088-terra-home-navigation-only", - "category": "structured-extraction", - "root_cause": "news_detail_routes_not_category_navigation", - "failure_mode": "false-positive", - "case": { - "url": "https://www.terra.com.br/", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 2, - "hosts": [ - "www.terra.com.br" - ], - "entry_path": "/.+,[A-Za-z0-9]+\\.html", - "stable_text_anchors": [], - "name_validation": "nonempty", - "entry_scope": { - "html_selector": "a.card-news__text--title,a.card-news__list-item--link,a.t360-carousel-news__card-body-title,a.card-v-mt--embed" - }, - "entry_identity_pattern": ",([A-Za-z0-9]+)\\.html$" - } - }, - "artifact": "[Notícias](https://www.terra.com.br/noticias/)\n[Mundo](https://www.terra.com.br/noticias/mundo/)", - "ok": true, - "expected_task_pass": false - }, - { - "id": "dynamic-088-terra-home-duplicate-story", - "category": "structured-extraction", - "root_cause": "news_detail_routes_not_category_navigation", - "failure_mode": "false-positive", - "case": { - "url": "https://www.terra.com.br/", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 2, - "hosts": [ - "www.terra.com.br" - ], - "entry_path": "/.+,[A-Za-z0-9]+\\.html", - "stable_text_anchors": [], - "name_validation": "nonempty", - "entry_scope": { - "html_selector": "a.card-news__text--title,a.card-news__list-item--link,a.t360-carousel-news__card-body-title,a.card-v-mt--embed" - }, - "entry_identity_pattern": ",([A-Za-z0-9]+)\\.html$" - } - }, - "artifact": "[Notícia nova A](https://www.terra.com.br/noticias/mundo/noticia-a,abc123.html)\n[Notícia nova B](https://www.terra.com.br/esportes/outro-titulo,abc123.html)", - "ok": true, - "expected_task_pass": false - }, - { - "id": "dynamic-093-pt-real-terra-noticias-new-stories", - "category": "structured-extraction", - "root_cause": "news_detail_routes_not_category_navigation", - "failure_mode": "false-negative", - "case": { - "url": "https://www.terra.com.br/noticias/", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 2, - "hosts": [ - "www.terra.com.br" - ], - "entry_path": "/.+,[A-Za-z0-9]+\\.html", - "stable_text_anchors": [], - "name_validation": "nonempty", - "entry_scope": { - "html_selector": "a.card-news__text--title,a.card-news__list-item--link,a.t360-carousel-news__card-body-title,a.card-v-mt--embed" - }, - "entry_identity_pattern": ",([A-Za-z0-9]+)\\.html$" - } - }, - "artifact": "[Notícia nova A](https://www.terra.com.br/noticias/mundo/noticia-a,abc123.html)\n[Notícia nova B](https://www.terra.com.br/esportes/noticia-b,def456.html)", - "ok": true, - "expected_task_pass": true - }, - { - "id": "dynamic-093-pt-real-terra-noticias-navigation-only", - "category": "structured-extraction", - "root_cause": "news_detail_routes_not_category_navigation", - "failure_mode": "false-positive", - "case": { - "url": "https://www.terra.com.br/noticias/", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 2, - "hosts": [ - "www.terra.com.br" - ], - "entry_path": "/.+,[A-Za-z0-9]+\\.html", - "stable_text_anchors": [], - "name_validation": "nonempty", - "entry_scope": { - "html_selector": "a.card-news__text--title,a.card-news__list-item--link,a.t360-carousel-news__card-body-title,a.card-v-mt--embed" - }, - "entry_identity_pattern": ",([A-Za-z0-9]+)\\.html$" - } - }, - "artifact": "[Notícias](https://www.terra.com.br/noticias/)\n[Mundo](https://www.terra.com.br/noticias/mundo/)", - "ok": true, - "expected_task_pass": false - }, - { - "id": "dynamic-093-pt-real-terra-noticias-duplicate-story", - "category": "structured-extraction", - "root_cause": "news_detail_routes_not_category_navigation", - "failure_mode": "false-positive", - "case": { - "url": "https://www.terra.com.br/noticias/", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 2, - "hosts": [ - "www.terra.com.br" - ], - "entry_path": "/.+,[A-Za-z0-9]+\\.html", - "stable_text_anchors": [], - "name_validation": "nonempty", - "entry_scope": { - "html_selector": "a.card-news__text--title,a.card-news__list-item--link,a.t360-carousel-news__card-body-title,a.card-v-mt--embed" - }, - "entry_identity_pattern": ",([A-Za-z0-9]+)\\.html$" - } - }, - "artifact": "[Notícia nova A](https://www.terra.com.br/noticias/mundo/noticia-a,abc123.html)\n[Notícia nova B](https://www.terra.com.br/esportes/outro-titulo,abc123.html)", - "ok": true, - "expected_task_pass": false - }, - { - "id": "dynamic-096-ar-real-cnn-world-new-stories", - "category": "structured-extraction", - "root_cause": "news_detail_routes_not_category_navigation", - "failure_mode": "false-negative", - "case": { - "url": "https://arabic.cnn.com/world", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 2, - "hosts": [ - "arabic.cnn.com" - ], - "entry_path": "/[^/]+/(?:article|video)/[0-9]{4}/[0-9]{2}/[0-9]{2}/[^/]+", - "stable_text_anchors": [], - "name_validation": "nonempty", - "entry_scope": { - "html_selector": ".container__item:has(.container__headline-text)", - "title_selector": ".container__headline-text", - "link_selector": "a:has(.container__headline-text)" - } - } - }, - "artifact": "[عنوان جديد ألف](/world/article/2026/09/20/new-story-a)\n[عنوان جديد باء](/middle-east/video/2026/09/20/new-story-b)", - "ok": true, - "expected_task_pass": true - }, - { - "id": "dynamic-096-ar-real-cnn-world-navigation-only", - "category": "structured-extraction", - "root_cause": "news_detail_routes_not_category_navigation", - "failure_mode": "false-positive", - "case": { - "url": "https://arabic.cnn.com/world", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 2, - "hosts": [ - "arabic.cnn.com" - ], - "entry_path": "/[^/]+/(?:article|video)/[0-9]{4}/[0-9]{2}/[0-9]{2}/[^/]+", - "stable_text_anchors": [], - "name_validation": "nonempty", - "entry_scope": { - "html_selector": ".container__item:has(.container__headline-text)", - "title_selector": ".container__headline-text", - "link_selector": "a:has(.container__headline-text)" - } - } - }, - "artifact": "[أمريكا](/world/us)\n[روسيا](/world/russia)", - "ok": true, - "expected_task_pass": false - }, - { - "id": "dynamic-096-ar-real-cnn-world-duplicate-story", - "category": "structured-extraction", - "root_cause": "news_detail_routes_not_category_navigation", - "failure_mode": "false-positive", - "case": { - "url": "https://arabic.cnn.com/world", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 2, - "hosts": [ - "arabic.cnn.com" - ], - "entry_path": "/[^/]+/(?:article|video)/[0-9]{4}/[0-9]{2}/[0-9]{2}/[^/]+", - "stable_text_anchors": [], - "name_validation": "nonempty", - "entry_scope": { - "html_selector": ".container__item:has(.container__headline-text)", - "title_selector": ".container__headline-text", - "link_selector": "a:has(.container__headline-text)" - } - } - }, - "artifact": "[عنوان جديد ألف](/world/article/2026/09/20/new-story-a)\n[عنوان جديد باء](/world/article/2026/09/20/new-story-a)", - "ok": true, - "expected_task_pass": false - }, - { - "id": "cnnbrasil-qualified-routes-new-election-and-blog", - "category": "structured-extraction", - "root_cause": "news_detail_routes_not_category_navigation", - "failure_mode": "false-negative", - "case": { - "url": "https://www.cnnbrasil.com.br/", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 2, - "hosts": [ - "www.cnnbrasil.com.br" - ], - "entry_path": "(?!/eleicoes/(?:[0-9]{4}/candidatos(?:/|$)|agregador-de-pesquisas(?:-[0-9]{4})?(?:/|$)|monitor-de-debates(?:/|$)))(?!(?:/economia/loterias|/eleicoes/2026/candidatos|/eleicoes/agregador\\-de\\-pesquisas\\-2025|/esportes/agenda|/esportes/automobilismo|/esportes/basquete|/esportes/esports|/esportes/futebol/tabela\\-do\\-brasileirao|/esportes/futebol/tabela\\-do\\-brasileirao\\-serie\\-b|/esportes/futebol/tabela\\-eliminatorias\\-2026|/esportes/olimpiadas|/pop/bbb|/pop/carnaval|/pop/celebridades|/pop/cinema|/pop/musica|/pop/realities|/pop/streaming|/pop/tv|/tudo\\-sobre/africa|/tudo\\-sobre/alimentacao|/tudo\\-sobre/asia|/tudo\\-sobre/auxilio\\-brasil|/tudo\\-sobre/banco\\-central\\-bc|/tudo\\-sobre/bolsa\\-de\\-valores|/tudo\\-sobre/brasilia|/tudo\\-sobre/camara\\-dos\\-deputados|/tudo\\-sobre/cancer|/tudo\\-sobre/china|/tudo\\-sobre/ciencia|/tudo\\-sobre/clima|/tudo\\-sobre/cnn\\-sinais\\-vitais|/tudo\\-sobre/copa\\-libertadores\\-da\\-america|/tudo\\-sobre/curiosidades|/tudo\\-sobre/drogas|/tudo\\-sobre/educacao|/tudo\\-sobre/estados\\-unidos|/tudo\\-sobre/europa|/tudo\\-sobre/exercicios\\-fisicos|/tudo\\-sobre/financas\\-pessoais|/tudo\\-sobre/golfe|/tudo\\-sobre/governo\\-lula|/tudo\\-sobre/guerra\\-de\\-israel|/tudo\\-sobre/guerra\\-na\\-ucrania|/tudo\\-sobre/imposto\\-de\\-renda|/tudo\\-sobre/investimentos|/tudo\\-sobre/meio\\-ambiente|/tudo\\-sobre/mercado|/tudo\\-sobre/minas\\-gerais|/tudo\\-sobre/nasa|/tudo\\-sobre/obesidade|/tudo\\-sobre/oriente\\-medio|/tudo\\-sobre/politicas\\-publicas|/tudo\\-sobre/rio\\-de\\-janeiro\\-estado|/tudo\\-sobre/russia|/tudo\\-sobre/sao\\-paulo\\-estado|/tudo\\-sobre/seguranca|/tudo\\-sobre/senado\\-federal|/tudo\\-sobre/supremo\\-tribunal\\-federal\\-stf|/tudo\\-sobre/sustentabilidade|/tudo\\-sobre/vacinacao|/tudo\\-sobre/volei|/viagemegastronomia/gastronomia|/viagemegastronomia/viagem)$)(?!/esportes/lances-ao-vivo/)/(?:blogs/[^/]+/[^/]+/.+|(?:agro|auto|ciencia|economia|educacao|eleicoes|esportes|infra|internacional|nacional|noticias|politica|pop|saude|style|viagemegastronomia)/.+)", - "stable_text_anchors": [], - "name_validation": "nonempty", - "entry_scope": { - "html_selector": "a:has(h2),a:has(h3),h2 > a,h3 > a" - } - } - }, - "artifact": "[Nova notícia](/eleicoes/nova-noticia/)\n[Nova análise](/blogs/novo-autor/politica/nova-analise/)", - "ok": true, - "expected_task_pass": true - }, - { - "id": "cnnbrasil-qualified-routes-new-science-and-education", - "category": "structured-extraction", - "root_cause": "news_detail_routes_not_category_navigation", - "failure_mode": "false-negative", - "case": { - "url": "https://www.cnnbrasil.com.br/", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 2, - "hosts": [ - "www.cnnbrasil.com.br" - ], - "entry_path": "(?!/eleicoes/(?:[0-9]{4}/candidatos(?:/|$)|agregador-de-pesquisas(?:-[0-9]{4})?(?:/|$)|monitor-de-debates(?:/|$)))(?!(?:/economia/loterias|/eleicoes/2026/candidatos|/eleicoes/agregador\\-de\\-pesquisas\\-2025|/esportes/agenda|/esportes/automobilismo|/esportes/basquete|/esportes/esports|/esportes/futebol/tabela\\-do\\-brasileirao|/esportes/futebol/tabela\\-do\\-brasileirao\\-serie\\-b|/esportes/futebol/tabela\\-eliminatorias\\-2026|/esportes/olimpiadas|/pop/bbb|/pop/carnaval|/pop/celebridades|/pop/cinema|/pop/musica|/pop/realities|/pop/streaming|/pop/tv|/tudo\\-sobre/africa|/tudo\\-sobre/alimentacao|/tudo\\-sobre/asia|/tudo\\-sobre/auxilio\\-brasil|/tudo\\-sobre/banco\\-central\\-bc|/tudo\\-sobre/bolsa\\-de\\-valores|/tudo\\-sobre/brasilia|/tudo\\-sobre/camara\\-dos\\-deputados|/tudo\\-sobre/cancer|/tudo\\-sobre/china|/tudo\\-sobre/ciencia|/tudo\\-sobre/clima|/tudo\\-sobre/cnn\\-sinais\\-vitais|/tudo\\-sobre/copa\\-libertadores\\-da\\-america|/tudo\\-sobre/curiosidades|/tudo\\-sobre/drogas|/tudo\\-sobre/educacao|/tudo\\-sobre/estados\\-unidos|/tudo\\-sobre/europa|/tudo\\-sobre/exercicios\\-fisicos|/tudo\\-sobre/financas\\-pessoais|/tudo\\-sobre/golfe|/tudo\\-sobre/governo\\-lula|/tudo\\-sobre/guerra\\-de\\-israel|/tudo\\-sobre/guerra\\-na\\-ucrania|/tudo\\-sobre/imposto\\-de\\-renda|/tudo\\-sobre/investimentos|/tudo\\-sobre/meio\\-ambiente|/tudo\\-sobre/mercado|/tudo\\-sobre/minas\\-gerais|/tudo\\-sobre/nasa|/tudo\\-sobre/obesidade|/tudo\\-sobre/oriente\\-medio|/tudo\\-sobre/politicas\\-publicas|/tudo\\-sobre/rio\\-de\\-janeiro\\-estado|/tudo\\-sobre/russia|/tudo\\-sobre/sao\\-paulo\\-estado|/tudo\\-sobre/seguranca|/tudo\\-sobre/senado\\-federal|/tudo\\-sobre/supremo\\-tribunal\\-federal\\-stf|/tudo\\-sobre/sustentabilidade|/tudo\\-sobre/vacinacao|/tudo\\-sobre/volei|/viagemegastronomia/gastronomia|/viagemegastronomia/viagem)$)(?!/esportes/lances-ao-vivo/)/(?:blogs/[^/]+/[^/]+/.+|(?:agro|auto|ciencia|economia|educacao|eleicoes|esportes|infra|internacional|nacional|noticias|politica|pop|saude|style|viagemegastronomia)/.+)", - "stable_text_anchors": [], - "name_validation": "nonempty", - "entry_scope": { - "html_selector": "a:has(h2),a:has(h3),h2 > a,h3 > a" - } - } - }, - "artifact": "[Ciência nova](/ciencia/descoberta-nova/)\n[Educação nova](/educacao/estudo-novo/)", - "ok": true, - "expected_task_pass": true - }, - { - "id": "cnnbrasil-qualified-routes-category-navigation", - "category": "structured-extraction", - "root_cause": "news_detail_routes_not_category_navigation", - "failure_mode": "false-positive", - "case": { - "url": "https://www.cnnbrasil.com.br/", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 2, - "hosts": [ - "www.cnnbrasil.com.br" - ], - "entry_path": "(?!/eleicoes/(?:[0-9]{4}/candidatos(?:/|$)|agregador-de-pesquisas(?:-[0-9]{4})?(?:/|$)|monitor-de-debates(?:/|$)))(?!(?:/economia/loterias|/eleicoes/2026/candidatos|/eleicoes/agregador\\-de\\-pesquisas\\-2025|/esportes/agenda|/esportes/automobilismo|/esportes/basquete|/esportes/esports|/esportes/futebol/tabela\\-do\\-brasileirao|/esportes/futebol/tabela\\-do\\-brasileirao\\-serie\\-b|/esportes/futebol/tabela\\-eliminatorias\\-2026|/esportes/olimpiadas|/pop/bbb|/pop/carnaval|/pop/celebridades|/pop/cinema|/pop/musica|/pop/realities|/pop/streaming|/pop/tv|/tudo\\-sobre/africa|/tudo\\-sobre/alimentacao|/tudo\\-sobre/asia|/tudo\\-sobre/auxilio\\-brasil|/tudo\\-sobre/banco\\-central\\-bc|/tudo\\-sobre/bolsa\\-de\\-valores|/tudo\\-sobre/brasilia|/tudo\\-sobre/camara\\-dos\\-deputados|/tudo\\-sobre/cancer|/tudo\\-sobre/china|/tudo\\-sobre/ciencia|/tudo\\-sobre/clima|/tudo\\-sobre/cnn\\-sinais\\-vitais|/tudo\\-sobre/copa\\-libertadores\\-da\\-america|/tudo\\-sobre/curiosidades|/tudo\\-sobre/drogas|/tudo\\-sobre/educacao|/tudo\\-sobre/estados\\-unidos|/tudo\\-sobre/europa|/tudo\\-sobre/exercicios\\-fisicos|/tudo\\-sobre/financas\\-pessoais|/tudo\\-sobre/golfe|/tudo\\-sobre/governo\\-lula|/tudo\\-sobre/guerra\\-de\\-israel|/tudo\\-sobre/guerra\\-na\\-ucrania|/tudo\\-sobre/imposto\\-de\\-renda|/tudo\\-sobre/investimentos|/tudo\\-sobre/meio\\-ambiente|/tudo\\-sobre/mercado|/tudo\\-sobre/minas\\-gerais|/tudo\\-sobre/nasa|/tudo\\-sobre/obesidade|/tudo\\-sobre/oriente\\-medio|/tudo\\-sobre/politicas\\-publicas|/tudo\\-sobre/rio\\-de\\-janeiro\\-estado|/tudo\\-sobre/russia|/tudo\\-sobre/sao\\-paulo\\-estado|/tudo\\-sobre/seguranca|/tudo\\-sobre/senado\\-federal|/tudo\\-sobre/supremo\\-tribunal\\-federal\\-stf|/tudo\\-sobre/sustentabilidade|/tudo\\-sobre/vacinacao|/tudo\\-sobre/volei|/viagemegastronomia/gastronomia|/viagemegastronomia/viagem)$)(?!/esportes/lances-ao-vivo/)/(?:blogs/[^/]+/[^/]+/.+|(?:agro|auto|ciencia|economia|educacao|eleicoes|esportes|infra|internacional|nacional|noticias|politica|pop|saude|style|viagemegastronomia)/.+)", - "stable_text_anchors": [], - "name_validation": "nonempty", - "entry_scope": { - "html_selector": "a:has(h2),a:has(h3),h2 > a,h3 > a" - } - } - }, - "artifact": "[Cinema](/pop/cinema/)\n[Loterias](/economia/loterias/)", - "ok": true, - "expected_task_pass": false - }, - { - "id": "cnnbrasil-qualified-routes-blog-author-index", - "category": "structured-extraction", - "root_cause": "news_detail_routes_not_category_navigation", - "failure_mode": "false-positive", - "case": { - "url": "https://www.cnnbrasil.com.br/", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 2, - "hosts": [ - "www.cnnbrasil.com.br" - ], - "entry_path": "(?!/eleicoes/(?:[0-9]{4}/candidatos(?:/|$)|agregador-de-pesquisas(?:-[0-9]{4})?(?:/|$)|monitor-de-debates(?:/|$)))(?!(?:/economia/loterias|/eleicoes/2026/candidatos|/eleicoes/agregador\\-de\\-pesquisas\\-2025|/esportes/agenda|/esportes/automobilismo|/esportes/basquete|/esportes/esports|/esportes/futebol/tabela\\-do\\-brasileirao|/esportes/futebol/tabela\\-do\\-brasileirao\\-serie\\-b|/esportes/futebol/tabela\\-eliminatorias\\-2026|/esportes/olimpiadas|/pop/bbb|/pop/carnaval|/pop/celebridades|/pop/cinema|/pop/musica|/pop/realities|/pop/streaming|/pop/tv|/tudo\\-sobre/africa|/tudo\\-sobre/alimentacao|/tudo\\-sobre/asia|/tudo\\-sobre/auxilio\\-brasil|/tudo\\-sobre/banco\\-central\\-bc|/tudo\\-sobre/bolsa\\-de\\-valores|/tudo\\-sobre/brasilia|/tudo\\-sobre/camara\\-dos\\-deputados|/tudo\\-sobre/cancer|/tudo\\-sobre/china|/tudo\\-sobre/ciencia|/tudo\\-sobre/clima|/tudo\\-sobre/cnn\\-sinais\\-vitais|/tudo\\-sobre/copa\\-libertadores\\-da\\-america|/tudo\\-sobre/curiosidades|/tudo\\-sobre/drogas|/tudo\\-sobre/educacao|/tudo\\-sobre/estados\\-unidos|/tudo\\-sobre/europa|/tudo\\-sobre/exercicios\\-fisicos|/tudo\\-sobre/financas\\-pessoais|/tudo\\-sobre/golfe|/tudo\\-sobre/governo\\-lula|/tudo\\-sobre/guerra\\-de\\-israel|/tudo\\-sobre/guerra\\-na\\-ucrania|/tudo\\-sobre/imposto\\-de\\-renda|/tudo\\-sobre/investimentos|/tudo\\-sobre/meio\\-ambiente|/tudo\\-sobre/mercado|/tudo\\-sobre/minas\\-gerais|/tudo\\-sobre/nasa|/tudo\\-sobre/obesidade|/tudo\\-sobre/oriente\\-medio|/tudo\\-sobre/politicas\\-publicas|/tudo\\-sobre/rio\\-de\\-janeiro\\-estado|/tudo\\-sobre/russia|/tudo\\-sobre/sao\\-paulo\\-estado|/tudo\\-sobre/seguranca|/tudo\\-sobre/senado\\-federal|/tudo\\-sobre/supremo\\-tribunal\\-federal\\-stf|/tudo\\-sobre/sustentabilidade|/tudo\\-sobre/vacinacao|/tudo\\-sobre/volei|/viagemegastronomia/gastronomia|/viagemegastronomia/viagem)$)(?!/esportes/lances-ao-vivo/)/(?:blogs/[^/]+/[^/]+/.+|(?:agro|auto|ciencia|economia|educacao|eleicoes|esportes|infra|internacional|nacional|noticias|politica|pop|saude|style|viagemegastronomia)/.+)", - "stable_text_anchors": [], - "name_validation": "nonempty", - "entry_scope": { - "html_selector": "a:has(h2),a:has(h3),h2 > a,h3 > a" - } - } - }, - "artifact": "[Autora](/blogs/nova-autora/)\n[Autor](/blogs/novo-autor/)", - "ok": true, - "expected_task_pass": false - }, - { - "id": "cnnbrasil-qualified-routes-scoreboard-only", - "category": "structured-extraction", - "root_cause": "news_detail_routes_not_category_navigation", - "failure_mode": "false-positive", - "case": { - "url": "https://www.cnnbrasil.com.br/", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 2, - "hosts": [ - "www.cnnbrasil.com.br" - ], - "entry_path": "(?!/eleicoes/(?:[0-9]{4}/candidatos(?:/|$)|agregador-de-pesquisas(?:-[0-9]{4})?(?:/|$)|monitor-de-debates(?:/|$)))(?!(?:/economia/loterias|/eleicoes/2026/candidatos|/eleicoes/agregador\\-de\\-pesquisas\\-2025|/esportes/agenda|/esportes/automobilismo|/esportes/basquete|/esportes/esports|/esportes/futebol/tabela\\-do\\-brasileirao|/esportes/futebol/tabela\\-do\\-brasileirao\\-serie\\-b|/esportes/futebol/tabela\\-eliminatorias\\-2026|/esportes/olimpiadas|/pop/bbb|/pop/carnaval|/pop/celebridades|/pop/cinema|/pop/musica|/pop/realities|/pop/streaming|/pop/tv|/tudo\\-sobre/africa|/tudo\\-sobre/alimentacao|/tudo\\-sobre/asia|/tudo\\-sobre/auxilio\\-brasil|/tudo\\-sobre/banco\\-central\\-bc|/tudo\\-sobre/bolsa\\-de\\-valores|/tudo\\-sobre/brasilia|/tudo\\-sobre/camara\\-dos\\-deputados|/tudo\\-sobre/cancer|/tudo\\-sobre/china|/tudo\\-sobre/ciencia|/tudo\\-sobre/clima|/tudo\\-sobre/cnn\\-sinais\\-vitais|/tudo\\-sobre/copa\\-libertadores\\-da\\-america|/tudo\\-sobre/curiosidades|/tudo\\-sobre/drogas|/tudo\\-sobre/educacao|/tudo\\-sobre/estados\\-unidos|/tudo\\-sobre/europa|/tudo\\-sobre/exercicios\\-fisicos|/tudo\\-sobre/financas\\-pessoais|/tudo\\-sobre/golfe|/tudo\\-sobre/governo\\-lula|/tudo\\-sobre/guerra\\-de\\-israel|/tudo\\-sobre/guerra\\-na\\-ucrania|/tudo\\-sobre/imposto\\-de\\-renda|/tudo\\-sobre/investimentos|/tudo\\-sobre/meio\\-ambiente|/tudo\\-sobre/mercado|/tudo\\-sobre/minas\\-gerais|/tudo\\-sobre/nasa|/tudo\\-sobre/obesidade|/tudo\\-sobre/oriente\\-medio|/tudo\\-sobre/politicas\\-publicas|/tudo\\-sobre/rio\\-de\\-janeiro\\-estado|/tudo\\-sobre/russia|/tudo\\-sobre/sao\\-paulo\\-estado|/tudo\\-sobre/seguranca|/tudo\\-sobre/senado\\-federal|/tudo\\-sobre/supremo\\-tribunal\\-federal\\-stf|/tudo\\-sobre/sustentabilidade|/tudo\\-sobre/vacinacao|/tudo\\-sobre/volei|/viagemegastronomia/gastronomia|/viagemegastronomia/viagem)$)(?!/esportes/lances-ao-vivo/)/(?:blogs/[^/]+/[^/]+/.+|(?:agro|auto|ciencia|economia|educacao|eleicoes|esportes|infra|internacional|nacional|noticias|politica|pop|saude|style|viagemegastronomia)/.+)", - "stable_text_anchors": [], - "name_validation": "nonempty", - "entry_scope": { - "html_selector": "a:has(h2),a:has(h3),h2 > a,h3 > a" - } - } - }, - "artifact": "[Partida A](/esportes/lances-ao-vivo/liga/2026/a-b/)\n[Partida B](/esportes/lances-ao-vivo/liga/2026/c-d/)", - "ok": true, - "expected_task_pass": false - }, - { - "id": "cnnbrasil-qualified-routes-election-tools", - "category": "structured-extraction", - "root_cause": "news_detail_routes_not_category_navigation", - "failure_mode": "false-positive", - "case": { - "url": "https://www.cnnbrasil.com.br/", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 2, - "hosts": [ - "www.cnnbrasil.com.br" - ], - "entry_path": "(?!/eleicoes/(?:[0-9]{4}/candidatos(?:/|$)|agregador-de-pesquisas(?:-[0-9]{4})?(?:/|$)|monitor-de-debates(?:/|$)))(?!(?:/economia/loterias|/eleicoes/2026/candidatos|/eleicoes/agregador\\-de\\-pesquisas\\-2025|/esportes/agenda|/esportes/automobilismo|/esportes/basquete|/esportes/esports|/esportes/futebol/tabela\\-do\\-brasileirao|/esportes/futebol/tabela\\-do\\-brasileirao\\-serie\\-b|/esportes/futebol/tabela\\-eliminatorias\\-2026|/esportes/olimpiadas|/pop/bbb|/pop/carnaval|/pop/celebridades|/pop/cinema|/pop/musica|/pop/realities|/pop/streaming|/pop/tv|/tudo\\-sobre/africa|/tudo\\-sobre/alimentacao|/tudo\\-sobre/asia|/tudo\\-sobre/auxilio\\-brasil|/tudo\\-sobre/banco\\-central\\-bc|/tudo\\-sobre/bolsa\\-de\\-valores|/tudo\\-sobre/brasilia|/tudo\\-sobre/camara\\-dos\\-deputados|/tudo\\-sobre/cancer|/tudo\\-sobre/china|/tudo\\-sobre/ciencia|/tudo\\-sobre/clima|/tudo\\-sobre/cnn\\-sinais\\-vitais|/tudo\\-sobre/copa\\-libertadores\\-da\\-america|/tudo\\-sobre/curiosidades|/tudo\\-sobre/drogas|/tudo\\-sobre/educacao|/tudo\\-sobre/estados\\-unidos|/tudo\\-sobre/europa|/tudo\\-sobre/exercicios\\-fisicos|/tudo\\-sobre/financas\\-pessoais|/tudo\\-sobre/golfe|/tudo\\-sobre/governo\\-lula|/tudo\\-sobre/guerra\\-de\\-israel|/tudo\\-sobre/guerra\\-na\\-ucrania|/tudo\\-sobre/imposto\\-de\\-renda|/tudo\\-sobre/investimentos|/tudo\\-sobre/meio\\-ambiente|/tudo\\-sobre/mercado|/tudo\\-sobre/minas\\-gerais|/tudo\\-sobre/nasa|/tudo\\-sobre/obesidade|/tudo\\-sobre/oriente\\-medio|/tudo\\-sobre/politicas\\-publicas|/tudo\\-sobre/rio\\-de\\-janeiro\\-estado|/tudo\\-sobre/russia|/tudo\\-sobre/sao\\-paulo\\-estado|/tudo\\-sobre/seguranca|/tudo\\-sobre/senado\\-federal|/tudo\\-sobre/supremo\\-tribunal\\-federal\\-stf|/tudo\\-sobre/sustentabilidade|/tudo\\-sobre/vacinacao|/tudo\\-sobre/volei|/viagemegastronomia/gastronomia|/viagemegastronomia/viagem)$)(?!/esportes/lances-ao-vivo/)/(?:blogs/[^/]+/[^/]+/.+|(?:agro|auto|ciencia|economia|educacao|eleicoes|esportes|infra|internacional|nacional|noticias|politica|pop|saude|style|viagemegastronomia)/.+)", - "stable_text_anchors": [], - "name_validation": "nonempty", - "entry_scope": { - "html_selector": "a:has(h2),a:has(h3),h2 > a,h3 > a" - } - } - }, - "artifact": "[Ver estatísticas](/eleicoes/2026/candidatos/estatisticas/)\n[Índice CNN](/eleicoes/agregador-de-pesquisas-2025/)", - "ok": true, - "expected_task_pass": false - }, - { - "id": "cnnbrasil-qualified-routes-duplicate-article", - "category": "structured-extraction", - "root_cause": "news_detail_routes_not_category_navigation", - "failure_mode": "false-positive", - "case": { - "url": "https://www.cnnbrasil.com.br/", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 2, - "hosts": [ - "www.cnnbrasil.com.br" - ], - "entry_path": "(?!/eleicoes/(?:[0-9]{4}/candidatos(?:/|$)|agregador-de-pesquisas(?:-[0-9]{4})?(?:/|$)|monitor-de-debates(?:/|$)))(?!(?:/economia/loterias|/eleicoes/2026/candidatos|/eleicoes/agregador\\-de\\-pesquisas\\-2025|/esportes/agenda|/esportes/automobilismo|/esportes/basquete|/esportes/esports|/esportes/futebol/tabela\\-do\\-brasileirao|/esportes/futebol/tabela\\-do\\-brasileirao\\-serie\\-b|/esportes/futebol/tabela\\-eliminatorias\\-2026|/esportes/olimpiadas|/pop/bbb|/pop/carnaval|/pop/celebridades|/pop/cinema|/pop/musica|/pop/realities|/pop/streaming|/pop/tv|/tudo\\-sobre/africa|/tudo\\-sobre/alimentacao|/tudo\\-sobre/asia|/tudo\\-sobre/auxilio\\-brasil|/tudo\\-sobre/banco\\-central\\-bc|/tudo\\-sobre/bolsa\\-de\\-valores|/tudo\\-sobre/brasilia|/tudo\\-sobre/camara\\-dos\\-deputados|/tudo\\-sobre/cancer|/tudo\\-sobre/china|/tudo\\-sobre/ciencia|/tudo\\-sobre/clima|/tudo\\-sobre/cnn\\-sinais\\-vitais|/tudo\\-sobre/copa\\-libertadores\\-da\\-america|/tudo\\-sobre/curiosidades|/tudo\\-sobre/drogas|/tudo\\-sobre/educacao|/tudo\\-sobre/estados\\-unidos|/tudo\\-sobre/europa|/tudo\\-sobre/exercicios\\-fisicos|/tudo\\-sobre/financas\\-pessoais|/tudo\\-sobre/golfe|/tudo\\-sobre/governo\\-lula|/tudo\\-sobre/guerra\\-de\\-israel|/tudo\\-sobre/guerra\\-na\\-ucrania|/tudo\\-sobre/imposto\\-de\\-renda|/tudo\\-sobre/investimentos|/tudo\\-sobre/meio\\-ambiente|/tudo\\-sobre/mercado|/tudo\\-sobre/minas\\-gerais|/tudo\\-sobre/nasa|/tudo\\-sobre/obesidade|/tudo\\-sobre/oriente\\-medio|/tudo\\-sobre/politicas\\-publicas|/tudo\\-sobre/rio\\-de\\-janeiro\\-estado|/tudo\\-sobre/russia|/tudo\\-sobre/sao\\-paulo\\-estado|/tudo\\-sobre/seguranca|/tudo\\-sobre/senado\\-federal|/tudo\\-sobre/supremo\\-tribunal\\-federal\\-stf|/tudo\\-sobre/sustentabilidade|/tudo\\-sobre/vacinacao|/tudo\\-sobre/volei|/viagemegastronomia/gastronomia|/viagemegastronomia/viagem)$)(?!/esportes/lances-ao-vivo/)/(?:blogs/[^/]+/[^/]+/.+|(?:agro|auto|ciencia|economia|educacao|eleicoes|esportes|infra|internacional|nacional|noticias|politica|pop|saude|style|viagemegastronomia)/.+)", - "stable_text_anchors": [], - "name_validation": "nonempty", - "entry_scope": { - "html_selector": "a:has(h2),a:has(h3),h2 > a,h3 > a" - } - } - }, - "artifact": "[Título A](/politica/noticia-a/)\n[Outro título](/politica/noticia-a/)", - "ok": true, - "expected_task_pass": false - }, - { - "id": "naver-politics-owned-visible-visible-owned-cards", - "category": "structured-extraction", - "root_cause": "initial_visible_news_scope_excludes_collapsed_cards", - "failure_mode": "false-negative", - "case": { - "url": "https://news.naver.com/section/100", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 2, - "hosts": [ - "n.news.naver.com" - ], - "entry_path": "/(?:mnews/)?article/[0-9]+/[0-9]+", - "stable_text_anchors": [], - "name_validation": "nonempty", - "entry_identity_pattern": "/article/([0-9]+/[0-9]+)(?:/|$)", - "entry_scope": { - "html_selector": "#newsct .sa_item:not(.is_blind)", - "title_selector": ".sa_text_strong", - "link_selector": "a.sa_text_title", - "start_pattern": "헤드라인 뉴스", - "end_pattern": "언론사별 가장 많이 본 뉴스" - } - } - }, - "artifact": "", - "ok": true, - "expected_task_pass": true - }, - { - "id": "naver-politics-owned-visible-hidden-expansion-not-denominator", - "category": "structured-extraction", - "root_cause": "initial_visible_news_scope_excludes_collapsed_cards", - "failure_mode": "false-positive", - "case": { - "url": "https://news.naver.com/section/100", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 2, - "hosts": [ - "n.news.naver.com" - ], - "entry_path": "/(?:mnews/)?article/[0-9]+/[0-9]+", - "stable_text_anchors": [], - "name_validation": "nonempty", - "entry_identity_pattern": "/article/([0-9]+/[0-9]+)(?:/|$)", - "entry_scope": { - "html_selector": "#newsct .sa_item:not(.is_blind)", - "title_selector": ".sa_text_strong", - "link_selector": "a.sa_text_title", - "start_pattern": "헤드라인 뉴스", - "end_pattern": "언론사별 가장 많이 본 뉴스" - } + { + "name": "species", + "label": "Species", + "type": "text" + }, + { + "name": "diet", + "label": "Diet", + "type": "text" + }, + { + "name": "habitat", + "label": "Habitat", + "type": "text" } + ], + "table_scope": { + "root_selector": "section#usage", + "root_index": 0, + "row_selector": "table tbody tr", + "cell_selector": "td", + "start_pattern": "The standard data table presumes that the entire data set is available locally\\.", + "end_pattern": "(?m)^[ \\t#]*API[ \\t]*$", + "plain_row_pattern": "(?P[A-Z][A-Za-z]+(?: [A-Z][A-Za-z]+)*)\\s+(?P[A-Z][a-z]+ [a-z]+)\\s+(?PHerbivore|Carnivore|Omnivore)\\s+(?P[A-Za-z, ]+)", + "plain_multiline_row_pattern": "(?m)^[ \\t]*(?P[A-Z][A-Za-z]+(?: [A-Z][A-Za-z]+)*)\\n[ \\t\\n]*(?P[A-Z][a-z]+ [a-z]+)\\n[ \\t\\n]*(?PHerbivore|Carnivore|Omnivore)\\n[ \\t\\n]*(?P[A-Za-z, ]+)[ \\t]*$", + "reference_kind": "fixed_demo", + "reference_rows": [ + { + "name": "African Elephant", + "species": "Loxodonta africana", + "diet": "Herbivore", + "habitat": "Savanna, Forests" + }, + { + "name": "Lion", + "species": "Panthera leo", + "diet": "Carnivore", + "habitat": "Savanna, Grassland" + } + ] + } }, - "artifact": "", - "ok": true, - "expected_task_pass": false - }, - { - "id": "naver-politics-owned-visible-empty-title", - "category": "structured-extraction", - "root_cause": "initial_visible_news_scope_excludes_collapsed_cards", - "failure_mode": "false-positive", - "case": { - "url": "https://news.naver.com/section/100", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 2, - "hosts": [ - "n.news.naver.com" - ], - "entry_path": "/(?:mnews/)?article/[0-9]+/[0-9]+", - "stable_text_anchors": [], - "name_validation": "nonempty", - "entry_identity_pattern": "/article/([0-9]+/[0-9]+)(?:/|$)", - "entry_scope": { - "html_selector": "#newsct .sa_item:not(.is_blind)", - "title_selector": ".sa_text_strong", - "link_selector": "a.sa_text_title", - "start_pattern": "헤드라인 뉴스", - "end_pattern": "언론사별 가장 많이 본 뉴스" - } + "body_requirements": { + "version": 3, + "unit": "region_compact_hexagram_recall", + "minimum_coverage": 0.8, + "minimum_floor": 0.6, + "minimum_region_passes": 1, + "segmentation_method": "balanced_tokens_fallback", + "regions": [ + { + "id": "body-region-1", + "text": "sorting pagination and filtering is supported and done internally by the component itself", + "source_text": "sorting pagination and filtering is supported and done internally by the component itself", + "segmentation_method": "balanced_tokens_fallback" } - }, - "artifact": "", - "ok": true, - "expected_task_pass": false - }, - { - "id": "naver-politics-owned-visible-sidebar-not-main", - "category": "structured-extraction", - "root_cause": "initial_visible_news_scope_excludes_collapsed_cards", - "failure_mode": "false-positive", - "case": { - "url": "https://news.naver.com/section/100", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 2, - "hosts": [ - "n.news.naver.com" - ], - "entry_path": "/(?:mnews/)?article/[0-9]+/[0-9]+", - "stable_text_anchors": [], - "name_validation": "nonempty", - "entry_identity_pattern": "/article/([0-9]+/[0-9]+)(?:/|$)", - "entry_scope": { - "html_selector": "#newsct .sa_item:not(.is_blind)", - "title_selector": ".sa_text_strong", - "link_selector": "a.sa_text_title", - "start_pattern": "헤드라인 뉴스", - "end_pattern": "언론사별 가장 많이 본 뉴스" - } + ] + } + }, + "artifact": "Sorting, pagination, and filtering is supported and done internally by the component itself.\nThe standard data table presumes that the entire data set is available locally.\n|Name|Species|Diet|Habitat|\n|---|---|---|---|\n|African Elephant|Loxodonta africana|Herbivore|Savanna, Forests|\n|Lion|Panthera leo|Carnivore|Savanna, Grassland|\n", + "ok": true, + "expected_task_pass": true + }, + { + "id": "usage-prose-alone-not-enough", + "category": "structured-extraction", + "root_cause": "usage_demo_rows_not_api_component_table", + "failure_mode": "false-positive", + "case": { + "url": "https://vuetifyjs.com/en/components/data-tables/basics/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 2, + "stable_text_anchors": [], + "record_identity_fields": [ + "name", + "species" + ], + "duplicate_policy": "deduplicate_identical", + "table_columns": [ + { + "name": "name", + "label": "Name", + "type": "text" + }, + { + "name": "species", + "label": "Species", + "type": "text" + }, + { + "name": "diet", + "label": "Diet", + "type": "text" + }, + { + "name": "habitat", + "label": "Habitat", + "type": "text" } + ], + "table_scope": { + "root_selector": "section#usage", + "root_index": 0, + "row_selector": "table tbody tr", + "cell_selector": "td", + "start_pattern": "The standard data table presumes that the entire data set is available locally\\.", + "end_pattern": "(?m)^[ \\t#]*API[ \\t]*$", + "plain_row_pattern": "(?P[A-Z][A-Za-z]+(?: [A-Z][A-Za-z]+)*)\\s+(?P[A-Z][a-z]+ [a-z]+)\\s+(?PHerbivore|Carnivore|Omnivore)\\s+(?P[A-Za-z, ]+)", + "plain_multiline_row_pattern": "(?m)^[ \\t]*(?P[A-Z][A-Za-z]+(?: [A-Z][A-Za-z]+)*)\\n[ \\t\\n]*(?P[A-Z][a-z]+ [a-z]+)\\n[ \\t\\n]*(?PHerbivore|Carnivore|Omnivore)\\n[ \\t\\n]*(?P[A-Za-z, ]+)[ \\t]*$", + "reference_kind": "fixed_demo", + "reference_rows": [ + { + "name": "African Elephant", + "species": "Loxodonta africana", + "diet": "Herbivore", + "habitat": "Savanna, Forests" + }, + { + "name": "Lion", + "species": "Panthera leo", + "diet": "Carnivore", + "habitat": "Savanna, Grassland" + } + ] + } }, - "artifact": "", - "ok": true, - "expected_task_pass": false - }, - { - "id": "naver-politics-owned-visible-fresh-markdown", - "category": "structured-extraction", - "root_cause": "initial_visible_news_scope_excludes_collapsed_cards", - "failure_mode": "false-negative", - "case": { - "url": "https://news.naver.com/section/100", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 2, - "hosts": [ - "n.news.naver.com" - ], - "entry_path": "/(?:mnews/)?article/[0-9]+/[0-9]+", - "stable_text_anchors": [], - "name_validation": "nonempty", - "entry_identity_pattern": "/article/([0-9]+/[0-9]+)(?:/|$)", - "entry_scope": { - "html_selector": "#newsct .sa_item:not(.is_blind)", - "title_selector": ".sa_text_strong", - "link_selector": "a.sa_text_title", - "start_pattern": "헤드라인 뉴스", - "end_pattern": "언론사별 가장 많이 본 뉴스" - } + "body_requirements": { + "version": 3, + "unit": "region_compact_hexagram_recall", + "minimum_coverage": 0.8, + "minimum_floor": 0.6, + "minimum_region_passes": 1, + "segmentation_method": "balanced_tokens_fallback", + "regions": [ + { + "id": "body-region-1", + "text": "sorting pagination and filtering is supported and done internally by the component itself", + "source_text": "sorting pagination and filtering is supported and done internally by the component itself", + "segmentation_method": "balanced_tokens_fallback" } - }, - "artifact": "헤드라인 뉴스\n[Fresh A](https://n.news.naver.com/mnews/article/001/789)\n[Fresh B](https://n.news.naver.com/mnews/article/002/987)\n언론사별 가장 많이 본 뉴스", - "ok": true, - "expected_task_pass": true - }, - { - "id": "naver-politics-owned-visible-repeated-id", - "category": "structured-extraction", - "root_cause": "initial_visible_news_scope_excludes_collapsed_cards", - "failure_mode": "false-positive", - "case": { - "url": "https://news.naver.com/section/100", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 2, - "hosts": [ - "n.news.naver.com" - ], - "entry_path": "/(?:mnews/)?article/[0-9]+/[0-9]+", - "stable_text_anchors": [], - "name_validation": "nonempty", - "entry_identity_pattern": "/article/([0-9]+/[0-9]+)(?:/|$)", - "entry_scope": { - "html_selector": "#newsct .sa_item:not(.is_blind)", - "title_selector": ".sa_text_strong", - "link_selector": "a.sa_text_title", - "start_pattern": "헤드라인 뉴스", - "end_pattern": "언론사별 가장 많이 본 뉴스" - } + ] + } + }, + "artifact": "Sorting, pagination, and filtering is supported and done internally by the component itself.\nThe standard data table presumes that the entire data set is available locally.", + "ok": true, + "expected_task_pass": false + }, + { + "id": "usage-api-table-not-demo", + "category": "structured-extraction", + "root_cause": "usage_demo_rows_not_api_component_table", + "failure_mode": "false-positive", + "case": { + "url": "https://vuetifyjs.com/en/components/data-tables/basics/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 2, + "stable_text_anchors": [], + "record_identity_fields": [ + "name", + "species" + ], + "duplicate_policy": "deduplicate_identical", + "table_columns": [ + { + "name": "name", + "label": "Name", + "type": "text" + }, + { + "name": "species", + "label": "Species", + "type": "text" + }, + { + "name": "diet", + "label": "Diet", + "type": "text" + }, + { + "name": "habitat", + "label": "Habitat", + "type": "text" } + ], + "table_scope": { + "root_selector": "section#usage", + "root_index": 0, + "row_selector": "table tbody tr", + "cell_selector": "td", + "start_pattern": "The standard data table presumes that the entire data set is available locally\\.", + "end_pattern": "(?m)^[ \\t#]*API[ \\t]*$", + "plain_row_pattern": "(?P[A-Z][A-Za-z]+(?: [A-Z][A-Za-z]+)*)\\s+(?P[A-Z][a-z]+ [a-z]+)\\s+(?PHerbivore|Carnivore|Omnivore)\\s+(?P[A-Za-z, ]+)", + "plain_multiline_row_pattern": "(?m)^[ \\t]*(?P[A-Z][A-Za-z]+(?: [A-Z][A-Za-z]+)*)\\n[ \\t\\n]*(?P[A-Z][a-z]+ [a-z]+)\\n[ \\t\\n]*(?PHerbivore|Carnivore|Omnivore)\\n[ \\t\\n]*(?P[A-Za-z, ]+)[ \\t]*$", + "reference_kind": "fixed_demo", + "reference_rows": [ + { + "name": "African Elephant", + "species": "Loxodonta africana", + "diet": "Herbivore", + "habitat": "Savanna, Forests" + }, + { + "name": "Lion", + "species": "Panthera leo", + "diet": "Carnivore", + "habitat": "Savanna, Grassland" + } + ] + } }, - "artifact": "", - "ok": true, - "expected_task_pass": false - }, - { - "id": "owned-anchor-card-anchor-is-card", - "category": "structured-extraction", - "root_cause": "card_anchor_owns_visible_title_and_destination", - "failure_mode": "false-negative", - "case": { - "url": "https://www.kabum.com.br/hardware", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 2, - "hosts": [ - "www.kabum.com.br" - ], - "entry_path": "/produto/\\d+/.+", - "stable_text_anchors": [], - "name_validation": "nonempty", - "entry_identity_pattern": "/produto/([0-9]+)(?:/|$)", - "entry_scope": { - "html_selector": "a[href*=\"/produto/\"]:has(.line-clamp-2)", - "title_selector": ".line-clamp-2", - "link_selector": "a[href]" - } + "body_requirements": { + "version": 3, + "unit": "region_compact_hexagram_recall", + "minimum_coverage": 0.8, + "minimum_floor": 0.6, + "minimum_region_passes": 1, + "segmentation_method": "balanced_tokens_fallback", + "regions": [ + { + "id": "body-region-1", + "text": "sorting pagination and filtering is supported and done internally by the component itself", + "source_text": "sorting pagination and filtering is supported and done internally by the component itself", + "segmentation_method": "balanced_tokens_fallback" } - }, - "artifact": "Fresh productFresh product", - "ok": true, - "expected_task_pass": true - }, - { - "id": "owned-anchor-card-empty-title-with-price", - "category": "structured-extraction", - "root_cause": "card_anchor_owns_visible_title_and_destination", - "failure_mode": "false-positive", - "case": { - "url": "https://www.kabum.com.br/hardware", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 2, - "hosts": [ - "www.kabum.com.br" - ], - "entry_path": "/produto/\\d+/.+", - "stable_text_anchors": [], - "name_validation": "nonempty", - "entry_identity_pattern": "/produto/([0-9]+)(?:/|$)", - "entry_scope": { - "html_selector": "a[href*=\"/produto/\"]:has(.line-clamp-2)", - "title_selector": ".line-clamp-2", - "link_selector": "a[href]" - } + ] + } + }, + "artifact": "Sorting, pagination, and filtering is supported and done internally by the component itself.\nThe standard data table presumes that the entire data set is available locally.\nAPI\n\n|Name|Species|Diet|Habitat|\n|---|---|---|---|\n|African Elephant|Loxodonta africana|Herbivore|Savanna, Forests|\n|Lion|Panthera leo|Carnivore|Savanna, Grassland|\n", + "ok": true, + "expected_task_pass": false + }, + { + "id": "usage-mixed-fields-rejected", + "category": "structured-extraction", + "root_cause": "usage_demo_rows_not_api_component_table", + "failure_mode": "false-positive", + "case": { + "url": "https://vuetifyjs.com/en/components/data-tables/basics/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 2, + "stable_text_anchors": [], + "record_identity_fields": [ + "name", + "species" + ], + "duplicate_policy": "deduplicate_identical", + "table_columns": [ + { + "name": "name", + "label": "Name", + "type": "text" + }, + { + "name": "species", + "label": "Species", + "type": "text" + }, + { + "name": "diet", + "label": "Diet", + "type": "text" + }, + { + "name": "habitat", + "label": "Habitat", + "type": "text" } + ], + "table_scope": { + "root_selector": "section#usage", + "root_index": 0, + "row_selector": "table tbody tr", + "cell_selector": "td", + "start_pattern": "The standard data table presumes that the entire data set is available locally\\.", + "end_pattern": "(?m)^[ \\t#]*API[ \\t]*$", + "plain_row_pattern": "(?P[A-Z][A-Za-z]+(?: [A-Z][A-Za-z]+)*)\\s+(?P[A-Z][a-z]+ [a-z]+)\\s+(?PHerbivore|Carnivore|Omnivore)\\s+(?P[A-Za-z, ]+)", + "plain_multiline_row_pattern": "(?m)^[ \\t]*(?P[A-Z][A-Za-z]+(?: [A-Z][A-Za-z]+)*)\\n[ \\t\\n]*(?P[A-Z][a-z]+ [a-z]+)\\n[ \\t\\n]*(?PHerbivore|Carnivore|Omnivore)\\n[ \\t\\n]*(?P[A-Za-z, ]+)[ \\t]*$", + "reference_kind": "fixed_demo", + "reference_rows": [ + { + "name": "African Elephant", + "species": "Loxodonta africana", + "diet": "Herbivore", + "habitat": "Savanna, Forests" + }, + { + "name": "Lion", + "species": "Panthera leo", + "diet": "Carnivore", + "habitat": "Savanna, Grassland" + } + ] + } }, - "artifact": "Fresh productR$ 100", - "ok": true, - "expected_task_pass": false - }, - { - "id": "owned-anchor-card-price-only", - "category": "structured-extraction", - "root_cause": "card_anchor_owns_visible_title_and_destination", - "failure_mode": "false-positive", - "case": { - "url": "https://www.kabum.com.br/hardware", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 2, - "hosts": [ - "www.kabum.com.br" - ], - "entry_path": "/produto/\\d+/.+", - "stable_text_anchors": [], - "name_validation": "nonempty", - "entry_identity_pattern": "/produto/([0-9]+)(?:/|$)", - "entry_scope": { - "html_selector": "a[href*=\"/produto/\"]:has(.line-clamp-2)", - "title_selector": ".line-clamp-2", - "link_selector": "a[href]" - } + "body_requirements": { + "version": 3, + "unit": "region_compact_hexagram_recall", + "minimum_coverage": 0.8, + "minimum_floor": 0.6, + "minimum_region_passes": 1, + "segmentation_method": "balanced_tokens_fallback", + "regions": [ + { + "id": "body-region-1", + "text": "sorting pagination and filtering is supported and done internally by the component itself", + "source_text": "sorting pagination and filtering is supported and done internally by the component itself", + "segmentation_method": "balanced_tokens_fallback" + } + ] + } + }, + "artifact": "Sorting, pagination, and filtering is supported and done internally by the component itself.\nThe standard data table presumes that the entire data set is available locally.\n|Name|Species|Diet|Habitat|\n|---|---|---|---|\n|African Elephant|Loxodonta africana|Herbivore|Savanna, Forests|\n|Lion|Loxodonta africana|Carnivore|Savanna, Grassland|\n", + "ok": true, + "expected_task_pass": false + }, + { + "id": "amazonmx-owned-card-fresh-products", + "category": "structured-extraction", + "root_cause": "same_card_title_detail_pairing", + "failure_mode": "false-negative", + "case": { + "url": "https://www.amazon.com.mx/gp/bestsellers/electronics/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "www.amazon.com.mx" + ], + "entry_path": "(?:/[^/]+)?/dp/[A-Z0-9]{10}(?:/[^?#]*)?", + "stable_text_anchors": [], + "entry_identity_pattern": "/dp/([A-Z0-9]{10})(?:/|$)", + "name_validation": "nonempty", + "entry_scope": { + "html_selector": "#gridItemRoot", + "title_selector": "a[role=link] [class*=\"line-clamp\"]", + "link_selector": "a[role=link]:has([class*=\"line-clamp\"])" + } + } + }, + "artifact": "", + "ok": true, + "expected_task_pass": true + }, + { + "id": "amazonmx-owned-card-image-only", + "category": "structured-extraction", + "root_cause": "same_card_title_detail_pairing", + "failure_mode": "false-positive", + "case": { + "url": "https://www.amazon.com.mx/gp/bestsellers/electronics/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "www.amazon.com.mx" + ], + "entry_path": "(?:/[^/]+)?/dp/[A-Z0-9]{10}(?:/[^?#]*)?", + "stable_text_anchors": [], + "entry_identity_pattern": "/dp/([A-Z0-9]{10})(?:/|$)", + "name_validation": "nonempty", + "entry_scope": { + "html_selector": "#gridItemRoot", + "title_selector": "a[role=link] [class*=\"line-clamp\"]", + "link_selector": "a[role=link]:has([class*=\"line-clamp\"])" + } + } + }, + "artifact": "
\"Producto
", + "ok": true, + "expected_task_pass": false + }, + { + "id": "amazonmx-owned-card-duplicate-asin", + "category": "structured-extraction", + "root_cause": "same_card_title_detail_pairing", + "failure_mode": "false-positive", + "case": { + "url": "https://www.amazon.com.mx/gp/bestsellers/electronics/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "www.amazon.com.mx" + ], + "entry_path": "(?:/[^/]+)?/dp/[A-Z0-9]{10}(?:/[^?#]*)?", + "stable_text_anchors": [], + "entry_identity_pattern": "/dp/([A-Z0-9]{10})(?:/|$)", + "name_validation": "nonempty", + "entry_scope": { + "html_selector": "#gridItemRoot", + "title_selector": "a[role=link] [class*=\"line-clamp\"]", + "link_selector": "a[role=link]:has([class*=\"line-clamp\"])" + } + } + }, + "artifact": "", + "ok": true, + "expected_task_pass": false + }, + { + "id": "amazonmx-owned-card-wrong-host", + "category": "structured-extraction", + "root_cause": "same_card_title_detail_pairing", + "failure_mode": "false-positive", + "case": { + "url": "https://www.amazon.com.mx/gp/bestsellers/electronics/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "www.amazon.com.mx" + ], + "entry_path": "(?:/[^/]+)?/dp/[A-Z0-9]{10}(?:/[^?#]*)?", + "stable_text_anchors": [], + "entry_identity_pattern": "/dp/([A-Z0-9]{10})(?:/|$)", + "name_validation": "nonempty", + "entry_scope": { + "html_selector": "#gridItemRoot", + "title_selector": "a[role=link] [class*=\"line-clamp\"]", + "link_selector": "a[role=link]:has([class*=\"line-clamp\"])" + } + } + }, + "artifact": "", + "ok": true, + "expected_task_pass": false + }, + { + "id": "falabella-product-scope-owned-title-detail", + "category": "structured-extraction", + "root_cause": "product_records_not_shop_navigation_or_empty_destinations", + "failure_mode": "false-negative", + "case": { + "url": "https://www.falabella.com/falabella-cl/category/cat3091/Equipos-de-musica-y-karaokes", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 1, + "hosts": [ + "www.falabella.com" + ], + "entry_path": "/falabella-cl/product/[0-9]+/.+", + "stable_text_anchors": [], + "name_validation": "nonempty", + "entry_identity_pattern": "/product/([0-9]+)(?:/|$)", + "entry_scope": { + "html_selector": "#testId-searchResults-products .grid-pod", + "title_selector": ".pod-subTitle", + "link_selector": "a[data-pod]" + } + } + }, + "artifact": "", + "ok": true, + "expected_task_pass": true + }, + { + "id": "falabella-product-scope-no-valid-product-destination", + "category": "structured-extraction", + "root_cause": "product_records_not_shop_navigation_or_empty_destinations", + "failure_mode": "false-positive", + "case": { + "url": "https://www.falabella.com/falabella-cl/category/cat3091/Equipos-de-musica-y-karaokes", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 1, + "hosts": [ + "www.falabella.com" + ], + "entry_path": "/falabella-cl/product/[0-9]+/.+", + "stable_text_anchors": [], + "name_validation": "nonempty", + "entry_identity_pattern": "/product/([0-9]+)(?:/|$)", + "entry_scope": { + "html_selector": "#testId-searchResults-products .grid-pod", + "title_selector": ".pod-subTitle", + "link_selector": "a[data-pod]" + } + } + }, + "artifact": "", + "ok": true, + "expected_task_pass": false + }, + { + "id": "falabella-product-scope-outside-product-card", + "category": "structured-extraction", + "root_cause": "product_records_not_shop_navigation_or_empty_destinations", + "failure_mode": "false-positive", + "case": { + "url": "https://www.falabella.com/falabella-cl/category/cat3091/Equipos-de-musica-y-karaokes", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 1, + "hosts": [ + "www.falabella.com" + ], + "entry_path": "/falabella-cl/product/[0-9]+/.+", + "stable_text_anchors": [], + "name_validation": "nonempty", + "entry_identity_pattern": "/product/([0-9]+)(?:/|$)", + "entry_scope": { + "html_selector": "#testId-searchResults-products .grid-pod", + "title_selector": ".pod-subTitle", + "link_selector": "a[data-pod]" + } + } + }, + "artifact": "", + "ok": true, + "expected_task_pass": false + }, + { + "id": "falabella-product-scope-missing-title", + "category": "structured-extraction", + "root_cause": "product_records_not_shop_navigation_or_empty_destinations", + "failure_mode": "false-positive", + "case": { + "url": "https://www.falabella.com/falabella-cl/category/cat3091/Equipos-de-musica-y-karaokes", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 1, + "hosts": [ + "www.falabella.com" + ], + "entry_path": "/falabella-cl/product/[0-9]+/.+", + "stable_text_anchors": [], + "name_validation": "nonempty", + "entry_identity_pattern": "/product/([0-9]+)(?:/|$)", + "entry_scope": { + "html_selector": "#testId-searchResults-products .grid-pod", + "title_selector": ".pod-subTitle", + "link_selector": "a[data-pod]" + } + } + }, + "artifact": "
", + "ok": true, + "expected_task_pass": false + }, + { + "id": "lotteon-product-scope-owned-title-detail", + "category": "structured-extraction", + "root_cause": "product_records_not_shop_navigation_or_empty_destinations", + "failure_mode": "false-negative", + "case": { + "url": "https://www.lotteon.com/p/display/main/lotteon", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 1, + "hosts": [ + "www.lotteon.com" + ], + "entry_path": "/p/product/[A-Za-z0-9]+", + "stable_text_anchors": [], + "name_validation": "nonempty", + "entry_identity_pattern": "/p/product/([A-Za-z0-9]+)(?:/|$)", + "entry_scope": { + "html_selector": ".c-product-card", + "title_selector": ".c-product-title__title", + "link_selector": "a.c-product-card__url" + } + } + }, + "artifact": "

새 상품 이름

", + "ok": true, + "expected_task_pass": true + }, + { + "id": "lotteon-product-scope-no-valid-product-destination", + "category": "structured-extraction", + "root_cause": "product_records_not_shop_navigation_or_empty_destinations", + "failure_mode": "false-positive", + "case": { + "url": "https://www.lotteon.com/p/display/main/lotteon", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 1, + "hosts": [ + "www.lotteon.com" + ], + "entry_path": "/p/product/[A-Za-z0-9]+", + "stable_text_anchors": [], + "name_validation": "nonempty", + "entry_identity_pattern": "/p/product/([A-Za-z0-9]+)(?:/|$)", + "entry_scope": { + "html_selector": ".c-product-card", + "title_selector": ".c-product-title__title", + "link_selector": "a.c-product-card__url" + } + } + }, + "artifact": "

새 상품 이름

", + "ok": true, + "expected_task_pass": false + }, + { + "id": "lotteon-product-scope-outside-product-card", + "category": "structured-extraction", + "root_cause": "product_records_not_shop_navigation_or_empty_destinations", + "failure_mode": "false-positive", + "case": { + "url": "https://www.lotteon.com/p/display/main/lotteon", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 1, + "hosts": [ + "www.lotteon.com" + ], + "entry_path": "/p/product/[A-Za-z0-9]+", + "stable_text_anchors": [], + "name_validation": "nonempty", + "entry_identity_pattern": "/p/product/([A-Za-z0-9]+)(?:/|$)", + "entry_scope": { + "html_selector": ".c-product-card", + "title_selector": ".c-product-title__title", + "link_selector": "a.c-product-card__url" + } + } + }, + "artifact": "

새 상품 이름

", + "ok": true, + "expected_task_pass": false + }, + { + "id": "lotteon-product-scope-missing-title", + "category": "structured-extraction", + "root_cause": "product_records_not_shop_navigation_or_empty_destinations", + "failure_mode": "false-positive", + "case": { + "url": "https://www.lotteon.com/p/display/main/lotteon", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 1, + "hosts": [ + "www.lotteon.com" + ], + "entry_path": "/p/product/[A-Za-z0-9]+", + "stable_text_anchors": [], + "name_validation": "nonempty", + "entry_identity_pattern": "/p/product/([A-Za-z0-9]+)(?:/|$)", + "entry_scope": { + "html_selector": ".c-product-card", + "title_selector": ".c-product-title__title", + "link_selector": "a.c-product-card__url" + } + } + }, + "artifact": "

", + "ok": true, + "expected_task_pass": false + }, + { + "id": "dynamic-088-terra-home-new-stories", + "category": "structured-extraction", + "root_cause": "news_detail_routes_not_category_navigation", + "failure_mode": "false-negative", + "case": { + "url": "https://www.terra.com.br/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "www.terra.com.br" + ], + "entry_path": "/.+,[A-Za-z0-9]+\\.html", + "stable_text_anchors": [], + "name_validation": "nonempty", + "entry_scope": { + "html_selector": "a.card-news__text--title,a.card-news__list-item--link,a.t360-carousel-news__card-body-title,a.card-v-mt--embed" + }, + "entry_identity_pattern": ",([A-Za-z0-9]+)\\.html$" + } + }, + "artifact": "[Notícia nova A](https://www.terra.com.br/noticias/mundo/noticia-a,abc123.html)\n[Notícia nova B](https://www.terra.com.br/esportes/noticia-b,def456.html)", + "ok": true, + "expected_task_pass": true + }, + { + "id": "dynamic-088-terra-home-navigation-only", + "category": "structured-extraction", + "root_cause": "news_detail_routes_not_category_navigation", + "failure_mode": "false-positive", + "case": { + "url": "https://www.terra.com.br/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "www.terra.com.br" + ], + "entry_path": "/.+,[A-Za-z0-9]+\\.html", + "stable_text_anchors": [], + "name_validation": "nonempty", + "entry_scope": { + "html_selector": "a.card-news__text--title,a.card-news__list-item--link,a.t360-carousel-news__card-body-title,a.card-v-mt--embed" + }, + "entry_identity_pattern": ",([A-Za-z0-9]+)\\.html$" + } + }, + "artifact": "[Notícias](https://www.terra.com.br/noticias/)\n[Mundo](https://www.terra.com.br/noticias/mundo/)", + "ok": true, + "expected_task_pass": false + }, + { + "id": "dynamic-088-terra-home-duplicate-story", + "category": "structured-extraction", + "root_cause": "news_detail_routes_not_category_navigation", + "failure_mode": "false-positive", + "case": { + "url": "https://www.terra.com.br/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "www.terra.com.br" + ], + "entry_path": "/.+,[A-Za-z0-9]+\\.html", + "stable_text_anchors": [], + "name_validation": "nonempty", + "entry_scope": { + "html_selector": "a.card-news__text--title,a.card-news__list-item--link,a.t360-carousel-news__card-body-title,a.card-v-mt--embed" + }, + "entry_identity_pattern": ",([A-Za-z0-9]+)\\.html$" + } + }, + "artifact": "[Notícia nova A](https://www.terra.com.br/noticias/mundo/noticia-a,abc123.html)\n[Notícia nova B](https://www.terra.com.br/esportes/outro-titulo,abc123.html)", + "ok": true, + "expected_task_pass": false + }, + { + "id": "dynamic-093-pt-real-terra-noticias-new-stories", + "category": "structured-extraction", + "root_cause": "news_detail_routes_not_category_navigation", + "failure_mode": "false-negative", + "case": { + "url": "https://www.terra.com.br/noticias/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "www.terra.com.br" + ], + "entry_path": "/.+,[A-Za-z0-9]+\\.html", + "stable_text_anchors": [], + "name_validation": "nonempty", + "entry_scope": { + "html_selector": "a.card-news__text--title,a.card-news__list-item--link,a.t360-carousel-news__card-body-title,a.card-v-mt--embed" + }, + "entry_identity_pattern": ",([A-Za-z0-9]+)\\.html$" + } + }, + "artifact": "[Notícia nova A](https://www.terra.com.br/noticias/mundo/noticia-a,abc123.html)\n[Notícia nova B](https://www.terra.com.br/esportes/noticia-b,def456.html)", + "ok": true, + "expected_task_pass": true + }, + { + "id": "dynamic-093-pt-real-terra-noticias-navigation-only", + "category": "structured-extraction", + "root_cause": "news_detail_routes_not_category_navigation", + "failure_mode": "false-positive", + "case": { + "url": "https://www.terra.com.br/noticias/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "www.terra.com.br" + ], + "entry_path": "/.+,[A-Za-z0-9]+\\.html", + "stable_text_anchors": [], + "name_validation": "nonempty", + "entry_scope": { + "html_selector": "a.card-news__text--title,a.card-news__list-item--link,a.t360-carousel-news__card-body-title,a.card-v-mt--embed" + }, + "entry_identity_pattern": ",([A-Za-z0-9]+)\\.html$" + } + }, + "artifact": "[Notícias](https://www.terra.com.br/noticias/)\n[Mundo](https://www.terra.com.br/noticias/mundo/)", + "ok": true, + "expected_task_pass": false + }, + { + "id": "dynamic-093-pt-real-terra-noticias-duplicate-story", + "category": "structured-extraction", + "root_cause": "news_detail_routes_not_category_navigation", + "failure_mode": "false-positive", + "case": { + "url": "https://www.terra.com.br/noticias/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "www.terra.com.br" + ], + "entry_path": "/.+,[A-Za-z0-9]+\\.html", + "stable_text_anchors": [], + "name_validation": "nonempty", + "entry_scope": { + "html_selector": "a.card-news__text--title,a.card-news__list-item--link,a.t360-carousel-news__card-body-title,a.card-v-mt--embed" + }, + "entry_identity_pattern": ",([A-Za-z0-9]+)\\.html$" + } + }, + "artifact": "[Notícia nova A](https://www.terra.com.br/noticias/mundo/noticia-a,abc123.html)\n[Notícia nova B](https://www.terra.com.br/esportes/outro-titulo,abc123.html)", + "ok": true, + "expected_task_pass": false + }, + { + "id": "dynamic-096-ar-real-cnn-world-new-stories", + "category": "structured-extraction", + "root_cause": "news_detail_routes_not_category_navigation", + "failure_mode": "false-negative", + "case": { + "url": "https://arabic.cnn.com/world", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "arabic.cnn.com" + ], + "entry_path": "/[^/]+/(?:article|video)/[0-9]{4}/[0-9]{2}/[0-9]{2}/[^/]+", + "stable_text_anchors": [], + "name_validation": "nonempty", + "entry_scope": { + "html_selector": ".container__item:has(.container__headline-text)", + "title_selector": ".container__headline-text", + "link_selector": "a:has(.container__headline-text)" + } + } + }, + "artifact": "[عنوان جديد ألف](/world/article/2026/09/20/new-story-a)\n[عنوان جديد باء](/middle-east/video/2026/09/20/new-story-b)", + "ok": true, + "expected_task_pass": true + }, + { + "id": "dynamic-096-ar-real-cnn-world-navigation-only", + "category": "structured-extraction", + "root_cause": "news_detail_routes_not_category_navigation", + "failure_mode": "false-positive", + "case": { + "url": "https://arabic.cnn.com/world", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "arabic.cnn.com" + ], + "entry_path": "/[^/]+/(?:article|video)/[0-9]{4}/[0-9]{2}/[0-9]{2}/[^/]+", + "stable_text_anchors": [], + "name_validation": "nonempty", + "entry_scope": { + "html_selector": ".container__item:has(.container__headline-text)", + "title_selector": ".container__headline-text", + "link_selector": "a:has(.container__headline-text)" + } + } + }, + "artifact": "[أمريكا](/world/us)\n[روسيا](/world/russia)", + "ok": true, + "expected_task_pass": false + }, + { + "id": "dynamic-096-ar-real-cnn-world-duplicate-story", + "category": "structured-extraction", + "root_cause": "news_detail_routes_not_category_navigation", + "failure_mode": "false-positive", + "case": { + "url": "https://arabic.cnn.com/world", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "arabic.cnn.com" + ], + "entry_path": "/[^/]+/(?:article|video)/[0-9]{4}/[0-9]{2}/[0-9]{2}/[^/]+", + "stable_text_anchors": [], + "name_validation": "nonempty", + "entry_scope": { + "html_selector": ".container__item:has(.container__headline-text)", + "title_selector": ".container__headline-text", + "link_selector": "a:has(.container__headline-text)" + } + } + }, + "artifact": "[عنوان جديد ألف](/world/article/2026/09/20/new-story-a)\n[عنوان جديد باء](/world/article/2026/09/20/new-story-a)", + "ok": true, + "expected_task_pass": false + }, + { + "id": "cnnbrasil-qualified-routes-new-election-and-blog", + "category": "structured-extraction", + "root_cause": "news_detail_routes_not_category_navigation", + "failure_mode": "false-negative", + "case": { + "url": "https://www.cnnbrasil.com.br/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "www.cnnbrasil.com.br" + ], + "entry_path": "(?!/eleicoes/(?:[0-9]{4}/candidatos(?:/|$)|agregador-de-pesquisas(?:-[0-9]{4})?(?:/|$)|monitor-de-debates(?:/|$)))(?!(?:/economia/loterias|/eleicoes/2026/candidatos|/eleicoes/agregador\\-de\\-pesquisas\\-2025|/esportes/agenda|/esportes/automobilismo|/esportes/basquete|/esportes/esports|/esportes/futebol/tabela\\-do\\-brasileirao|/esportes/futebol/tabela\\-do\\-brasileirao\\-serie\\-b|/esportes/futebol/tabela\\-eliminatorias\\-2026|/esportes/olimpiadas|/pop/bbb|/pop/carnaval|/pop/celebridades|/pop/cinema|/pop/musica|/pop/realities|/pop/streaming|/pop/tv|/tudo\\-sobre/africa|/tudo\\-sobre/alimentacao|/tudo\\-sobre/asia|/tudo\\-sobre/auxilio\\-brasil|/tudo\\-sobre/banco\\-central\\-bc|/tudo\\-sobre/bolsa\\-de\\-valores|/tudo\\-sobre/brasilia|/tudo\\-sobre/camara\\-dos\\-deputados|/tudo\\-sobre/cancer|/tudo\\-sobre/china|/tudo\\-sobre/ciencia|/tudo\\-sobre/clima|/tudo\\-sobre/cnn\\-sinais\\-vitais|/tudo\\-sobre/copa\\-libertadores\\-da\\-america|/tudo\\-sobre/curiosidades|/tudo\\-sobre/drogas|/tudo\\-sobre/educacao|/tudo\\-sobre/estados\\-unidos|/tudo\\-sobre/europa|/tudo\\-sobre/exercicios\\-fisicos|/tudo\\-sobre/financas\\-pessoais|/tudo\\-sobre/golfe|/tudo\\-sobre/governo\\-lula|/tudo\\-sobre/guerra\\-de\\-israel|/tudo\\-sobre/guerra\\-na\\-ucrania|/tudo\\-sobre/imposto\\-de\\-renda|/tudo\\-sobre/investimentos|/tudo\\-sobre/meio\\-ambiente|/tudo\\-sobre/mercado|/tudo\\-sobre/minas\\-gerais|/tudo\\-sobre/nasa|/tudo\\-sobre/obesidade|/tudo\\-sobre/oriente\\-medio|/tudo\\-sobre/politicas\\-publicas|/tudo\\-sobre/rio\\-de\\-janeiro\\-estado|/tudo\\-sobre/russia|/tudo\\-sobre/sao\\-paulo\\-estado|/tudo\\-sobre/seguranca|/tudo\\-sobre/senado\\-federal|/tudo\\-sobre/supremo\\-tribunal\\-federal\\-stf|/tudo\\-sobre/sustentabilidade|/tudo\\-sobre/vacinacao|/tudo\\-sobre/volei|/viagemegastronomia/gastronomia|/viagemegastronomia/viagem)$)(?!/esportes/lances-ao-vivo/)/(?:blogs/[^/]+/[^/]+/.+|(?:agro|auto|ciencia|economia|educacao|eleicoes|esportes|infra|internacional|nacional|noticias|politica|pop|saude|style|viagemegastronomia)/.+)", + "stable_text_anchors": [], + "name_validation": "nonempty", + "entry_scope": { + "html_selector": "a:has(h2),a:has(h3),h2 > a,h3 > a" + } + } + }, + "artifact": "[Nova notícia](/eleicoes/nova-noticia/)\n[Nova análise](/blogs/novo-autor/politica/nova-analise/)", + "ok": true, + "expected_task_pass": true + }, + { + "id": "cnnbrasil-qualified-routes-new-science-and-education", + "category": "structured-extraction", + "root_cause": "news_detail_routes_not_category_navigation", + "failure_mode": "false-negative", + "case": { + "url": "https://www.cnnbrasil.com.br/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "www.cnnbrasil.com.br" + ], + "entry_path": "(?!/eleicoes/(?:[0-9]{4}/candidatos(?:/|$)|agregador-de-pesquisas(?:-[0-9]{4})?(?:/|$)|monitor-de-debates(?:/|$)))(?!(?:/economia/loterias|/eleicoes/2026/candidatos|/eleicoes/agregador\\-de\\-pesquisas\\-2025|/esportes/agenda|/esportes/automobilismo|/esportes/basquete|/esportes/esports|/esportes/futebol/tabela\\-do\\-brasileirao|/esportes/futebol/tabela\\-do\\-brasileirao\\-serie\\-b|/esportes/futebol/tabela\\-eliminatorias\\-2026|/esportes/olimpiadas|/pop/bbb|/pop/carnaval|/pop/celebridades|/pop/cinema|/pop/musica|/pop/realities|/pop/streaming|/pop/tv|/tudo\\-sobre/africa|/tudo\\-sobre/alimentacao|/tudo\\-sobre/asia|/tudo\\-sobre/auxilio\\-brasil|/tudo\\-sobre/banco\\-central\\-bc|/tudo\\-sobre/bolsa\\-de\\-valores|/tudo\\-sobre/brasilia|/tudo\\-sobre/camara\\-dos\\-deputados|/tudo\\-sobre/cancer|/tudo\\-sobre/china|/tudo\\-sobre/ciencia|/tudo\\-sobre/clima|/tudo\\-sobre/cnn\\-sinais\\-vitais|/tudo\\-sobre/copa\\-libertadores\\-da\\-america|/tudo\\-sobre/curiosidades|/tudo\\-sobre/drogas|/tudo\\-sobre/educacao|/tudo\\-sobre/estados\\-unidos|/tudo\\-sobre/europa|/tudo\\-sobre/exercicios\\-fisicos|/tudo\\-sobre/financas\\-pessoais|/tudo\\-sobre/golfe|/tudo\\-sobre/governo\\-lula|/tudo\\-sobre/guerra\\-de\\-israel|/tudo\\-sobre/guerra\\-na\\-ucrania|/tudo\\-sobre/imposto\\-de\\-renda|/tudo\\-sobre/investimentos|/tudo\\-sobre/meio\\-ambiente|/tudo\\-sobre/mercado|/tudo\\-sobre/minas\\-gerais|/tudo\\-sobre/nasa|/tudo\\-sobre/obesidade|/tudo\\-sobre/oriente\\-medio|/tudo\\-sobre/politicas\\-publicas|/tudo\\-sobre/rio\\-de\\-janeiro\\-estado|/tudo\\-sobre/russia|/tudo\\-sobre/sao\\-paulo\\-estado|/tudo\\-sobre/seguranca|/tudo\\-sobre/senado\\-federal|/tudo\\-sobre/supremo\\-tribunal\\-federal\\-stf|/tudo\\-sobre/sustentabilidade|/tudo\\-sobre/vacinacao|/tudo\\-sobre/volei|/viagemegastronomia/gastronomia|/viagemegastronomia/viagem)$)(?!/esportes/lances-ao-vivo/)/(?:blogs/[^/]+/[^/]+/.+|(?:agro|auto|ciencia|economia|educacao|eleicoes|esportes|infra|internacional|nacional|noticias|politica|pop|saude|style|viagemegastronomia)/.+)", + "stable_text_anchors": [], + "name_validation": "nonempty", + "entry_scope": { + "html_selector": "a:has(h2),a:has(h3),h2 > a,h3 > a" + } + } + }, + "artifact": "[Ciência nova](/ciencia/descoberta-nova/)\n[Educação nova](/educacao/estudo-novo/)", + "ok": true, + "expected_task_pass": true + }, + { + "id": "cnnbrasil-qualified-routes-category-navigation", + "category": "structured-extraction", + "root_cause": "news_detail_routes_not_category_navigation", + "failure_mode": "false-positive", + "case": { + "url": "https://www.cnnbrasil.com.br/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "www.cnnbrasil.com.br" + ], + "entry_path": "(?!/eleicoes/(?:[0-9]{4}/candidatos(?:/|$)|agregador-de-pesquisas(?:-[0-9]{4})?(?:/|$)|monitor-de-debates(?:/|$)))(?!(?:/economia/loterias|/eleicoes/2026/candidatos|/eleicoes/agregador\\-de\\-pesquisas\\-2025|/esportes/agenda|/esportes/automobilismo|/esportes/basquete|/esportes/esports|/esportes/futebol/tabela\\-do\\-brasileirao|/esportes/futebol/tabela\\-do\\-brasileirao\\-serie\\-b|/esportes/futebol/tabela\\-eliminatorias\\-2026|/esportes/olimpiadas|/pop/bbb|/pop/carnaval|/pop/celebridades|/pop/cinema|/pop/musica|/pop/realities|/pop/streaming|/pop/tv|/tudo\\-sobre/africa|/tudo\\-sobre/alimentacao|/tudo\\-sobre/asia|/tudo\\-sobre/auxilio\\-brasil|/tudo\\-sobre/banco\\-central\\-bc|/tudo\\-sobre/bolsa\\-de\\-valores|/tudo\\-sobre/brasilia|/tudo\\-sobre/camara\\-dos\\-deputados|/tudo\\-sobre/cancer|/tudo\\-sobre/china|/tudo\\-sobre/ciencia|/tudo\\-sobre/clima|/tudo\\-sobre/cnn\\-sinais\\-vitais|/tudo\\-sobre/copa\\-libertadores\\-da\\-america|/tudo\\-sobre/curiosidades|/tudo\\-sobre/drogas|/tudo\\-sobre/educacao|/tudo\\-sobre/estados\\-unidos|/tudo\\-sobre/europa|/tudo\\-sobre/exercicios\\-fisicos|/tudo\\-sobre/financas\\-pessoais|/tudo\\-sobre/golfe|/tudo\\-sobre/governo\\-lula|/tudo\\-sobre/guerra\\-de\\-israel|/tudo\\-sobre/guerra\\-na\\-ucrania|/tudo\\-sobre/imposto\\-de\\-renda|/tudo\\-sobre/investimentos|/tudo\\-sobre/meio\\-ambiente|/tudo\\-sobre/mercado|/tudo\\-sobre/minas\\-gerais|/tudo\\-sobre/nasa|/tudo\\-sobre/obesidade|/tudo\\-sobre/oriente\\-medio|/tudo\\-sobre/politicas\\-publicas|/tudo\\-sobre/rio\\-de\\-janeiro\\-estado|/tudo\\-sobre/russia|/tudo\\-sobre/sao\\-paulo\\-estado|/tudo\\-sobre/seguranca|/tudo\\-sobre/senado\\-federal|/tudo\\-sobre/supremo\\-tribunal\\-federal\\-stf|/tudo\\-sobre/sustentabilidade|/tudo\\-sobre/vacinacao|/tudo\\-sobre/volei|/viagemegastronomia/gastronomia|/viagemegastronomia/viagem)$)(?!/esportes/lances-ao-vivo/)/(?:blogs/[^/]+/[^/]+/.+|(?:agro|auto|ciencia|economia|educacao|eleicoes|esportes|infra|internacional|nacional|noticias|politica|pop|saude|style|viagemegastronomia)/.+)", + "stable_text_anchors": [], + "name_validation": "nonempty", + "entry_scope": { + "html_selector": "a:has(h2),a:has(h3),h2 > a,h3 > a" + } + } + }, + "artifact": "[Cinema](/pop/cinema/)\n[Loterias](/economia/loterias/)", + "ok": true, + "expected_task_pass": false + }, + { + "id": "cnnbrasil-qualified-routes-blog-author-index", + "category": "structured-extraction", + "root_cause": "news_detail_routes_not_category_navigation", + "failure_mode": "false-positive", + "case": { + "url": "https://www.cnnbrasil.com.br/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "www.cnnbrasil.com.br" + ], + "entry_path": "(?!/eleicoes/(?:[0-9]{4}/candidatos(?:/|$)|agregador-de-pesquisas(?:-[0-9]{4})?(?:/|$)|monitor-de-debates(?:/|$)))(?!(?:/economia/loterias|/eleicoes/2026/candidatos|/eleicoes/agregador\\-de\\-pesquisas\\-2025|/esportes/agenda|/esportes/automobilismo|/esportes/basquete|/esportes/esports|/esportes/futebol/tabela\\-do\\-brasileirao|/esportes/futebol/tabela\\-do\\-brasileirao\\-serie\\-b|/esportes/futebol/tabela\\-eliminatorias\\-2026|/esportes/olimpiadas|/pop/bbb|/pop/carnaval|/pop/celebridades|/pop/cinema|/pop/musica|/pop/realities|/pop/streaming|/pop/tv|/tudo\\-sobre/africa|/tudo\\-sobre/alimentacao|/tudo\\-sobre/asia|/tudo\\-sobre/auxilio\\-brasil|/tudo\\-sobre/banco\\-central\\-bc|/tudo\\-sobre/bolsa\\-de\\-valores|/tudo\\-sobre/brasilia|/tudo\\-sobre/camara\\-dos\\-deputados|/tudo\\-sobre/cancer|/tudo\\-sobre/china|/tudo\\-sobre/ciencia|/tudo\\-sobre/clima|/tudo\\-sobre/cnn\\-sinais\\-vitais|/tudo\\-sobre/copa\\-libertadores\\-da\\-america|/tudo\\-sobre/curiosidades|/tudo\\-sobre/drogas|/tudo\\-sobre/educacao|/tudo\\-sobre/estados\\-unidos|/tudo\\-sobre/europa|/tudo\\-sobre/exercicios\\-fisicos|/tudo\\-sobre/financas\\-pessoais|/tudo\\-sobre/golfe|/tudo\\-sobre/governo\\-lula|/tudo\\-sobre/guerra\\-de\\-israel|/tudo\\-sobre/guerra\\-na\\-ucrania|/tudo\\-sobre/imposto\\-de\\-renda|/tudo\\-sobre/investimentos|/tudo\\-sobre/meio\\-ambiente|/tudo\\-sobre/mercado|/tudo\\-sobre/minas\\-gerais|/tudo\\-sobre/nasa|/tudo\\-sobre/obesidade|/tudo\\-sobre/oriente\\-medio|/tudo\\-sobre/politicas\\-publicas|/tudo\\-sobre/rio\\-de\\-janeiro\\-estado|/tudo\\-sobre/russia|/tudo\\-sobre/sao\\-paulo\\-estado|/tudo\\-sobre/seguranca|/tudo\\-sobre/senado\\-federal|/tudo\\-sobre/supremo\\-tribunal\\-federal\\-stf|/tudo\\-sobre/sustentabilidade|/tudo\\-sobre/vacinacao|/tudo\\-sobre/volei|/viagemegastronomia/gastronomia|/viagemegastronomia/viagem)$)(?!/esportes/lances-ao-vivo/)/(?:blogs/[^/]+/[^/]+/.+|(?:agro|auto|ciencia|economia|educacao|eleicoes|esportes|infra|internacional|nacional|noticias|politica|pop|saude|style|viagemegastronomia)/.+)", + "stable_text_anchors": [], + "name_validation": "nonempty", + "entry_scope": { + "html_selector": "a:has(h2),a:has(h3),h2 > a,h3 > a" + } + } + }, + "artifact": "[Autora](/blogs/nova-autora/)\n[Autor](/blogs/novo-autor/)", + "ok": true, + "expected_task_pass": false + }, + { + "id": "cnnbrasil-qualified-routes-scoreboard-only", + "category": "structured-extraction", + "root_cause": "news_detail_routes_not_category_navigation", + "failure_mode": "false-positive", + "case": { + "url": "https://www.cnnbrasil.com.br/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "www.cnnbrasil.com.br" + ], + "entry_path": "(?!/eleicoes/(?:[0-9]{4}/candidatos(?:/|$)|agregador-de-pesquisas(?:-[0-9]{4})?(?:/|$)|monitor-de-debates(?:/|$)))(?!(?:/economia/loterias|/eleicoes/2026/candidatos|/eleicoes/agregador\\-de\\-pesquisas\\-2025|/esportes/agenda|/esportes/automobilismo|/esportes/basquete|/esportes/esports|/esportes/futebol/tabela\\-do\\-brasileirao|/esportes/futebol/tabela\\-do\\-brasileirao\\-serie\\-b|/esportes/futebol/tabela\\-eliminatorias\\-2026|/esportes/olimpiadas|/pop/bbb|/pop/carnaval|/pop/celebridades|/pop/cinema|/pop/musica|/pop/realities|/pop/streaming|/pop/tv|/tudo\\-sobre/africa|/tudo\\-sobre/alimentacao|/tudo\\-sobre/asia|/tudo\\-sobre/auxilio\\-brasil|/tudo\\-sobre/banco\\-central\\-bc|/tudo\\-sobre/bolsa\\-de\\-valores|/tudo\\-sobre/brasilia|/tudo\\-sobre/camara\\-dos\\-deputados|/tudo\\-sobre/cancer|/tudo\\-sobre/china|/tudo\\-sobre/ciencia|/tudo\\-sobre/clima|/tudo\\-sobre/cnn\\-sinais\\-vitais|/tudo\\-sobre/copa\\-libertadores\\-da\\-america|/tudo\\-sobre/curiosidades|/tudo\\-sobre/drogas|/tudo\\-sobre/educacao|/tudo\\-sobre/estados\\-unidos|/tudo\\-sobre/europa|/tudo\\-sobre/exercicios\\-fisicos|/tudo\\-sobre/financas\\-pessoais|/tudo\\-sobre/golfe|/tudo\\-sobre/governo\\-lula|/tudo\\-sobre/guerra\\-de\\-israel|/tudo\\-sobre/guerra\\-na\\-ucrania|/tudo\\-sobre/imposto\\-de\\-renda|/tudo\\-sobre/investimentos|/tudo\\-sobre/meio\\-ambiente|/tudo\\-sobre/mercado|/tudo\\-sobre/minas\\-gerais|/tudo\\-sobre/nasa|/tudo\\-sobre/obesidade|/tudo\\-sobre/oriente\\-medio|/tudo\\-sobre/politicas\\-publicas|/tudo\\-sobre/rio\\-de\\-janeiro\\-estado|/tudo\\-sobre/russia|/tudo\\-sobre/sao\\-paulo\\-estado|/tudo\\-sobre/seguranca|/tudo\\-sobre/senado\\-federal|/tudo\\-sobre/supremo\\-tribunal\\-federal\\-stf|/tudo\\-sobre/sustentabilidade|/tudo\\-sobre/vacinacao|/tudo\\-sobre/volei|/viagemegastronomia/gastronomia|/viagemegastronomia/viagem)$)(?!/esportes/lances-ao-vivo/)/(?:blogs/[^/]+/[^/]+/.+|(?:agro|auto|ciencia|economia|educacao|eleicoes|esportes|infra|internacional|nacional|noticias|politica|pop|saude|style|viagemegastronomia)/.+)", + "stable_text_anchors": [], + "name_validation": "nonempty", + "entry_scope": { + "html_selector": "a:has(h2),a:has(h3),h2 > a,h3 > a" + } + } + }, + "artifact": "[Partida A](/esportes/lances-ao-vivo/liga/2026/a-b/)\n[Partida B](/esportes/lances-ao-vivo/liga/2026/c-d/)", + "ok": true, + "expected_task_pass": false + }, + { + "id": "cnnbrasil-qualified-routes-election-tools", + "category": "structured-extraction", + "root_cause": "news_detail_routes_not_category_navigation", + "failure_mode": "false-positive", + "case": { + "url": "https://www.cnnbrasil.com.br/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "www.cnnbrasil.com.br" + ], + "entry_path": "(?!/eleicoes/(?:[0-9]{4}/candidatos(?:/|$)|agregador-de-pesquisas(?:-[0-9]{4})?(?:/|$)|monitor-de-debates(?:/|$)))(?!(?:/economia/loterias|/eleicoes/2026/candidatos|/eleicoes/agregador\\-de\\-pesquisas\\-2025|/esportes/agenda|/esportes/automobilismo|/esportes/basquete|/esportes/esports|/esportes/futebol/tabela\\-do\\-brasileirao|/esportes/futebol/tabela\\-do\\-brasileirao\\-serie\\-b|/esportes/futebol/tabela\\-eliminatorias\\-2026|/esportes/olimpiadas|/pop/bbb|/pop/carnaval|/pop/celebridades|/pop/cinema|/pop/musica|/pop/realities|/pop/streaming|/pop/tv|/tudo\\-sobre/africa|/tudo\\-sobre/alimentacao|/tudo\\-sobre/asia|/tudo\\-sobre/auxilio\\-brasil|/tudo\\-sobre/banco\\-central\\-bc|/tudo\\-sobre/bolsa\\-de\\-valores|/tudo\\-sobre/brasilia|/tudo\\-sobre/camara\\-dos\\-deputados|/tudo\\-sobre/cancer|/tudo\\-sobre/china|/tudo\\-sobre/ciencia|/tudo\\-sobre/clima|/tudo\\-sobre/cnn\\-sinais\\-vitais|/tudo\\-sobre/copa\\-libertadores\\-da\\-america|/tudo\\-sobre/curiosidades|/tudo\\-sobre/drogas|/tudo\\-sobre/educacao|/tudo\\-sobre/estados\\-unidos|/tudo\\-sobre/europa|/tudo\\-sobre/exercicios\\-fisicos|/tudo\\-sobre/financas\\-pessoais|/tudo\\-sobre/golfe|/tudo\\-sobre/governo\\-lula|/tudo\\-sobre/guerra\\-de\\-israel|/tudo\\-sobre/guerra\\-na\\-ucrania|/tudo\\-sobre/imposto\\-de\\-renda|/tudo\\-sobre/investimentos|/tudo\\-sobre/meio\\-ambiente|/tudo\\-sobre/mercado|/tudo\\-sobre/minas\\-gerais|/tudo\\-sobre/nasa|/tudo\\-sobre/obesidade|/tudo\\-sobre/oriente\\-medio|/tudo\\-sobre/politicas\\-publicas|/tudo\\-sobre/rio\\-de\\-janeiro\\-estado|/tudo\\-sobre/russia|/tudo\\-sobre/sao\\-paulo\\-estado|/tudo\\-sobre/seguranca|/tudo\\-sobre/senado\\-federal|/tudo\\-sobre/supremo\\-tribunal\\-federal\\-stf|/tudo\\-sobre/sustentabilidade|/tudo\\-sobre/vacinacao|/tudo\\-sobre/volei|/viagemegastronomia/gastronomia|/viagemegastronomia/viagem)$)(?!/esportes/lances-ao-vivo/)/(?:blogs/[^/]+/[^/]+/.+|(?:agro|auto|ciencia|economia|educacao|eleicoes|esportes|infra|internacional|nacional|noticias|politica|pop|saude|style|viagemegastronomia)/.+)", + "stable_text_anchors": [], + "name_validation": "nonempty", + "entry_scope": { + "html_selector": "a:has(h2),a:has(h3),h2 > a,h3 > a" + } + } + }, + "artifact": "[Ver estatísticas](/eleicoes/2026/candidatos/estatisticas/)\n[Índice CNN](/eleicoes/agregador-de-pesquisas-2025/)", + "ok": true, + "expected_task_pass": false + }, + { + "id": "cnnbrasil-qualified-routes-duplicate-article", + "category": "structured-extraction", + "root_cause": "news_detail_routes_not_category_navigation", + "failure_mode": "false-positive", + "case": { + "url": "https://www.cnnbrasil.com.br/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "www.cnnbrasil.com.br" + ], + "entry_path": "(?!/eleicoes/(?:[0-9]{4}/candidatos(?:/|$)|agregador-de-pesquisas(?:-[0-9]{4})?(?:/|$)|monitor-de-debates(?:/|$)))(?!(?:/economia/loterias|/eleicoes/2026/candidatos|/eleicoes/agregador\\-de\\-pesquisas\\-2025|/esportes/agenda|/esportes/automobilismo|/esportes/basquete|/esportes/esports|/esportes/futebol/tabela\\-do\\-brasileirao|/esportes/futebol/tabela\\-do\\-brasileirao\\-serie\\-b|/esportes/futebol/tabela\\-eliminatorias\\-2026|/esportes/olimpiadas|/pop/bbb|/pop/carnaval|/pop/celebridades|/pop/cinema|/pop/musica|/pop/realities|/pop/streaming|/pop/tv|/tudo\\-sobre/africa|/tudo\\-sobre/alimentacao|/tudo\\-sobre/asia|/tudo\\-sobre/auxilio\\-brasil|/tudo\\-sobre/banco\\-central\\-bc|/tudo\\-sobre/bolsa\\-de\\-valores|/tudo\\-sobre/brasilia|/tudo\\-sobre/camara\\-dos\\-deputados|/tudo\\-sobre/cancer|/tudo\\-sobre/china|/tudo\\-sobre/ciencia|/tudo\\-sobre/clima|/tudo\\-sobre/cnn\\-sinais\\-vitais|/tudo\\-sobre/copa\\-libertadores\\-da\\-america|/tudo\\-sobre/curiosidades|/tudo\\-sobre/drogas|/tudo\\-sobre/educacao|/tudo\\-sobre/estados\\-unidos|/tudo\\-sobre/europa|/tudo\\-sobre/exercicios\\-fisicos|/tudo\\-sobre/financas\\-pessoais|/tudo\\-sobre/golfe|/tudo\\-sobre/governo\\-lula|/tudo\\-sobre/guerra\\-de\\-israel|/tudo\\-sobre/guerra\\-na\\-ucrania|/tudo\\-sobre/imposto\\-de\\-renda|/tudo\\-sobre/investimentos|/tudo\\-sobre/meio\\-ambiente|/tudo\\-sobre/mercado|/tudo\\-sobre/minas\\-gerais|/tudo\\-sobre/nasa|/tudo\\-sobre/obesidade|/tudo\\-sobre/oriente\\-medio|/tudo\\-sobre/politicas\\-publicas|/tudo\\-sobre/rio\\-de\\-janeiro\\-estado|/tudo\\-sobre/russia|/tudo\\-sobre/sao\\-paulo\\-estado|/tudo\\-sobre/seguranca|/tudo\\-sobre/senado\\-federal|/tudo\\-sobre/supremo\\-tribunal\\-federal\\-stf|/tudo\\-sobre/sustentabilidade|/tudo\\-sobre/vacinacao|/tudo\\-sobre/volei|/viagemegastronomia/gastronomia|/viagemegastronomia/viagem)$)(?!/esportes/lances-ao-vivo/)/(?:blogs/[^/]+/[^/]+/.+|(?:agro|auto|ciencia|economia|educacao|eleicoes|esportes|infra|internacional|nacional|noticias|politica|pop|saude|style|viagemegastronomia)/.+)", + "stable_text_anchors": [], + "name_validation": "nonempty", + "entry_scope": { + "html_selector": "a:has(h2),a:has(h3),h2 > a,h3 > a" + } + } + }, + "artifact": "[Título A](/politica/noticia-a/)\n[Outro título](/politica/noticia-a/)", + "ok": true, + "expected_task_pass": false + }, + { + "id": "naver-politics-owned-visible-visible-owned-cards", + "category": "structured-extraction", + "root_cause": "initial_visible_news_scope_excludes_collapsed_cards", + "failure_mode": "false-negative", + "case": { + "url": "https://news.naver.com/section/100", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "n.news.naver.com" + ], + "entry_path": "/(?:mnews/)?article/[0-9]+/[0-9]+", + "stable_text_anchors": [], + "name_validation": "nonempty", + "entry_identity_pattern": "/article/([0-9]+/[0-9]+)(?:/|$)", + "entry_scope": { + "html_selector": "#newsct .sa_item:not(.is_blind)", + "title_selector": ".sa_text_strong", + "link_selector": "a.sa_text_title", + "start_pattern": "헤드라인 뉴스", + "end_pattern": "언론사별 가장 많이 본 뉴스" + } + } + }, + "artifact": "", + "ok": true, + "expected_task_pass": true + }, + { + "id": "naver-politics-owned-visible-hidden-expansion-not-denominator", + "category": "structured-extraction", + "root_cause": "initial_visible_news_scope_excludes_collapsed_cards", + "failure_mode": "false-positive", + "case": { + "url": "https://news.naver.com/section/100", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "n.news.naver.com" + ], + "entry_path": "/(?:mnews/)?article/[0-9]+/[0-9]+", + "stable_text_anchors": [], + "name_validation": "nonempty", + "entry_identity_pattern": "/article/([0-9]+/[0-9]+)(?:/|$)", + "entry_scope": { + "html_selector": "#newsct .sa_item:not(.is_blind)", + "title_selector": ".sa_text_strong", + "link_selector": "a.sa_text_title", + "start_pattern": "헤드라인 뉴스", + "end_pattern": "언론사별 가장 많이 본 뉴스" + } + } + }, + "artifact": "", + "ok": true, + "expected_task_pass": false + }, + { + "id": "naver-politics-owned-visible-empty-title", + "category": "structured-extraction", + "root_cause": "initial_visible_news_scope_excludes_collapsed_cards", + "failure_mode": "false-positive", + "case": { + "url": "https://news.naver.com/section/100", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "n.news.naver.com" + ], + "entry_path": "/(?:mnews/)?article/[0-9]+/[0-9]+", + "stable_text_anchors": [], + "name_validation": "nonempty", + "entry_identity_pattern": "/article/([0-9]+/[0-9]+)(?:/|$)", + "entry_scope": { + "html_selector": "#newsct .sa_item:not(.is_blind)", + "title_selector": ".sa_text_strong", + "link_selector": "a.sa_text_title", + "start_pattern": "헤드라인 뉴스", + "end_pattern": "언론사별 가장 많이 본 뉴스" + } + } + }, + "artifact": "", + "ok": true, + "expected_task_pass": false + }, + { + "id": "naver-politics-owned-visible-sidebar-not-main", + "category": "structured-extraction", + "root_cause": "initial_visible_news_scope_excludes_collapsed_cards", + "failure_mode": "false-positive", + "case": { + "url": "https://news.naver.com/section/100", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "n.news.naver.com" + ], + "entry_path": "/(?:mnews/)?article/[0-9]+/[0-9]+", + "stable_text_anchors": [], + "name_validation": "nonempty", + "entry_identity_pattern": "/article/([0-9]+/[0-9]+)(?:/|$)", + "entry_scope": { + "html_selector": "#newsct .sa_item:not(.is_blind)", + "title_selector": ".sa_text_strong", + "link_selector": "a.sa_text_title", + "start_pattern": "헤드라인 뉴스", + "end_pattern": "언론사별 가장 많이 본 뉴스" + } + } + }, + "artifact": "", + "ok": true, + "expected_task_pass": false + }, + { + "id": "naver-politics-owned-visible-fresh-markdown", + "category": "structured-extraction", + "root_cause": "initial_visible_news_scope_excludes_collapsed_cards", + "failure_mode": "false-negative", + "case": { + "url": "https://news.naver.com/section/100", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "n.news.naver.com" + ], + "entry_path": "/(?:mnews/)?article/[0-9]+/[0-9]+", + "stable_text_anchors": [], + "name_validation": "nonempty", + "entry_identity_pattern": "/article/([0-9]+/[0-9]+)(?:/|$)", + "entry_scope": { + "html_selector": "#newsct .sa_item:not(.is_blind)", + "title_selector": ".sa_text_strong", + "link_selector": "a.sa_text_title", + "start_pattern": "헤드라인 뉴스", + "end_pattern": "언론사별 가장 많이 본 뉴스" + } + } + }, + "artifact": "헤드라인 뉴스\n[Fresh A](https://n.news.naver.com/mnews/article/001/789)\n[Fresh B](https://n.news.naver.com/mnews/article/002/987)\n언론사별 가장 많이 본 뉴스", + "ok": true, + "expected_task_pass": true + }, + { + "id": "naver-politics-owned-visible-repeated-id", + "category": "structured-extraction", + "root_cause": "initial_visible_news_scope_excludes_collapsed_cards", + "failure_mode": "false-positive", + "case": { + "url": "https://news.naver.com/section/100", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "n.news.naver.com" + ], + "entry_path": "/(?:mnews/)?article/[0-9]+/[0-9]+", + "stable_text_anchors": [], + "name_validation": "nonempty", + "entry_identity_pattern": "/article/([0-9]+/[0-9]+)(?:/|$)", + "entry_scope": { + "html_selector": "#newsct .sa_item:not(.is_blind)", + "title_selector": ".sa_text_strong", + "link_selector": "a.sa_text_title", + "start_pattern": "헤드라인 뉴스", + "end_pattern": "언론사별 가장 많이 본 뉴스" + } + } + }, + "artifact": "", + "ok": true, + "expected_task_pass": false + }, + { + "id": "owned-anchor-card-anchor-is-card", + "category": "structured-extraction", + "root_cause": "card_anchor_owns_visible_title_and_destination", + "failure_mode": "false-negative", + "case": { + "url": "https://www.kabum.com.br/hardware", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "www.kabum.com.br" + ], + "entry_path": "/produto/\\d+/.+", + "stable_text_anchors": [], + "name_validation": "nonempty", + "entry_identity_pattern": "/produto/([0-9]+)(?:/|$)", + "entry_scope": { + "html_selector": "a[href*=\"/produto/\"]:has(.line-clamp-2)", + "title_selector": ".line-clamp-2", + "link_selector": "a[href]" + } + } + }, + "artifact": "Fresh productFresh product", + "ok": true, + "expected_task_pass": true + }, + { + "id": "owned-anchor-card-empty-title-with-price", + "category": "structured-extraction", + "root_cause": "card_anchor_owns_visible_title_and_destination", + "failure_mode": "false-positive", + "case": { + "url": "https://www.kabum.com.br/hardware", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "www.kabum.com.br" + ], + "entry_path": "/produto/\\d+/.+", + "stable_text_anchors": [], + "name_validation": "nonempty", + "entry_identity_pattern": "/produto/([0-9]+)(?:/|$)", + "entry_scope": { + "html_selector": "a[href*=\"/produto/\"]:has(.line-clamp-2)", + "title_selector": ".line-clamp-2", + "link_selector": "a[href]" + } + } + }, + "artifact": "Fresh productR$ 100", + "ok": true, + "expected_task_pass": false + }, + { + "id": "owned-anchor-card-price-only", + "category": "structured-extraction", + "root_cause": "card_anchor_owns_visible_title_and_destination", + "failure_mode": "false-positive", + "case": { + "url": "https://www.kabum.com.br/hardware", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "www.kabum.com.br" + ], + "entry_path": "/produto/\\d+/.+", + "stable_text_anchors": [], + "name_validation": "nonempty", + "entry_identity_pattern": "/produto/([0-9]+)(?:/|$)", + "entry_scope": { + "html_selector": "a[href*=\"/produto/\"]:has(.line-clamp-2)", + "title_selector": ".line-clamp-2", + "link_selector": "a[href]" + } + } + }, + "artifact": "R$ 100R$ 200", + "ok": true, + "expected_task_pass": false + }, + { + "id": "owned-anchor-card-duplicate-id", + "category": "structured-extraction", + "root_cause": "card_anchor_owns_visible_title_and_destination", + "failure_mode": "false-positive", + "case": { + "url": "https://www.kabum.com.br/hardware", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "www.kabum.com.br" + ], + "entry_path": "/produto/\\d+/.+", + "stable_text_anchors": [], + "name_validation": "nonempty", + "entry_identity_pattern": "/produto/([0-9]+)(?:/|$)", + "entry_scope": { + "html_selector": "a[href*=\"/produto/\"]:has(.line-clamp-2)", + "title_selector": ".line-clamp-2", + "link_selector": "a[href]" + } + } + }, + "artifact": "Fresh productDifferent text", + "ok": true, + "expected_task_pass": false + }, + { + "id": "owned-anchor-card-fresh-markdown", + "category": "structured-extraction", + "root_cause": "card_anchor_owns_visible_title_and_destination", + "failure_mode": "false-negative", + "case": { + "url": "https://www.kabum.com.br/hardware", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "www.kabum.com.br" + ], + "entry_path": "/produto/\\d+/.+", + "stable_text_anchors": [], + "name_validation": "nonempty", + "entry_identity_pattern": "/produto/([0-9]+)(?:/|$)", + "entry_scope": { + "html_selector": "a[href*=\"/produto/\"]:has(.line-clamp-2)", + "title_selector": ".line-clamp-2", + "link_selector": "a[href]" + } + } + }, + "artifact": "[New product A](/produto/123/a)\n[New product B](/produto/456/b)", + "ok": true, + "expected_task_pass": true + }, + { + "id": "kabum-privacy-footer-without-consent", + "category": "structured-extraction", + "root_cause": "optional_consent_label_not_required_footer_text", + "failure_mode": "false-negative", + "case": { + "url": "https://www.kabum.com.br/hardware", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 1, + "hosts": [ + "www.kabum.com.br" + ], + "entry_path": "/produto/\\d+/.+", + "stable_text_anchors": [ + { + "id": "privacy-footer", + "alternatives": [ + "Políticas de Privacidade" + ], + "match_scope": "non_record_lines" } - }, - "artifact": "R$ 100R$ 200", - "ok": true, - "expected_task_pass": false - }, - { - "id": "owned-anchor-card-duplicate-id", - "category": "structured-extraction", - "root_cause": "card_anchor_owns_visible_title_and_destination", - "failure_mode": "false-positive", - "case": { - "url": "https://www.kabum.com.br/hardware", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 2, - "hosts": [ - "www.kabum.com.br" - ], - "entry_path": "/produto/\\d+/.+", - "stable_text_anchors": [], - "name_validation": "nonempty", - "entry_identity_pattern": "/produto/([0-9]+)(?:/|$)", - "entry_scope": { - "html_selector": "a[href*=\"/produto/\"]:has(.line-clamp-2)", - "title_selector": ".line-clamp-2", - "link_selector": "a[href]" - } + ], + "name_validation": "nonempty", + "entry_identity_pattern": "/produto/([0-9]+)(?:/|$)", + "entry_scope": { + "html_selector": "a[href*=\"/produto/\"]:has(.line-clamp-2)", + "title_selector": ".line-clamp-2", + "link_selector": "a[href]" + } + } + }, + "artifact": "[Políticas de Privacidade](/privacidade)\n[Fresh product](/produto/123/a)", + "ok": true, + "expected_task_pass": true + }, + { + "id": "kabum-privacy-privacy-only", + "category": "structured-extraction", + "root_cause": "optional_consent_label_not_required_footer_text", + "failure_mode": "false-positive", + "case": { + "url": "https://www.kabum.com.br/hardware", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 1, + "hosts": [ + "www.kabum.com.br" + ], + "entry_path": "/produto/\\d+/.+", + "stable_text_anchors": [ + { + "id": "privacy-footer", + "alternatives": [ + "Políticas de Privacidade" + ], + "match_scope": "non_record_lines" } - }, - "artifact": "Fresh productDifferent text", - "ok": true, - "expected_task_pass": false - }, - { - "id": "owned-anchor-card-fresh-markdown", - "category": "structured-extraction", - "root_cause": "card_anchor_owns_visible_title_and_destination", - "failure_mode": "false-negative", - "case": { - "url": "https://www.kabum.com.br/hardware", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 2, - "hosts": [ - "www.kabum.com.br" - ], - "entry_path": "/produto/\\d+/.+", - "stable_text_anchors": [], - "name_validation": "nonempty", - "entry_identity_pattern": "/produto/([0-9]+)(?:/|$)", - "entry_scope": { - "html_selector": "a[href*=\"/produto/\"]:has(.line-clamp-2)", - "title_selector": ".line-clamp-2", - "link_selector": "a[href]" - } + ], + "name_validation": "nonempty", + "entry_identity_pattern": "/produto/([0-9]+)(?:/|$)", + "entry_scope": { + "html_selector": "a[href*=\"/produto/\"]:has(.line-clamp-2)", + "title_selector": ".line-clamp-2", + "link_selector": "a[href]" + } + } + }, + "artifact": "[Políticas de Privacidade](/privacidade)", + "ok": true, + "expected_task_pass": false + }, + { + "id": "kabum-privacy-product-cannot-impersonate-footer", + "category": "structured-extraction", + "root_cause": "optional_consent_label_not_required_footer_text", + "failure_mode": "false-positive", + "case": { + "url": "https://www.kabum.com.br/hardware", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 1, + "hosts": [ + "www.kabum.com.br" + ], + "entry_path": "/produto/\\d+/.+", + "stable_text_anchors": [ + { + "id": "privacy-footer", + "alternatives": [ + "Políticas de Privacidade" + ], + "match_scope": "non_record_lines" } - }, - "artifact": "[New product A](/produto/123/a)\n[New product B](/produto/456/b)", - "ok": true, - "expected_task_pass": true - }, - { - "id": "kabum-privacy-footer-without-consent", - "category": "structured-extraction", - "root_cause": "optional_consent_label_not_required_footer_text", - "failure_mode": "false-negative", - "case": { - "url": "https://www.kabum.com.br/hardware", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 1, - "hosts": [ - "www.kabum.com.br" - ], - "entry_path": "/produto/\\d+/.+", - "stable_text_anchors": [ - { - "id": "privacy-footer", - "alternatives": [ - "Políticas de Privacidade" - ], - "match_scope": "non_record_lines" - } - ], - "name_validation": "nonempty", - "entry_identity_pattern": "/produto/([0-9]+)(?:/|$)", - "entry_scope": { - "html_selector": "a[href*=\"/produto/\"]:has(.line-clamp-2)", - "title_selector": ".line-clamp-2", - "link_selector": "a[href]" - } + ], + "name_validation": "nonempty", + "entry_identity_pattern": "/produto/([0-9]+)(?:/|$)", + "entry_scope": { + "html_selector": "a[href*=\"/produto/\"]:has(.line-clamp-2)", + "title_selector": ".line-clamp-2", + "link_selector": "a[href]" + } + } + }, + "artifact": "[Políticas de Privacidade](/produto/123/a)", + "ok": true, + "expected_task_pass": false + }, + { + "id": "kooora-news-scope-new-news-cards", + "category": "structured-extraction", + "root_cause": "news_detail_routes_not_category_navigation", + "failure_mode": "false-negative", + "case": { + "url": "https://www.kooora.com/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "www.kooora.com" + ], + "entry_path": "/[^/]+/(?:أخبار|القوائم|%D8%A3%D8%AE%D8%A8%D8%A7%D8%B1|%D8%A7%D9%84%D9%82%D9%88%D8%A7%D8%A6%D9%85)/[^/]+/blt[0-9a-f]+", + "stable_text_anchors": [], + "name_validation": "nonempty", + "entry_identity_pattern": "/(blt[0-9a-f]+)$", + "entry_scope": { + "html_selector": "a:is(:has(.fco-card__headline-text), :has(.fco-high-impact-card__headline)):not(.fco-selective-scroll-section__item--hidden *):not([class*=\"videos-section_videos-section__container\"] [class*=\"scrollable-section_scrollable-section\"] *)", + "title_selector": ".fco-card__headline-text, .fco-high-impact-card__headline", + "link_selector": "a[href]" + } + } + }, + "artifact": "خبر جديدخبر جديد", + "ok": true, + "expected_task_pass": true + }, + { + "id": "kooora-news-scope-teams-and-players", + "category": "structured-extraction", + "root_cause": "news_detail_routes_not_category_navigation", + "failure_mode": "false-positive", + "case": { + "url": "https://www.kooora.com/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "www.kooora.com" + ], + "entry_path": "/[^/]+/(?:أخبار|القوائم|%D8%A3%D8%AE%D8%A8%D8%A7%D8%B1|%D8%A7%D9%84%D9%82%D9%88%D8%A7%D8%A6%D9%85)/[^/]+/blt[0-9a-f]+", + "stable_text_anchors": [], + "name_validation": "nonempty", + "entry_identity_pattern": "/(blt[0-9a-f]+)$", + "entry_scope": { + "html_selector": "a:is(:has(.fco-card__headline-text), :has(.fco-high-impact-card__headline)):not(.fco-selective-scroll-section__item--hidden *):not([class*=\"videos-section_videos-section__container\"] [class*=\"scrollable-section_scrollable-section\"] *)", + "title_selector": ".fco-card__headline-text, .fco-high-impact-card__headline", + "link_selector": "a[href]" + } + } + }, + "artifact": "[Team](/كرة-القدم/فريق/name/123)\n[Player](/كرة-القدم/لاعب/name/أخبار/456)", + "ok": true, + "expected_task_pass": false + }, + { + "id": "kooora-news-scope-category-index", + "category": "structured-extraction", + "root_cause": "news_detail_routes_not_category_navigation", + "failure_mode": "false-positive", + "case": { + "url": "https://www.kooora.com/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "www.kooora.com" + ], + "entry_path": "/[^/]+/(?:أخبار|القوائم|%D8%A3%D8%AE%D8%A8%D8%A7%D8%B1|%D8%A7%D9%84%D9%82%D9%88%D8%A7%D8%A6%D9%85)/[^/]+/blt[0-9a-f]+", + "stable_text_anchors": [], + "name_validation": "nonempty", + "entry_identity_pattern": "/(blt[0-9a-f]+)$", + "entry_scope": { + "html_selector": "a:is(:has(.fco-card__headline-text), :has(.fco-high-impact-card__headline)):not(.fco-selective-scroll-section__item--hidden *):not([class*=\"videos-section_videos-section__container\"] [class*=\"scrollable-section_scrollable-section\"] *)", + "title_selector": ".fco-card__headline-text, .fco-high-impact-card__headline", + "link_selector": "a[href]" + } + } + }, + "artifact": "[Tennis](/تنس/bltabc123)\n[News](/أخبار)", + "ok": true, + "expected_task_pass": false + }, + { + "id": "kooora-news-scope-collapsed-card", + "category": "structured-extraction", + "root_cause": "news_detail_routes_not_category_navigation", + "failure_mode": "false-positive", + "case": { + "url": "https://www.kooora.com/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "www.kooora.com" + ], + "entry_path": "/[^/]+/(?:أخبار|القوائم|%D8%A3%D8%AE%D8%A8%D8%A7%D8%B1|%D8%A7%D9%84%D9%82%D9%88%D8%A7%D8%A6%D9%85)/[^/]+/blt[0-9a-f]+", + "stable_text_anchors": [], + "name_validation": "nonempty", + "entry_identity_pattern": "/(blt[0-9a-f]+)$", + "entry_scope": { + "html_selector": "a:is(:has(.fco-card__headline-text), :has(.fco-high-impact-card__headline)):not(.fco-selective-scroll-section__item--hidden *):not([class*=\"videos-section_videos-section__container\"] [class*=\"scrollable-section_scrollable-section\"] *)", + "title_selector": ".fco-card__headline-text, .fco-high-impact-card__headline", + "link_selector": "a[href]" + } + } + }, + "artifact": "خبر جديد
خبر جديد
", + "ok": true, + "expected_task_pass": false + }, + { + "id": "kooora-news-scope-same-id-new-slug", + "category": "structured-extraction", + "root_cause": "news_detail_routes_not_category_navigation", + "failure_mode": "false-positive", + "case": { + "url": "https://www.kooora.com/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "www.kooora.com" + ], + "entry_path": "/[^/]+/(?:أخبار|القوائم|%D8%A3%D8%AE%D8%A8%D8%A7%D8%B1|%D8%A7%D9%84%D9%82%D9%88%D8%A7%D8%A6%D9%85)/[^/]+/blt[0-9a-f]+", + "stable_text_anchors": [], + "name_validation": "nonempty", + "entry_identity_pattern": "/(blt[0-9a-f]+)$", + "entry_scope": { + "html_selector": "a:is(:has(.fco-card__headline-text), :has(.fco-high-impact-card__headline)):not(.fco-selective-scroll-section__item--hidden *):not([class*=\"videos-section_videos-section__container\"] [class*=\"scrollable-section_scrollable-section\"] *)", + "title_selector": ".fco-card__headline-text, .fco-high-impact-card__headline", + "link_selector": "a[href]" + } + } + }, + "artifact": "[First](/كرة-قدم/أخبار/title/bltabc123)\n[Second](/كرة-قدم/أخبار/changed-title/bltabc123)", + "ok": true, + "expected_task_pass": false + }, + { + "id": "kooora-news-scope-fresh-listicle", + "category": "structured-extraction", + "root_cause": "news_detail_routes_not_category_navigation", + "failure_mode": "false-negative", + "case": { + "url": "https://www.kooora.com/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "www.kooora.com" + ], + "entry_path": "/[^/]+/(?:أخبار|القوائم|%D8%A3%D8%AE%D8%A8%D8%A7%D8%B1|%D8%A7%D9%84%D9%82%D9%88%D8%A7%D8%A6%D9%85)/[^/]+/blt[0-9a-f]+", + "stable_text_anchors": [], + "name_validation": "nonempty", + "entry_identity_pattern": "/(blt[0-9a-f]+)$", + "entry_scope": { + "html_selector": "a:is(:has(.fco-card__headline-text), :has(.fco-high-impact-card__headline)):not(.fco-selective-scroll-section__item--hidden *):not([class*=\"videos-section_videos-section__container\"] [class*=\"scrollable-section_scrollable-section\"] *)", + "title_selector": ".fco-card__headline-text, .fco-high-impact-card__headline", + "link_selector": "a[href]" + } + } + }, + "artifact": "[News](/كرة-قدم/أخبار/new/bltabc123)\n[Listicle](/كرة-قدم/القوائم/new/bltdef456)", + "ok": true, + "expected_task_pass": true + }, + { + "id": "quake-joined-location-time", + "category": "structured-extraction", + "root_cause": "earthquake_inline_location_timestamp", + "failure_mode": "false-negative", + "case": { + "url": "https://example.test/earthquakes", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "earthquake", + "minimum_records": 2 + } + }, + "artifact": "4.2\nHarbor Island2026-09-20 12:00:00 (UTC+08:00)\n10.0 km\n3.1\nNorth Valley2026-09-20T05:00:00Z\n7.4 km", + "ok": true, + "expected_task_pass": true + }, + { + "id": "quake-joined-missing-depth", + "category": "structured-extraction", + "root_cause": "earthquake_inline_location_timestamp", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/earthquakes", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "earthquake", + "minimum_records": 2 + } + }, + "artifact": "4.2\nHarbor Island2026-09-20 12:00:00 (UTC+08:00)\n3.1\nNorth Valley2026-09-20T05:00:00Z\n7.4 km", + "ok": true, + "expected_task_pass": false + }, + { + "id": "quake-joined-invalid-time", + "category": "structured-extraction", + "root_cause": "earthquake_inline_location_timestamp", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/earthquakes", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "earthquake", + "minimum_records": 2 + } + }, + "artifact": "4.2\nHarbor Island2026-99-20 12:00:00 (UTC+08:00)\n10.0 km\n3.1\nNorth Valley2026-09-20T05:00:00Z\n7.4 km", + "ok": true, + "expected_task_pass": false + }, + { + "id": "quake-joined-duplicate-record", + "category": "structured-extraction", + "root_cause": "earthquake_inline_location_timestamp", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/earthquakes", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "earthquake", + "minimum_records": 2 + } + }, + "artifact": "4.2\nHarbor Island2026-09-20 12:00:00 (UTC+08:00)\n10.0 km\n4.2\nHarbor Island2026-09-20 12:00:00 (UTC+08:00)\n10.0 km", + "ok": true, + "expected_task_pass": false + }, + { + "id": "dynamic22-037-complete", + "category": "structured-extraction", + "root_cause": "dynamic_initial_scope_and_owned_records", + "failure_mode": "false-negative", + "case": { + "url": "https://quasar.dev/vue-components/virtual-scroll/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 8, + "table_columns": [ + { + "name": "item", + "label": "Option", + "type": "text", + "pattern": "#?\\d+\\s*-\\s*Option\\s+\\d+" } - }, - "artifact": "[Políticas de Privacidade](/privacidade)\n[Fresh product](/produto/123/a)", - "ok": true, - "expected_task_pass": true - }, - { - "id": "kabum-privacy-privacy-only", - "category": "structured-extraction", - "root_cause": "optional_consent_label_not_required_footer_text", - "failure_mode": "false-positive", - "case": { - "url": "https://www.kabum.com.br/hardware", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 1, - "hosts": [ - "www.kabum.com.br" - ], - "entry_path": "/produto/\\d+/.+", - "stable_text_anchors": [ - { - "id": "privacy-footer", - "alternatives": [ - "Políticas de Privacidade" - ], - "match_scope": "non_record_lines" - } - ], - "name_validation": "nonempty", - "entry_identity_pattern": "/produto/([0-9]+)(?:/|$)", - "entry_scope": { - "html_selector": "a[href*=\"/produto/\"]:has(.line-clamp-2)", - "title_selector": ".line-clamp-2", - "link_selector": "a[href]" - } + ], + "record_identity_fields": [ + "item" + ], + "table_scope": { + "root_selector": ".q-virtual-scroll", + "root_index": 0, + "row_selector": ".q-virtual-scroll__content > .q-item", + "cell_selector": ":scope", + "start_pattern": "(?m)^\\s*#{0,6}\\s*Basic\\s*$", + "end_pattern": "(?m)^[ \\t#]*Horizontal[ \\t]*$", + "plain_row_pattern": "(?P#?\\d+\\s*-\\s*Option\\s+\\d+)", + "reference_kind": "fixed_demo", + "reference_rows": [ + { + "item": "#0 - Option 1" + }, + { + "item": "#1 - Option 2" + }, + { + "item": "#2 - Option 3" + }, + { + "item": "#3 - Option 4" + }, + { + "item": "#4 - Option 5" + }, + { + "item": "#5 - Option 6" + }, + { + "item": "#6 - Option 7" + }, + { + "item": "#7 - Option 8" + }, + { + "item": "#8 - Option 9" + }, + { + "item": "#9 - Option 10" + } + ] + } + } + }, + "artifact": "
#0 - Option 1
#1 - Option 2
#2 - Option 3
#3 - Option 4
#4 - Option 5
#5 - Option 6
#6 - Option 7
#7 - Option 8
", + "ok": true, + "expected_task_pass": true + }, + { + "id": "dynamic22-037-below80", + "category": "structured-extraction", + "root_cause": "dynamic_initial_scope_and_owned_records", + "failure_mode": "false-positive", + "case": { + "url": "https://quasar.dev/vue-components/virtual-scroll/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 8, + "table_columns": [ + { + "name": "item", + "label": "Option", + "type": "text", + "pattern": "#?\\d+\\s*-\\s*Option\\s+\\d+" } - }, - "artifact": "[Políticas de Privacidade](/privacidade)", - "ok": true, - "expected_task_pass": false - }, - { - "id": "kabum-privacy-product-cannot-impersonate-footer", - "category": "structured-extraction", - "root_cause": "optional_consent_label_not_required_footer_text", - "failure_mode": "false-positive", - "case": { - "url": "https://www.kabum.com.br/hardware", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 1, - "hosts": [ - "www.kabum.com.br" - ], - "entry_path": "/produto/\\d+/.+", - "stable_text_anchors": [ - { - "id": "privacy-footer", - "alternatives": [ - "Políticas de Privacidade" - ], - "match_scope": "non_record_lines" - } - ], - "name_validation": "nonempty", - "entry_identity_pattern": "/produto/([0-9]+)(?:/|$)", - "entry_scope": { - "html_selector": "a[href*=\"/produto/\"]:has(.line-clamp-2)", - "title_selector": ".line-clamp-2", - "link_selector": "a[href]" - } + ], + "record_identity_fields": [ + "item" + ], + "table_scope": { + "root_selector": ".q-virtual-scroll", + "root_index": 0, + "row_selector": ".q-virtual-scroll__content > .q-item", + "cell_selector": ":scope", + "start_pattern": "(?m)^\\s*#{0,6}\\s*Basic\\s*$", + "end_pattern": "(?m)^[ \\t#]*Horizontal[ \\t]*$", + "plain_row_pattern": "(?P#?\\d+\\s*-\\s*Option\\s+\\d+)", + "reference_kind": "fixed_demo", + "reference_rows": [ + { + "item": "#0 - Option 1" + }, + { + "item": "#1 - Option 2" + }, + { + "item": "#2 - Option 3" + }, + { + "item": "#3 - Option 4" + }, + { + "item": "#4 - Option 5" + }, + { + "item": "#5 - Option 6" + }, + { + "item": "#6 - Option 7" + }, + { + "item": "#7 - Option 8" + }, + { + "item": "#8 - Option 9" + }, + { + "item": "#9 - Option 10" + } + ] + } + } + }, + "artifact": "
#0 - Option 1
#1 - Option 2
#2 - Option 3
#3 - Option 4
#4 - Option 5
#5 - Option 6
#6 - Option 7
", + "ok": true, + "expected_task_pass": false + }, + { + "id": "dynamic22-037-duplicate", + "category": "structured-extraction", + "root_cause": "dynamic_initial_scope_and_owned_records", + "failure_mode": "false-positive", + "case": { + "url": "https://quasar.dev/vue-components/virtual-scroll/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 8, + "table_columns": [ + { + "name": "item", + "label": "Option", + "type": "text", + "pattern": "#?\\d+\\s*-\\s*Option\\s+\\d+" } - }, - "artifact": "[Políticas de Privacidade](/produto/123/a)", - "ok": true, - "expected_task_pass": false - }, - { - "id": "kooora-news-scope-new-news-cards", - "category": "structured-extraction", - "root_cause": "news_detail_routes_not_category_navigation", - "failure_mode": "false-negative", - "case": { - "url": "https://www.kooora.com/", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 2, - "hosts": [ - "www.kooora.com" - ], - "entry_path": "/[^/]+/(?:أخبار|القوائم|%D8%A3%D8%AE%D8%A8%D8%A7%D8%B1|%D8%A7%D9%84%D9%82%D9%88%D8%A7%D8%A6%D9%85)/[^/]+/blt[0-9a-f]+", - "stable_text_anchors": [], - "name_validation": "nonempty", - "entry_identity_pattern": "/(blt[0-9a-f]+)$", - "entry_scope": { - "html_selector": "a:is(:has(.fco-card__headline-text), :has(.fco-high-impact-card__headline)):not(.fco-selective-scroll-section__item--hidden *):not([class*=\"videos-section_videos-section__container\"] [class*=\"scrollable-section_scrollable-section\"] *)", - "title_selector": ".fco-card__headline-text, .fco-high-impact-card__headline", - "link_selector": "a[href]" - } + ], + "record_identity_fields": [ + "item" + ], + "table_scope": { + "root_selector": ".q-virtual-scroll", + "root_index": 0, + "row_selector": ".q-virtual-scroll__content > .q-item", + "cell_selector": ":scope", + "start_pattern": "(?m)^\\s*#{0,6}\\s*Basic\\s*$", + "end_pattern": "(?m)^[ \\t#]*Horizontal[ \\t]*$", + "plain_row_pattern": "(?P#?\\d+\\s*-\\s*Option\\s+\\d+)", + "reference_kind": "fixed_demo", + "reference_rows": [ + { + "item": "#0 - Option 1" + }, + { + "item": "#1 - Option 2" + }, + { + "item": "#2 - Option 3" + }, + { + "item": "#3 - Option 4" + }, + { + "item": "#4 - Option 5" + }, + { + "item": "#5 - Option 6" + }, + { + "item": "#6 - Option 7" + }, + { + "item": "#7 - Option 8" + }, + { + "item": "#8 - Option 9" + }, + { + "item": "#9 - Option 10" + } + ] + } + } + }, + "artifact": "
#0 - Option 1
#0 - Option 1
#0 - Option 1
#0 - Option 1
#0 - Option 1
#0 - Option 1
#0 - Option 1
#0 - Option 1
", + "ok": true, + "expected_task_pass": false + }, + { + "id": "dynamic22-037-wrong-scope", + "category": "structured-extraction", + "root_cause": "dynamic_initial_scope_and_owned_records", + "failure_mode": "false-positive", + "case": { + "url": "https://quasar.dev/vue-components/virtual-scroll/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 8, + "table_columns": [ + { + "name": "item", + "label": "Option", + "type": "text", + "pattern": "#?\\d+\\s*-\\s*Option\\s+\\d+" } - }, - "artifact": "خبر جديدخبر جديد", - "ok": true, - "expected_task_pass": true - }, - { - "id": "kooora-news-scope-teams-and-players", - "category": "structured-extraction", - "root_cause": "news_detail_routes_not_category_navigation", - "failure_mode": "false-positive", - "case": { - "url": "https://www.kooora.com/", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 2, - "hosts": [ - "www.kooora.com" - ], - "entry_path": "/[^/]+/(?:أخبار|القوائم|%D8%A3%D8%AE%D8%A8%D8%A7%D8%B1|%D8%A7%D9%84%D9%82%D9%88%D8%A7%D8%A6%D9%85)/[^/]+/blt[0-9a-f]+", - "stable_text_anchors": [], - "name_validation": "nonempty", - "entry_identity_pattern": "/(blt[0-9a-f]+)$", - "entry_scope": { - "html_selector": "a:is(:has(.fco-card__headline-text), :has(.fco-high-impact-card__headline)):not(.fco-selective-scroll-section__item--hidden *):not([class*=\"videos-section_videos-section__container\"] [class*=\"scrollable-section_scrollable-section\"] *)", - "title_selector": ".fco-card__headline-text, .fco-high-impact-card__headline", - "link_selector": "a[href]" - } + ], + "record_identity_fields": [ + "item" + ], + "table_scope": { + "root_selector": ".q-virtual-scroll", + "root_index": 0, + "row_selector": ".q-virtual-scroll__content > .q-item", + "cell_selector": ":scope", + "start_pattern": "(?m)^\\s*#{0,6}\\s*Basic\\s*$", + "end_pattern": "(?m)^[ \\t#]*Horizontal[ \\t]*$", + "plain_row_pattern": "(?P#?\\d+\\s*-\\s*Option\\s+\\d+)", + "reference_kind": "fixed_demo", + "reference_rows": [ + { + "item": "#0 - Option 1" + }, + { + "item": "#1 - Option 2" + }, + { + "item": "#2 - Option 3" + }, + { + "item": "#3 - Option 4" + }, + { + "item": "#4 - Option 5" + }, + { + "item": "#5 - Option 6" + }, + { + "item": "#6 - Option 7" + }, + { + "item": "#7 - Option 8" + }, + { + "item": "#8 - Option 9" + }, + { + "item": "#9 - Option 10" + } + ] + } + } + }, + "artifact": "
#0 - Option 1
#1 - Option 2
#2 - Option 3
#3 - Option 4
#4 - Option 5
#5 - Option 6
#6 - Option 7
#7 - Option 8
", + "ok": true, + "expected_task_pass": false + }, + { + "id": "dynamic22-030-complete", + "category": "structured-extraction", + "root_cause": "dynamic_initial_scope_and_owned_records", + "failure_mode": "false-negative", + "case": { + "url": "https://primevue.dev/virtualscroller/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 4, + "table_columns": [ + { + "name": "item", + "label": "Item", + "type": "text", + "pattern": "(?i)Item\\s*#\\s*\\d+" } - }, - "artifact": "[Team](/كرة-القدم/فريق/name/123)\n[Player](/كرة-القدم/لاعب/name/أخبار/456)", - "ok": true, - "expected_task_pass": false - }, - { - "id": "kooora-news-scope-category-index", - "category": "structured-extraction", - "root_cause": "news_detail_routes_not_category_navigation", - "failure_mode": "false-positive", - "case": { - "url": "https://www.kooora.com/", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 2, - "hosts": [ - "www.kooora.com" - ], - "entry_path": "/[^/]+/(?:أخبار|القوائم|%D8%A3%D8%AE%D8%A8%D8%A7%D8%B1|%D8%A7%D9%84%D9%82%D9%88%D8%A7%D8%A6%D9%85)/[^/]+/blt[0-9a-f]+", - "stable_text_anchors": [], - "name_validation": "nonempty", - "entry_identity_pattern": "/(blt[0-9a-f]+)$", - "entry_scope": { - "html_selector": "a:is(:has(.fco-card__headline-text), :has(.fco-high-impact-card__headline)):not(.fco-selective-scroll-section__item--hidden *):not([class*=\"videos-section_videos-section__container\"] [class*=\"scrollable-section_scrollable-section\"] *)", - "title_selector": ".fco-card__headline-text, .fco-high-impact-card__headline", - "link_selector": "a[href]" - } + ], + "record_identity_fields": [ + "item" + ], + "table_scope": { + "root_selector": ".doc-section-demo", + "root_index": 0, + "row_selector": ".p-virtualscroller-content > div", + "cell_selector": ":scope", + "start_pattern": "(?im)^(?=[ \\t|]*(?:[-*][ \\t]+)?Item[ \\t]*#[ \\t]*\\d+|[ \\t]*\\|[ \\t]*Item[ \\t]*\\|)", + "end_pattern": "(?im)^[ \\t#]*(?:PREVIEW|Usage|Examples|Horizontal)[ \\t]*$", + "plain_row_pattern": "[ \\t|]*(?:[-*][ \\t]+)?(?P(?i:Item)[ \\t]*#[ \\t]*\\d+)[ \\t|]*", + "reference_kind": "fixed_demo", + "reference_rows": [ + { + "item": "Item #0" + }, + { + "item": "Item #1" + }, + { + "item": "Item #2" + }, + { + "item": "Item #3" + } + ] + } + } + }, + "artifact": "
Item #0
Item #1
Item #2
Item #3
", + "ok": true, + "expected_task_pass": true + }, + { + "id": "dynamic22-030-below80", + "category": "structured-extraction", + "root_cause": "dynamic_initial_scope_and_owned_records", + "failure_mode": "false-positive", + "case": { + "url": "https://primevue.dev/virtualscroller/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 4, + "table_columns": [ + { + "name": "item", + "label": "Item", + "type": "text", + "pattern": "(?i)Item\\s*#\\s*\\d+" } - }, - "artifact": "[Tennis](/تنس/bltabc123)\n[News](/أخبار)", - "ok": true, - "expected_task_pass": false - }, - { - "id": "kooora-news-scope-collapsed-card", - "category": "structured-extraction", - "root_cause": "news_detail_routes_not_category_navigation", - "failure_mode": "false-positive", - "case": { - "url": "https://www.kooora.com/", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 2, - "hosts": [ - "www.kooora.com" - ], - "entry_path": "/[^/]+/(?:أخبار|القوائم|%D8%A3%D8%AE%D8%A8%D8%A7%D8%B1|%D8%A7%D9%84%D9%82%D9%88%D8%A7%D8%A6%D9%85)/[^/]+/blt[0-9a-f]+", - "stable_text_anchors": [], - "name_validation": "nonempty", - "entry_identity_pattern": "/(blt[0-9a-f]+)$", - "entry_scope": { - "html_selector": "a:is(:has(.fco-card__headline-text), :has(.fco-high-impact-card__headline)):not(.fco-selective-scroll-section__item--hidden *):not([class*=\"videos-section_videos-section__container\"] [class*=\"scrollable-section_scrollable-section\"] *)", - "title_selector": ".fco-card__headline-text, .fco-high-impact-card__headline", - "link_selector": "a[href]" - } + ], + "record_identity_fields": [ + "item" + ], + "table_scope": { + "root_selector": ".doc-section-demo", + "root_index": 0, + "row_selector": ".p-virtualscroller-content > div", + "cell_selector": ":scope", + "start_pattern": "(?im)^(?=[ \\t|]*(?:[-*][ \\t]+)?Item[ \\t]*#[ \\t]*\\d+|[ \\t]*\\|[ \\t]*Item[ \\t]*\\|)", + "end_pattern": "(?im)^[ \\t#]*(?:PREVIEW|Usage|Examples|Horizontal)[ \\t]*$", + "plain_row_pattern": "[ \\t|]*(?:[-*][ \\t]+)?(?P(?i:Item)[ \\t]*#[ \\t]*\\d+)[ \\t|]*", + "reference_kind": "fixed_demo", + "reference_rows": [ + { + "item": "Item #0" + }, + { + "item": "Item #1" + }, + { + "item": "Item #2" + }, + { + "item": "Item #3" + } + ] + } + } + }, + "artifact": "
Item #0
Item #1
Item #2
", + "ok": true, + "expected_task_pass": false + }, + { + "id": "dynamic22-030-duplicate", + "category": "structured-extraction", + "root_cause": "dynamic_initial_scope_and_owned_records", + "failure_mode": "false-positive", + "case": { + "url": "https://primevue.dev/virtualscroller/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 4, + "table_columns": [ + { + "name": "item", + "label": "Item", + "type": "text", + "pattern": "(?i)Item\\s*#\\s*\\d+" } - }, - "artifact": "خبر جديد
خبر جديد
", - "ok": true, - "expected_task_pass": false - }, - { - "id": "kooora-news-scope-same-id-new-slug", - "category": "structured-extraction", - "root_cause": "news_detail_routes_not_category_navigation", - "failure_mode": "false-positive", - "case": { - "url": "https://www.kooora.com/", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 2, - "hosts": [ - "www.kooora.com" - ], - "entry_path": "/[^/]+/(?:أخبار|القوائم|%D8%A3%D8%AE%D8%A8%D8%A7%D8%B1|%D8%A7%D9%84%D9%82%D9%88%D8%A7%D8%A6%D9%85)/[^/]+/blt[0-9a-f]+", - "stable_text_anchors": [], - "name_validation": "nonempty", - "entry_identity_pattern": "/(blt[0-9a-f]+)$", - "entry_scope": { - "html_selector": "a:is(:has(.fco-card__headline-text), :has(.fco-high-impact-card__headline)):not(.fco-selective-scroll-section__item--hidden *):not([class*=\"videos-section_videos-section__container\"] [class*=\"scrollable-section_scrollable-section\"] *)", - "title_selector": ".fco-card__headline-text, .fco-high-impact-card__headline", - "link_selector": "a[href]" - } + ], + "record_identity_fields": [ + "item" + ], + "table_scope": { + "root_selector": ".doc-section-demo", + "root_index": 0, + "row_selector": ".p-virtualscroller-content > div", + "cell_selector": ":scope", + "start_pattern": "(?im)^(?=[ \\t|]*(?:[-*][ \\t]+)?Item[ \\t]*#[ \\t]*\\d+|[ \\t]*\\|[ \\t]*Item[ \\t]*\\|)", + "end_pattern": "(?im)^[ \\t#]*(?:PREVIEW|Usage|Examples|Horizontal)[ \\t]*$", + "plain_row_pattern": "[ \\t|]*(?:[-*][ \\t]+)?(?P(?i:Item)[ \\t]*#[ \\t]*\\d+)[ \\t|]*", + "reference_kind": "fixed_demo", + "reference_rows": [ + { + "item": "Item #0" + }, + { + "item": "Item #1" + }, + { + "item": "Item #2" + }, + { + "item": "Item #3" + } + ] + } + } + }, + "artifact": "
Item #0
Item #0
Item #0
Item #0
", + "ok": true, + "expected_task_pass": false + }, + { + "id": "dynamic22-030-wrong-scope", + "category": "structured-extraction", + "root_cause": "dynamic_initial_scope_and_owned_records", + "failure_mode": "false-positive", + "case": { + "url": "https://primevue.dev/virtualscroller/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 4, + "table_columns": [ + { + "name": "item", + "label": "Item", + "type": "text", + "pattern": "(?i)Item\\s*#\\s*\\d+" } - }, - "artifact": "[First](/كرة-قدم/أخبار/title/bltabc123)\n[Second](/كرة-قدم/أخبار/changed-title/bltabc123)", - "ok": true, - "expected_task_pass": false - }, - { - "id": "kooora-news-scope-fresh-listicle", - "category": "structured-extraction", - "root_cause": "news_detail_routes_not_category_navigation", - "failure_mode": "false-negative", - "case": { - "url": "https://www.kooora.com/", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 2, - "hosts": [ - "www.kooora.com" - ], - "entry_path": "/[^/]+/(?:أخبار|القوائم|%D8%A3%D8%AE%D8%A8%D8%A7%D8%B1|%D8%A7%D9%84%D9%82%D9%88%D8%A7%D8%A6%D9%85)/[^/]+/blt[0-9a-f]+", - "stable_text_anchors": [], - "name_validation": "nonempty", - "entry_identity_pattern": "/(blt[0-9a-f]+)$", - "entry_scope": { - "html_selector": "a:is(:has(.fco-card__headline-text), :has(.fco-high-impact-card__headline)):not(.fco-selective-scroll-section__item--hidden *):not([class*=\"videos-section_videos-section__container\"] [class*=\"scrollable-section_scrollable-section\"] *)", - "title_selector": ".fco-card__headline-text, .fco-high-impact-card__headline", - "link_selector": "a[href]" - } + ], + "record_identity_fields": [ + "item" + ], + "table_scope": { + "root_selector": ".doc-section-demo", + "root_index": 0, + "row_selector": ".p-virtualscroller-content > div", + "cell_selector": ":scope", + "start_pattern": "(?im)^(?=[ \\t|]*(?:[-*][ \\t]+)?Item[ \\t]*#[ \\t]*\\d+|[ \\t]*\\|[ \\t]*Item[ \\t]*\\|)", + "end_pattern": "(?im)^[ \\t#]*(?:PREVIEW|Usage|Examples|Horizontal)[ \\t]*$", + "plain_row_pattern": "[ \\t|]*(?:[-*][ \\t]+)?(?P(?i:Item)[ \\t]*#[ \\t]*\\d+)[ \\t|]*", + "reference_kind": "fixed_demo", + "reference_rows": [ + { + "item": "Item #0" + }, + { + "item": "Item #1" + }, + { + "item": "Item #2" + }, + { + "item": "Item #3" + } + ] + } + } + }, + "artifact": "
Item #0
Item #1
Item #2
Item #3
", + "ok": true, + "expected_task_pass": false + }, + { + "id": "dynamic22-032-complete", + "category": "structured-extraction", + "root_cause": "dynamic_initial_scope_and_owned_records", + "failure_mode": "false-negative", + "case": { + "url": "https://primeng.dev/virtualscroller", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 4, + "table_columns": [ + { + "name": "item", + "label": "Item", + "type": "text", + "pattern": "(?i)Item\\s*#\\s*\\d+" } - }, - "artifact": "[News](/كرة-قدم/أخبار/new/bltabc123)\n[Listicle](/كرة-قدم/القوائم/new/bltdef456)", - "ok": true, - "expected_task_pass": true - }, - { - "id": "quake-joined-location-time", - "category": "structured-extraction", - "root_cause": "earthquake_inline_location_timestamp", - "failure_mode": "false-negative", - "case": { - "url": "https://example.test/earthquakes", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "earthquake", - "minimum_records": 2 + ], + "record_identity_fields": [ + "item" + ], + "table_scope": { + "root_selector": ".doc-section-demo", + "root_index": 0, + "row_selector": ".p-virtualscroller-content > div", + "cell_selector": ":scope", + "start_pattern": "(?im)^(?=[ \\t|]*(?:[-*][ \\t]+)?Item[ \\t]*#[ \\t]*\\d+|[ \\t]*\\|[ \\t]*Item[ \\t]*\\|)", + "end_pattern": "(?im)^[ \\t#]*(?:PREVIEW|Usage|Examples|Horizontal)[ \\t]*$", + "plain_row_pattern": "[ \\t|]*(?:[-*][ \\t]+)?(?P(?i:Item)[ \\t]*#[ \\t]*\\d+)[ \\t|]*", + "reference_kind": "fixed_demo", + "reference_rows": [ + { + "item": "Item #0" + }, + { + "item": "Item #1" + }, + { + "item": "Item #2" + }, + { + "item": "Item #3" + } + ] + } + } + }, + "artifact": "
Item #0
Item #1
Item #2
Item #3
", + "ok": true, + "expected_task_pass": true + }, + { + "id": "dynamic22-032-below80", + "category": "structured-extraction", + "root_cause": "dynamic_initial_scope_and_owned_records", + "failure_mode": "false-positive", + "case": { + "url": "https://primeng.dev/virtualscroller", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 4, + "table_columns": [ + { + "name": "item", + "label": "Item", + "type": "text", + "pattern": "(?i)Item\\s*#\\s*\\d+" } - }, - "artifact": "4.2\nHarbor Island2026-09-20 12:00:00 (UTC+08:00)\n10.0 km\n3.1\nNorth Valley2026-09-20T05:00:00Z\n7.4 km", - "ok": true, - "expected_task_pass": true - }, - { - "id": "quake-joined-missing-depth", - "category": "structured-extraction", - "root_cause": "earthquake_inline_location_timestamp", - "failure_mode": "false-positive", - "case": { - "url": "https://example.test/earthquakes", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "earthquake", - "minimum_records": 2 + ], + "record_identity_fields": [ + "item" + ], + "table_scope": { + "root_selector": ".doc-section-demo", + "root_index": 0, + "row_selector": ".p-virtualscroller-content > div", + "cell_selector": ":scope", + "start_pattern": "(?im)^(?=[ \\t|]*(?:[-*][ \\t]+)?Item[ \\t]*#[ \\t]*\\d+|[ \\t]*\\|[ \\t]*Item[ \\t]*\\|)", + "end_pattern": "(?im)^[ \\t#]*(?:PREVIEW|Usage|Examples|Horizontal)[ \\t]*$", + "plain_row_pattern": "[ \\t|]*(?:[-*][ \\t]+)?(?P(?i:Item)[ \\t]*#[ \\t]*\\d+)[ \\t|]*", + "reference_kind": "fixed_demo", + "reference_rows": [ + { + "item": "Item #0" + }, + { + "item": "Item #1" + }, + { + "item": "Item #2" + }, + { + "item": "Item #3" + } + ] + } + } + }, + "artifact": "
Item #0
Item #1
Item #2
", + "ok": true, + "expected_task_pass": false + }, + { + "id": "dynamic22-032-duplicate", + "category": "structured-extraction", + "root_cause": "dynamic_initial_scope_and_owned_records", + "failure_mode": "false-positive", + "case": { + "url": "https://primeng.dev/virtualscroller", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 4, + "table_columns": [ + { + "name": "item", + "label": "Item", + "type": "text", + "pattern": "(?i)Item\\s*#\\s*\\d+" } - }, - "artifact": "4.2\nHarbor Island2026-09-20 12:00:00 (UTC+08:00)\n3.1\nNorth Valley2026-09-20T05:00:00Z\n7.4 km", - "ok": true, - "expected_task_pass": false - }, - { - "id": "quake-joined-invalid-time", - "category": "structured-extraction", - "root_cause": "earthquake_inline_location_timestamp", - "failure_mode": "false-positive", - "case": { - "url": "https://example.test/earthquakes", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "earthquake", - "minimum_records": 2 + ], + "record_identity_fields": [ + "item" + ], + "table_scope": { + "root_selector": ".doc-section-demo", + "root_index": 0, + "row_selector": ".p-virtualscroller-content > div", + "cell_selector": ":scope", + "start_pattern": "(?im)^(?=[ \\t|]*(?:[-*][ \\t]+)?Item[ \\t]*#[ \\t]*\\d+|[ \\t]*\\|[ \\t]*Item[ \\t]*\\|)", + "end_pattern": "(?im)^[ \\t#]*(?:PREVIEW|Usage|Examples|Horizontal)[ \\t]*$", + "plain_row_pattern": "[ \\t|]*(?:[-*][ \\t]+)?(?P(?i:Item)[ \\t]*#[ \\t]*\\d+)[ \\t|]*", + "reference_kind": "fixed_demo", + "reference_rows": [ + { + "item": "Item #0" + }, + { + "item": "Item #1" + }, + { + "item": "Item #2" + }, + { + "item": "Item #3" + } + ] + } + } + }, + "artifact": "
Item #0
Item #0
Item #0
Item #0
", + "ok": true, + "expected_task_pass": false + }, + { + "id": "dynamic22-032-wrong-scope", + "category": "structured-extraction", + "root_cause": "dynamic_initial_scope_and_owned_records", + "failure_mode": "false-positive", + "case": { + "url": "https://primeng.dev/virtualscroller", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 4, + "table_columns": [ + { + "name": "item", + "label": "Item", + "type": "text", + "pattern": "(?i)Item\\s*#\\s*\\d+" } - }, - "artifact": "4.2\nHarbor Island2026-99-20 12:00:00 (UTC+08:00)\n10.0 km\n3.1\nNorth Valley2026-09-20T05:00:00Z\n7.4 km", - "ok": true, - "expected_task_pass": false - }, - { - "id": "quake-joined-duplicate-record", - "category": "structured-extraction", - "root_cause": "earthquake_inline_location_timestamp", - "failure_mode": "false-positive", - "case": { - "url": "https://example.test/earthquakes", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "earthquake", - "minimum_records": 2 + ], + "record_identity_fields": [ + "item" + ], + "table_scope": { + "root_selector": ".doc-section-demo", + "root_index": 0, + "row_selector": ".p-virtualscroller-content > div", + "cell_selector": ":scope", + "start_pattern": "(?im)^(?=[ \\t|]*(?:[-*][ \\t]+)?Item[ \\t]*#[ \\t]*\\d+|[ \\t]*\\|[ \\t]*Item[ \\t]*\\|)", + "end_pattern": "(?im)^[ \\t#]*(?:PREVIEW|Usage|Examples|Horizontal)[ \\t]*$", + "plain_row_pattern": "[ \\t|]*(?:[-*][ \\t]+)?(?P(?i:Item)[ \\t]*#[ \\t]*\\d+)[ \\t|]*", + "reference_kind": "fixed_demo", + "reference_rows": [ + { + "item": "Item #0" + }, + { + "item": "Item #1" + }, + { + "item": "Item #2" + }, + { + "item": "Item #3" + } + ] + } + } + }, + "artifact": "
Item #0
Item #1
Item #2
Item #3
", + "ok": true, + "expected_task_pass": false + }, + { + "id": "dynamic22-098-complete", + "category": "structured-extraction", + "root_cause": "dynamic_initial_scope_and_owned_records", + "failure_mode": "false-negative", + "case": { + "url": "https://shahid.mbc.net/ar", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 5, + "hosts": [ + "shahid.mbc.net" + ], + "entry_path": "/ar/player/episodes/[^/]+/id-[0-9]+", + "entry_identity_pattern": "/id-([0-9]+)$", + "name_validation": "nonempty", + "entry_scope": { + "html_selector": ".carousel-item", + "title_selector": "h3", + "link_selector": "a[href*=\"/ar/player/episodes/\"]", + "start_pattern": "الأكثر مشاهدة الأن", + "end_pattern": "اتصل بنا" + } + } + }, + "artifact": "

الموسم 2، الحلقة 1

الموسم 2، الحلقة 2

الموسم 2، الحلقة 3

الموسم 2، الحلقة 4

الموسم 2، الحلقة 5

", + "ok": true, + "expected_task_pass": true + }, + { + "id": "dynamic22-098-below80", + "category": "structured-extraction", + "root_cause": "dynamic_initial_scope_and_owned_records", + "failure_mode": "false-positive", + "case": { + "url": "https://shahid.mbc.net/ar", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 5, + "hosts": [ + "shahid.mbc.net" + ], + "entry_path": "/ar/player/episodes/[^/]+/id-[0-9]+", + "entry_identity_pattern": "/id-([0-9]+)$", + "name_validation": "nonempty", + "entry_scope": { + "html_selector": ".carousel-item", + "title_selector": "h3", + "link_selector": "a[href*=\"/ar/player/episodes/\"]", + "start_pattern": "الأكثر مشاهدة الأن", + "end_pattern": "اتصل بنا" + } + } + }, + "artifact": "

الموسم 2، الحلقة 1

الموسم 2، الحلقة 2

الموسم 2، الحلقة 3

الموسم 2، الحلقة 4

", + "ok": true, + "expected_task_pass": false + }, + { + "id": "dynamic22-098-duplicate", + "category": "structured-extraction", + "root_cause": "dynamic_initial_scope_and_owned_records", + "failure_mode": "false-positive", + "case": { + "url": "https://shahid.mbc.net/ar", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 5, + "hosts": [ + "shahid.mbc.net" + ], + "entry_path": "/ar/player/episodes/[^/]+/id-[0-9]+", + "entry_identity_pattern": "/id-([0-9]+)$", + "name_validation": "nonempty", + "entry_scope": { + "html_selector": ".carousel-item", + "title_selector": "h3", + "link_selector": "a[href*=\"/ar/player/episodes/\"]", + "start_pattern": "الأكثر مشاهدة الأن", + "end_pattern": "اتصل بنا" + } + } + }, + "artifact": "

الموسم 2، الحلقة 1

الموسم 2، الحلقة 2

الموسم 2، الحلقة 3

الموسم 2، الحلقة 4

الموسم 2، الحلقة 5

", + "ok": true, + "expected_task_pass": false + }, + { + "id": "dynamic22-098-wrong-scope", + "category": "structured-extraction", + "root_cause": "dynamic_initial_scope_and_owned_records", + "failure_mode": "false-positive", + "case": { + "url": "https://shahid.mbc.net/ar", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 5, + "hosts": [ + "shahid.mbc.net" + ], + "entry_path": "/ar/player/episodes/[^/]+/id-[0-9]+", + "entry_identity_pattern": "/id-([0-9]+)$", + "name_validation": "nonempty", + "entry_scope": { + "html_selector": ".carousel-item", + "title_selector": "h3", + "link_selector": "a[href*=\"/ar/player/episodes/\"]", + "start_pattern": "الأكثر مشاهدة الأن", + "end_pattern": "اتصل بنا" + } + } + }, + "artifact": "

الموسم 2، الحلقة 1

الموسم 2، الحلقة 2

الموسم 2، الحلقة 3

الموسم 2، الحلقة 4

الموسم 2، الحلقة 5

", + "ok": true, + "expected_task_pass": false + }, + { + "id": "dynamic22-061-complete", + "category": "structured-extraction", + "root_cause": "dynamic_initial_scope_and_owned_records", + "failure_mode": "false-negative", + "case": { + "url": "https://juejin.cn/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 16, + "hosts": [ + "juejin.cn" + ], + "entry_path": "/post/[0-9]+", + "name_validation": "nonempty", + "entry_scope": { + "html_selector": ".entry-list .entry a.title" + } + } + }, + "artifact": "
新文章1
新文章2
新文章3
新文章4
新文章5
新文章6
新文章7
新文章8
新文章9
新文章10
新文章11
新文章12
新文章13
新文章14
新文章15
新文章16
", + "ok": true, + "expected_task_pass": true + }, + { + "id": "dynamic22-061-below80", + "category": "structured-extraction", + "root_cause": "dynamic_initial_scope_and_owned_records", + "failure_mode": "false-positive", + "case": { + "url": "https://juejin.cn/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 16, + "hosts": [ + "juejin.cn" + ], + "entry_path": "/post/[0-9]+", + "name_validation": "nonempty", + "entry_scope": { + "html_selector": ".entry-list .entry a.title" + } + } + }, + "artifact": "
新文章1
新文章2
新文章3
新文章4
新文章5
新文章6
新文章7
新文章8
新文章9
新文章10
新文章11
新文章12
新文章13
新文章14
新文章15
", + "ok": true, + "expected_task_pass": false + }, + { + "id": "dynamic22-061-duplicate", + "category": "structured-extraction", + "root_cause": "dynamic_initial_scope_and_owned_records", + "failure_mode": "false-positive", + "case": { + "url": "https://juejin.cn/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 16, + "hosts": [ + "juejin.cn" + ], + "entry_path": "/post/[0-9]+", + "name_validation": "nonempty", + "entry_scope": { + "html_selector": ".entry-list .entry a.title" + } + } + }, + "artifact": "
新文章1
新文章2
新文章3
新文章4
新文章5
新文章6
新文章7
新文章8
新文章9
新文章10
新文章11
新文章12
新文章13
新文章14
新文章15
新文章16
", + "ok": true, + "expected_task_pass": false + }, + { + "id": "dynamic22-061-wrong-scope", + "category": "structured-extraction", + "root_cause": "dynamic_initial_scope_and_owned_records", + "failure_mode": "false-positive", + "case": { + "url": "https://juejin.cn/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 16, + "hosts": [ + "juejin.cn" + ], + "entry_path": "/post/[0-9]+", + "name_validation": "nonempty", + "entry_scope": { + "html_selector": ".entry-list .entry a.title" + } + } + }, + "artifact": "
新文章1
新文章2
新文章3
新文章4
新文章5
新文章6
新文章7
新文章8
新文章9
新文章10
新文章11
新文章12
新文章13
新文章14
新文章15
新文章16
", + "ok": true, + "expected_task_pass": false + }, + { + "id": "dynamic22-037-prefetch-only", + "category": "structured-extraction", + "root_cause": "dynamic_initial_scope_and_owned_records", + "failure_mode": "false-positive", + "case": { + "url": "https://quasar.dev/vue-components/virtual-scroll/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 8, + "table_columns": [ + { + "name": "item", + "label": "Option", + "type": "text", + "pattern": "#?\\d+\\s*-\\s*Option\\s+\\d+" } - }, - "artifact": "4.2\nHarbor Island2026-09-20 12:00:00 (UTC+08:00)\n10.0 km\n4.2\nHarbor Island2026-09-20 12:00:00 (UTC+08:00)\n10.0 km", - "ok": true, - "expected_task_pass": false - }, - { - "id": "dynamic22-037-complete", - "category": "structured-extraction", - "root_cause": "dynamic_initial_scope_and_owned_records", - "failure_mode": "false-negative", - "case": { - "url": "https://quasar.dev/vue-components/virtual-scroll/", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "table_row", - "minimum_records": 8, - "table_columns": [ - { - "name": "item", - "label": "Option", - "type": "text", - "pattern": "#?\\d+\\s*-\\s*Option\\s+\\d+" - } - ], - "record_identity_fields": [ - "item" - ], - "table_scope": { - "root_selector": ".q-virtual-scroll", - "root_index": 0, - "row_selector": ".q-virtual-scroll__content > .q-item", - "cell_selector": ":scope", - "start_pattern": "(?m)^\\s*#{0,6}\\s*Basic\\s*$", - "end_pattern": "(?m)^[ \\t#]*Horizontal[ \\t]*$", - "plain_row_pattern": "(?P#?\\d+\\s*-\\s*Option\\s+\\d+)", - "reference_kind": "fixed_demo", - "reference_rows": [ - { - "item": "#0 - Option 1" - }, - { - "item": "#1 - Option 2" - }, - { - "item": "#2 - Option 3" - }, - { - "item": "#3 - Option 4" - }, - { - "item": "#4 - Option 5" - }, - { - "item": "#5 - Option 6" - }, - { - "item": "#6 - Option 7" - }, - { - "item": "#7 - Option 8" - }, - { - "item": "#8 - Option 9" - }, - { - "item": "#9 - Option 10" - } - ] - } + ], + "record_identity_fields": [ + "item" + ], + "table_scope": { + "root_selector": ".q-virtual-scroll", + "root_index": 0, + "row_selector": ".q-virtual-scroll__content > .q-item", + "cell_selector": ":scope", + "start_pattern": "(?m)^\\s*#{0,6}\\s*Basic\\s*$", + "end_pattern": "(?m)^[ \\t#]*Horizontal[ \\t]*$", + "plain_row_pattern": "(?P#?\\d+\\s*-\\s*Option\\s+\\d+)", + "reference_kind": "fixed_demo", + "reference_rows": [ + { + "item": "#0 - Option 1" + }, + { + "item": "#1 - Option 2" + }, + { + "item": "#2 - Option 3" + }, + { + "item": "#3 - Option 4" + }, + { + "item": "#4 - Option 5" + }, + { + "item": "#5 - Option 6" + }, + { + "item": "#6 - Option 7" + }, + { + "item": "#7 - Option 8" + }, + { + "item": "#8 - Option 9" + }, + { + "item": "#9 - Option 10" + } + ] + } + } + }, + "artifact": "
#20 - Option 21
#21 - Option 22
#22 - Option 23
#23 - Option 24
#24 - Option 25
#25 - Option 26
#26 - Option 27
#27 - Option 28
", + "ok": true, + "expected_task_pass": false + }, + { + "id": "dynamic22-030-prefetch-only", + "category": "structured-extraction", + "root_cause": "dynamic_initial_scope_and_owned_records", + "failure_mode": "false-positive", + "case": { + "url": "https://primevue.dev/virtualscroller/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 4, + "table_columns": [ + { + "name": "item", + "label": "Item", + "type": "text", + "pattern": "(?i)Item\\s*#\\s*\\d+" } - }, - "artifact": "
#0 - Option 1
#1 - Option 2
#2 - Option 3
#3 - Option 4
#4 - Option 5
#5 - Option 6
#6 - Option 7
#7 - Option 8
", - "ok": true, - "expected_task_pass": true - }, - { - "id": "dynamic22-037-below80", - "category": "structured-extraction", - "root_cause": "dynamic_initial_scope_and_owned_records", - "failure_mode": "false-positive", - "case": { - "url": "https://quasar.dev/vue-components/virtual-scroll/", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "table_row", - "minimum_records": 8, - "table_columns": [ - { - "name": "item", - "label": "Option", - "type": "text", - "pattern": "#?\\d+\\s*-\\s*Option\\s+\\d+" - } - ], - "record_identity_fields": [ - "item" - ], - "table_scope": { - "root_selector": ".q-virtual-scroll", - "root_index": 0, - "row_selector": ".q-virtual-scroll__content > .q-item", - "cell_selector": ":scope", - "start_pattern": "(?m)^\\s*#{0,6}\\s*Basic\\s*$", - "end_pattern": "(?m)^[ \\t#]*Horizontal[ \\t]*$", - "plain_row_pattern": "(?P#?\\d+\\s*-\\s*Option\\s+\\d+)", - "reference_kind": "fixed_demo", - "reference_rows": [ - { - "item": "#0 - Option 1" - }, - { - "item": "#1 - Option 2" - }, - { - "item": "#2 - Option 3" - }, - { - "item": "#3 - Option 4" - }, - { - "item": "#4 - Option 5" - }, - { - "item": "#5 - Option 6" - }, - { - "item": "#6 - Option 7" - }, - { - "item": "#7 - Option 8" - }, - { - "item": "#8 - Option 9" - }, - { - "item": "#9 - Option 10" - } - ] - } + ], + "record_identity_fields": [ + "item" + ], + "table_scope": { + "root_selector": ".doc-section-demo", + "root_index": 0, + "row_selector": ".p-virtualscroller-content > div", + "cell_selector": ":scope", + "start_pattern": "(?im)^(?=[ \\t|]*(?:[-*][ \\t]+)?Item[ \\t]*#[ \\t]*\\d+|[ \\t]*\\|[ \\t]*Item[ \\t]*\\|)", + "end_pattern": "(?im)^[ \\t#]*(?:PREVIEW|Usage|Examples|Horizontal)[ \\t]*$", + "plain_row_pattern": "[ \\t|]*(?:[-*][ \\t]+)?(?P(?i:Item)[ \\t]*#[ \\t]*\\d+)[ \\t|]*", + "reference_kind": "fixed_demo", + "reference_rows": [ + { + "item": "Item #0" + }, + { + "item": "Item #1" + }, + { + "item": "Item #2" + }, + { + "item": "Item #3" + } + ] + } + } + }, + "artifact": "
Item #20
Item #21
Item #22
Item #23
", + "ok": true, + "expected_task_pass": false + }, + { + "id": "dynamic22-032-prefetch-only", + "category": "structured-extraction", + "root_cause": "dynamic_initial_scope_and_owned_records", + "failure_mode": "false-positive", + "case": { + "url": "https://primeng.dev/virtualscroller", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 4, + "table_columns": [ + { + "name": "item", + "label": "Item", + "type": "text", + "pattern": "(?i)Item\\s*#\\s*\\d+" } - }, - "artifact": "
#0 - Option 1
#1 - Option 2
#2 - Option 3
#3 - Option 4
#4 - Option 5
#5 - Option 6
#6 - Option 7
", - "ok": true, - "expected_task_pass": false - }, - { - "id": "dynamic22-037-duplicate", - "category": "structured-extraction", - "root_cause": "dynamic_initial_scope_and_owned_records", - "failure_mode": "false-positive", - "case": { - "url": "https://quasar.dev/vue-components/virtual-scroll/", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "table_row", - "minimum_records": 8, - "table_columns": [ - { - "name": "item", - "label": "Option", - "type": "text", - "pattern": "#?\\d+\\s*-\\s*Option\\s+\\d+" - } - ], - "record_identity_fields": [ - "item" - ], - "table_scope": { - "root_selector": ".q-virtual-scroll", - "root_index": 0, - "row_selector": ".q-virtual-scroll__content > .q-item", - "cell_selector": ":scope", - "start_pattern": "(?m)^\\s*#{0,6}\\s*Basic\\s*$", - "end_pattern": "(?m)^[ \\t#]*Horizontal[ \\t]*$", - "plain_row_pattern": "(?P#?\\d+\\s*-\\s*Option\\s+\\d+)", - "reference_kind": "fixed_demo", - "reference_rows": [ - { - "item": "#0 - Option 1" - }, - { - "item": "#1 - Option 2" - }, - { - "item": "#2 - Option 3" - }, - { - "item": "#3 - Option 4" - }, - { - "item": "#4 - Option 5" - }, - { - "item": "#5 - Option 6" - }, - { - "item": "#6 - Option 7" - }, - { - "item": "#7 - Option 8" - }, - { - "item": "#8 - Option 9" - }, - { - "item": "#9 - Option 10" - } - ] - } + ], + "record_identity_fields": [ + "item" + ], + "table_scope": { + "root_selector": ".doc-section-demo", + "root_index": 0, + "row_selector": ".p-virtualscroller-content > div", + "cell_selector": ":scope", + "start_pattern": "(?im)^(?=[ \\t|]*(?:[-*][ \\t]+)?Item[ \\t]*#[ \\t]*\\d+|[ \\t]*\\|[ \\t]*Item[ \\t]*\\|)", + "end_pattern": "(?im)^[ \\t#]*(?:PREVIEW|Usage|Examples|Horizontal)[ \\t]*$", + "plain_row_pattern": "[ \\t|]*(?:[-*][ \\t]+)?(?P(?i:Item)[ \\t]*#[ \\t]*\\d+)[ \\t|]*", + "reference_kind": "fixed_demo", + "reference_rows": [ + { + "item": "Item #0" + }, + { + "item": "Item #1" + }, + { + "item": "Item #2" + }, + { + "item": "Item #3" + } + ] + } + } + }, + "artifact": "
Item #20
Item #21
Item #22
Item #23
", + "ok": true, + "expected_task_pass": false + }, + { + "id": "table-copy-marker-complete", + "category": "format-normalization", + "root_cause": "table_auxiliary_copy_markers", + "failure_mode": "false-negative", + "case": { + "url": "https://example.test/table", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 1, + "table_columns": [ + { + "name": "name", + "label": "Name", + "type": "text" + }, + { + "name": "date", + "label": "Date", + "type": "date_ordinal_en" + }, + { + "name": "state", + "label": "State", + "type": "text", + "pattern": "Online|Offline" } - }, - "artifact": "
#0 - Option 1
#0 - Option 1
#0 - Option 1
#0 - Option 1
#0 - Option 1
#0 - Option 1
#0 - Option 1
#0 - Option 1
", - "ok": true, - "expected_task_pass": false - }, - { - "id": "dynamic22-037-wrong-scope", - "category": "structured-extraction", - "root_cause": "dynamic_initial_scope_and_owned_records", - "failure_mode": "false-positive", - "case": { - "url": "https://quasar.dev/vue-components/virtual-scroll/", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "table_row", - "minimum_records": 8, - "table_columns": [ - { - "name": "item", - "label": "Option", - "type": "text", - "pattern": "#?\\d+\\s*-\\s*Option\\s+\\d+" - } - ], - "record_identity_fields": [ - "item" - ], - "table_scope": { - "root_selector": ".q-virtual-scroll", - "root_index": 0, - "row_selector": ".q-virtual-scroll__content > .q-item", - "cell_selector": ":scope", - "start_pattern": "(?m)^\\s*#{0,6}\\s*Basic\\s*$", - "end_pattern": "(?m)^[ \\t#]*Horizontal[ \\t]*$", - "plain_row_pattern": "(?P#?\\d+\\s*-\\s*Option\\s+\\d+)", - "reference_kind": "fixed_demo", - "reference_rows": [ - { - "item": "#0 - Option 1" - }, - { - "item": "#1 - Option 2" - }, - { - "item": "#2 - Option 3" - }, - { - "item": "#3 - Option 4" - }, - { - "item": "#4 - Option 5" - }, - { - "item": "#5 - Option 6" - }, - { - "item": "#6 - Option 7" - }, - { - "item": "#7 - Option 8" - }, - { - "item": "#8 - Option 9" - }, - { - "item": "#9 - Option 10" - } - ] - } + ], + "record_identity_fields": [ + "name" + ], + "table_scope": { + "row_selector": "tbody tr", + "cell_selector": "td", + "start_pattern": "(?m)^(?=\\| Name \\|)", + "end_pattern": "END", + "plain_row_pattern": "(?P[A-Za-z]+) (?P\\d{1,2}(?:st|nd|rd|th) [A-Za-z]{3} \\d{2}) (?POnline|Offline)", + "plain_text_ignored_patterns": [ + "[↦↵](?=[ \\t]*(?:\\||$))", + "\\(external, opens in a new tab or window\\)" + ] + } + } + }, + "artifact": "| Name↦ | Date↦ | State↵ |\n| --- | --- | --- |\n| Ada(external, opens in a new tab or window)↦ | 29th Sep 25↦ | Online↵ |\nEND", + "ok": true, + "expected_task_pass": true + }, + { + "id": "table-copy-marker-bad-date", + "category": "format-normalization", + "root_cause": "table_auxiliary_copy_markers", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/table", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 1, + "table_columns": [ + { + "name": "name", + "label": "Name", + "type": "text" + }, + { + "name": "date", + "label": "Date", + "type": "date_ordinal_en" + }, + { + "name": "state", + "label": "State", + "type": "text", + "pattern": "Online|Offline" } - }, - "artifact": "
#0 - Option 1
#1 - Option 2
#2 - Option 3
#3 - Option 4
#4 - Option 5
#5 - Option 6
#6 - Option 7
#7 - Option 8
", - "ok": true, - "expected_task_pass": false - }, - { - "id": "dynamic22-030-complete", - "category": "structured-extraction", - "root_cause": "dynamic_initial_scope_and_owned_records", - "failure_mode": "false-negative", - "case": { - "url": "https://primevue.dev/virtualscroller/", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "table_row", - "minimum_records": 4, - "table_columns": [ - { - "name": "item", - "label": "Item", - "type": "text", - "pattern": "(?i)Item\\s*#\\s*\\d+" - } - ], - "record_identity_fields": [ - "item" - ], - "table_scope": { - "root_selector": ".doc-section-demo", - "root_index": 0, - "row_selector": ".p-virtualscroller-content > div", - "cell_selector": ":scope", - "start_pattern": "(?im)^(?=[ \\t|]*(?:[-*][ \\t]+)?Item[ \\t]*#[ \\t]*\\d+|[ \\t]*\\|[ \\t]*Item[ \\t]*\\|)", - "end_pattern": "(?im)^[ \\t#]*(?:PREVIEW|Usage|Examples|Horizontal)[ \\t]*$", - "plain_row_pattern": "[ \\t|]*(?:[-*][ \\t]+)?(?P(?i:Item)[ \\t]*#[ \\t]*\\d+)[ \\t|]*", - "reference_kind": "fixed_demo", - "reference_rows": [ - { - "item": "Item #0" - }, - { - "item": "Item #1" - }, - { - "item": "Item #2" - }, - { - "item": "Item #3" - } - ] - } + ], + "record_identity_fields": [ + "name" + ], + "table_scope": { + "row_selector": "tbody tr", + "cell_selector": "td", + "start_pattern": "(?m)^(?=\\| Name \\|)", + "end_pattern": "END", + "plain_row_pattern": "(?P[A-Za-z]+) (?P\\d{1,2}(?:st|nd|rd|th) [A-Za-z]{3} \\d{2}) (?POnline|Offline)", + "plain_text_ignored_patterns": [ + "[↦↵](?=[ \\t]*(?:\\||$))", + "\\(external, opens in a new tab or window\\)" + ] + } + } + }, + "artifact": "| Name↦ | Date↦ | State↵ |\n| --- | --- | --- |\n| Ada↦ | 32nd Sep 25↦ | Online↵ |\nEND", + "ok": true, + "expected_task_pass": false + }, + { + "id": "table-copy-marker-missing-field", + "category": "format-normalization", + "root_cause": "table_auxiliary_copy_markers", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/table", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 1, + "table_columns": [ + { + "name": "name", + "label": "Name", + "type": "text" + }, + { + "name": "date", + "label": "Date", + "type": "date_ordinal_en" + }, + { + "name": "state", + "label": "State", + "type": "text", + "pattern": "Online|Offline" } - }, - "artifact": "
Item #0
Item #1
Item #2
Item #3
", - "ok": true, - "expected_task_pass": true - }, - { - "id": "dynamic22-030-below80", - "category": "structured-extraction", - "root_cause": "dynamic_initial_scope_and_owned_records", - "failure_mode": "false-positive", - "case": { - "url": "https://primevue.dev/virtualscroller/", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "table_row", - "minimum_records": 4, - "table_columns": [ - { - "name": "item", - "label": "Item", - "type": "text", - "pattern": "(?i)Item\\s*#\\s*\\d+" - } - ], - "record_identity_fields": [ - "item" - ], - "table_scope": { - "root_selector": ".doc-section-demo", - "root_index": 0, - "row_selector": ".p-virtualscroller-content > div", - "cell_selector": ":scope", - "start_pattern": "(?im)^(?=[ \\t|]*(?:[-*][ \\t]+)?Item[ \\t]*#[ \\t]*\\d+|[ \\t]*\\|[ \\t]*Item[ \\t]*\\|)", - "end_pattern": "(?im)^[ \\t#]*(?:PREVIEW|Usage|Examples|Horizontal)[ \\t]*$", - "plain_row_pattern": "[ \\t|]*(?:[-*][ \\t]+)?(?P(?i:Item)[ \\t]*#[ \\t]*\\d+)[ \\t|]*", - "reference_kind": "fixed_demo", - "reference_rows": [ - { - "item": "Item #0" - }, - { - "item": "Item #1" - }, - { - "item": "Item #2" - }, - { - "item": "Item #3" - } - ] - } + ], + "record_identity_fields": [ + "name" + ], + "table_scope": { + "row_selector": "tbody tr", + "cell_selector": "td", + "start_pattern": "(?m)^(?=\\| Name \\|)", + "end_pattern": "END", + "plain_row_pattern": "(?P[A-Za-z]+) (?P\\d{1,2}(?:st|nd|rd|th) [A-Za-z]{3} \\d{2}) (?POnline|Offline)", + "plain_text_ignored_patterns": [ + "[↦↵](?=[ \\t]*(?:\\||$))", + "\\(external, opens in a new tab or window\\)" + ] + } + } + }, + "artifact": "| Name↦ | Date↦ | State↵ |\n| --- | --- | --- |\n| Ada↦ | ↦ | Online↵ |\nEND", + "ok": true, + "expected_task_pass": false + }, + { + "id": "table-copy-marker-literal-arrow", + "category": "format-normalization", + "root_cause": "table_auxiliary_copy_markers", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/table", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 1, + "table_columns": [ + { + "name": "name", + "label": "Name", + "type": "text" + }, + { + "name": "date", + "label": "Date", + "type": "date_ordinal_en" + }, + { + "name": "state", + "label": "State", + "type": "text", + "pattern": "Online|Offline" } - }, - "artifact": "
Item #0
Item #1
Item #2
", - "ok": true, - "expected_task_pass": false - }, - { - "id": "dynamic22-030-duplicate", - "category": "structured-extraction", - "root_cause": "dynamic_initial_scope_and_owned_records", - "failure_mode": "false-positive", - "case": { - "url": "https://primevue.dev/virtualscroller/", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "table_row", - "minimum_records": 4, - "table_columns": [ - { - "name": "item", - "label": "Item", - "type": "text", - "pattern": "(?i)Item\\s*#\\s*\\d+" - } - ], - "record_identity_fields": [ - "item" - ], - "table_scope": { - "root_selector": ".doc-section-demo", - "root_index": 0, - "row_selector": ".p-virtualscroller-content > div", - "cell_selector": ":scope", - "start_pattern": "(?im)^(?=[ \\t|]*(?:[-*][ \\t]+)?Item[ \\t]*#[ \\t]*\\d+|[ \\t]*\\|[ \\t]*Item[ \\t]*\\|)", - "end_pattern": "(?im)^[ \\t#]*(?:PREVIEW|Usage|Examples|Horizontal)[ \\t]*$", - "plain_row_pattern": "[ \\t|]*(?:[-*][ \\t]+)?(?P(?i:Item)[ \\t]*#[ \\t]*\\d+)[ \\t|]*", - "reference_kind": "fixed_demo", - "reference_rows": [ - { - "item": "Item #0" - }, - { - "item": "Item #1" - }, - { - "item": "Item #2" - }, - { - "item": "Item #3" - } - ] - } + ], + "record_identity_fields": [ + "name" + ], + "table_scope": { + "row_selector": "tbody tr", + "cell_selector": "td", + "start_pattern": "(?m)^(?=\\| Name \\|)", + "end_pattern": "END", + "plain_row_pattern": "(?P[A-Za-z]+) (?P\\d{1,2}(?:st|nd|rd|th) [A-Za-z]{3} \\d{2}) (?POnline|Offline)", + "plain_text_ignored_patterns": [ + "[↦↵](?=[ \\t]*(?:\\||$))", + "\\(external, opens in a new tab or window\\)" + ] + } + } + }, + "artifact": "| Name↦ | Date↦ | State↵ |\n| --- | --- | --- |\n| Ada↦ | 29th Sep 25↦ | On↦line↵ |\nEND", + "ok": true, + "expected_task_pass": false + }, + { + "id": "table-compact-cell-joined", + "category": "format-normalization", + "root_cause": "adjacent_cell_elements_and_empty_column_preservation", + "failure_mode": "false-negative", + "case": { + "url": "https://example.test/table", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 1, + "table_columns": [ + { + "name": "product", + "label": "Product", + "type": "text", + "compact_whitespace": true + }, + { + "name": "category", + "label": "Category", + "type": "text" + }, + { + "name": "price", + "label": "Price", + "type": "currency_usd" + }, + { + "name": "status", + "label": "Status", + "type": "text" } - }, - "artifact": "
Item #0
Item #0
Item #0
Item #0
", - "ok": true, - "expected_task_pass": false - }, - { - "id": "dynamic22-030-wrong-scope", - "category": "structured-extraction", - "root_cause": "dynamic_initial_scope_and_owned_records", - "failure_mode": "false-positive", - "case": { - "url": "https://primevue.dev/virtualscroller/", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "table_row", - "minimum_records": 4, - "table_columns": [ - { - "name": "item", - "label": "Item", - "type": "text", - "pattern": "(?i)Item\\s*#\\s*\\d+" - } - ], - "record_identity_fields": [ - "item" - ], - "table_scope": { - "root_selector": ".doc-section-demo", - "root_index": 0, - "row_selector": ".p-virtualscroller-content > div", - "cell_selector": ":scope", - "start_pattern": "(?im)^(?=[ \\t|]*(?:[-*][ \\t]+)?Item[ \\t]*#[ \\t]*\\d+|[ \\t]*\\|[ \\t]*Item[ \\t]*\\|)", - "end_pattern": "(?im)^[ \\t#]*(?:PREVIEW|Usage|Examples|Horizontal)[ \\t]*$", - "plain_row_pattern": "[ \\t|]*(?:[-*][ \\t]+)?(?P(?i:Item)[ \\t]*#[ \\t]*\\d+)[ \\t|]*", - "reference_kind": "fixed_demo", - "reference_rows": [ - { - "item": "Item #0" - }, - { - "item": "Item #1" - }, - { - "item": "Item #2" - }, - { - "item": "Item #3" - } - ] - } + ], + "record_identity_fields": [ + "product" + ], + "table_scope": { + "root_selector": ".doc-section-demo", + "root_index": 0, + "row_selector": "tbody tr", + "cell_selector": ":scope > td:not(:nth-child(4))", + "start_pattern": "(?im)^(?=[ \\t|]*Product[ \\t|]+Category)", + "end_pattern": "(?im)^[ \\t#]*(?:PREVIEW|Usage)[ \\t]*$", + "plain_row_pattern": "(?P[^\\n]+?\\s+[a-z0-9]{9})\\s+(?PAccessories|Fitness|Clothing)\\s+(?P\\$\\s*\\d+(?:\\.\\d{1,2})?)\\s+(?PIn Stock|Low Stock|Out of Stock)", + "plain_multiline_row_pattern": "(?m)^(?P[^\\n]+\\n[a-z0-9]{9})\\s*\\n[\\s]*(?PAccessories|Fitness|Clothing)[\\s]+(?P\\$\\s*\\d+(?:\\.\\d{1,2})?)[\\s]+(?PIn Stock|Low Stock|Out of Stock)(?=\\n|$)", + "reference_kind": "fixed_demo", + "reference_rows": [ + { + "product": "Bamboo Watch f230fh0g3", + "category": "Accessories", + "price": "$65", + "status": "In Stock" + }, + { + "product": "Black Watch nvklal433", + "category": "Accessories", + "price": "$72", + "status": "In Stock" + }, + { + "product": "Blue Band zz21cz3c1", + "category": "Fitness", + "price": "$79", + "status": "Low Stock" + }, + { + "product": "Blue T-Shirt 244wgerg2", + "category": "Clothing", + "price": "$29", + "status": "In Stock" + }, + { + "product": "Bracelet h456wer53", + "category": "Accessories", + "price": "$15", + "status": "In Stock" + }, + { + "product": "Brown Purse av2231fwg", + "category": "Accessories", + "price": "$120", + "status": "Out of Stock" + }, + { + "product": "Chakra Bracelet bib36pfvm", + "category": "Accessories", + "price": "$32", + "status": "Low Stock" + }, + { + "product": "Galaxy Earrings mbvjkgip5", + "category": "Accessories", + "price": "$34", + "status": "In Stock" + } + ], + "ignored_column_labels": [ + "Rating" + ] + } + } + }, + "artifact": "Product Category\n| Product | Category | Price | Rating | Status |\n| --- | --- | --- | --- | --- |\n| Bamboo Watchf230fh0g3 | Accessories | $65 | | In Stock |\nPREVIEW", + "ok": true, + "expected_task_pass": true + }, + { + "id": "table-compact-cell-wrong-id", + "category": "format-normalization", + "root_cause": "adjacent_cell_elements_and_empty_column_preservation", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/table", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 1, + "table_columns": [ + { + "name": "product", + "label": "Product", + "type": "text", + "compact_whitespace": true + }, + { + "name": "category", + "label": "Category", + "type": "text" + }, + { + "name": "price", + "label": "Price", + "type": "currency_usd" + }, + { + "name": "status", + "label": "Status", + "type": "text" } - }, - "artifact": "
Item #0
Item #1
Item #2
Item #3
", - "ok": true, - "expected_task_pass": false - }, - { - "id": "dynamic22-032-complete", - "category": "structured-extraction", - "root_cause": "dynamic_initial_scope_and_owned_records", - "failure_mode": "false-negative", - "case": { - "url": "https://primeng.dev/virtualscroller", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "table_row", - "minimum_records": 4, - "table_columns": [ - { - "name": "item", - "label": "Item", - "type": "text", - "pattern": "(?i)Item\\s*#\\s*\\d+" - } - ], - "record_identity_fields": [ - "item" - ], - "table_scope": { - "root_selector": ".doc-section-demo", - "root_index": 0, - "row_selector": ".p-virtualscroller-content > div", - "cell_selector": ":scope", - "start_pattern": "(?im)^(?=[ \\t|]*(?:[-*][ \\t]+)?Item[ \\t]*#[ \\t]*\\d+|[ \\t]*\\|[ \\t]*Item[ \\t]*\\|)", - "end_pattern": "(?im)^[ \\t#]*(?:PREVIEW|Usage|Examples|Horizontal)[ \\t]*$", - "plain_row_pattern": "[ \\t|]*(?:[-*][ \\t]+)?(?P(?i:Item)[ \\t]*#[ \\t]*\\d+)[ \\t|]*", - "reference_kind": "fixed_demo", - "reference_rows": [ - { - "item": "Item #0" - }, - { - "item": "Item #1" - }, - { - "item": "Item #2" - }, - { - "item": "Item #3" - } - ] - } + ], + "record_identity_fields": [ + "product" + ], + "table_scope": { + "root_selector": ".doc-section-demo", + "root_index": 0, + "row_selector": "tbody tr", + "cell_selector": ":scope > td:not(:nth-child(4))", + "start_pattern": "(?im)^(?=[ \\t|]*Product[ \\t|]+Category)", + "end_pattern": "(?im)^[ \\t#]*(?:PREVIEW|Usage)[ \\t]*$", + "plain_row_pattern": "(?P[^\\n]+?\\s+[a-z0-9]{9})\\s+(?PAccessories|Fitness|Clothing)\\s+(?P\\$\\s*\\d+(?:\\.\\d{1,2})?)\\s+(?PIn Stock|Low Stock|Out of Stock)", + "plain_multiline_row_pattern": "(?m)^(?P[^\\n]+\\n[a-z0-9]{9})\\s*\\n[\\s]*(?PAccessories|Fitness|Clothing)[\\s]+(?P\\$\\s*\\d+(?:\\.\\d{1,2})?)[\\s]+(?PIn Stock|Low Stock|Out of Stock)(?=\\n|$)", + "reference_kind": "fixed_demo", + "reference_rows": [ + { + "product": "Bamboo Watch f230fh0g3", + "category": "Accessories", + "price": "$65", + "status": "In Stock" + }, + { + "product": "Black Watch nvklal433", + "category": "Accessories", + "price": "$72", + "status": "In Stock" + }, + { + "product": "Blue Band zz21cz3c1", + "category": "Fitness", + "price": "$79", + "status": "Low Stock" + }, + { + "product": "Blue T-Shirt 244wgerg2", + "category": "Clothing", + "price": "$29", + "status": "In Stock" + }, + { + "product": "Bracelet h456wer53", + "category": "Accessories", + "price": "$15", + "status": "In Stock" + }, + { + "product": "Brown Purse av2231fwg", + "category": "Accessories", + "price": "$120", + "status": "Out of Stock" + }, + { + "product": "Chakra Bracelet bib36pfvm", + "category": "Accessories", + "price": "$32", + "status": "Low Stock" + }, + { + "product": "Galaxy Earrings mbvjkgip5", + "category": "Accessories", + "price": "$34", + "status": "In Stock" + } + ], + "ignored_column_labels": [ + "Rating" + ] + } + } + }, + "artifact": "Product Category\n| Product | Category | Price | Rating | Status |\n| --- | --- | --- | --- | --- |\n| Bamboo Watchwrongid | Accessories | $65 | | In Stock |\nPREVIEW", + "ok": true, + "expected_task_pass": false + }, + { + "id": "table-compact-cell-wrong-price", + "category": "format-normalization", + "root_cause": "adjacent_cell_elements_and_empty_column_preservation", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/table", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 1, + "table_columns": [ + { + "name": "product", + "label": "Product", + "type": "text", + "compact_whitespace": true + }, + { + "name": "category", + "label": "Category", + "type": "text" + }, + { + "name": "price", + "label": "Price", + "type": "currency_usd" + }, + { + "name": "status", + "label": "Status", + "type": "text" } - }, - "artifact": "
Item #0
Item #1
Item #2
Item #3
", - "ok": true, - "expected_task_pass": true - }, - { - "id": "dynamic22-032-below80", - "category": "structured-extraction", - "root_cause": "dynamic_initial_scope_and_owned_records", - "failure_mode": "false-positive", - "case": { - "url": "https://primeng.dev/virtualscroller", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "table_row", - "minimum_records": 4, - "table_columns": [ - { - "name": "item", - "label": "Item", - "type": "text", - "pattern": "(?i)Item\\s*#\\s*\\d+" - } - ], - "record_identity_fields": [ - "item" - ], - "table_scope": { - "root_selector": ".doc-section-demo", - "root_index": 0, - "row_selector": ".p-virtualscroller-content > div", - "cell_selector": ":scope", - "start_pattern": "(?im)^(?=[ \\t|]*(?:[-*][ \\t]+)?Item[ \\t]*#[ \\t]*\\d+|[ \\t]*\\|[ \\t]*Item[ \\t]*\\|)", - "end_pattern": "(?im)^[ \\t#]*(?:PREVIEW|Usage|Examples|Horizontal)[ \\t]*$", - "plain_row_pattern": "[ \\t|]*(?:[-*][ \\t]+)?(?P(?i:Item)[ \\t]*#[ \\t]*\\d+)[ \\t|]*", - "reference_kind": "fixed_demo", - "reference_rows": [ - { - "item": "Item #0" - }, - { - "item": "Item #1" - }, - { - "item": "Item #2" - }, - { - "item": "Item #3" - } - ] - } + ], + "record_identity_fields": [ + "product" + ], + "table_scope": { + "root_selector": ".doc-section-demo", + "root_index": 0, + "row_selector": "tbody tr", + "cell_selector": ":scope > td:not(:nth-child(4))", + "start_pattern": "(?im)^(?=[ \\t|]*Product[ \\t|]+Category)", + "end_pattern": "(?im)^[ \\t#]*(?:PREVIEW|Usage)[ \\t]*$", + "plain_row_pattern": "(?P[^\\n]+?\\s+[a-z0-9]{9})\\s+(?PAccessories|Fitness|Clothing)\\s+(?P\\$\\s*\\d+(?:\\.\\d{1,2})?)\\s+(?PIn Stock|Low Stock|Out of Stock)", + "plain_multiline_row_pattern": "(?m)^(?P[^\\n]+\\n[a-z0-9]{9})\\s*\\n[\\s]*(?PAccessories|Fitness|Clothing)[\\s]+(?P\\$\\s*\\d+(?:\\.\\d{1,2})?)[\\s]+(?PIn Stock|Low Stock|Out of Stock)(?=\\n|$)", + "reference_kind": "fixed_demo", + "reference_rows": [ + { + "product": "Bamboo Watch f230fh0g3", + "category": "Accessories", + "price": "$65", + "status": "In Stock" + }, + { + "product": "Black Watch nvklal433", + "category": "Accessories", + "price": "$72", + "status": "In Stock" + }, + { + "product": "Blue Band zz21cz3c1", + "category": "Fitness", + "price": "$79", + "status": "Low Stock" + }, + { + "product": "Blue T-Shirt 244wgerg2", + "category": "Clothing", + "price": "$29", + "status": "In Stock" + }, + { + "product": "Bracelet h456wer53", + "category": "Accessories", + "price": "$15", + "status": "In Stock" + }, + { + "product": "Brown Purse av2231fwg", + "category": "Accessories", + "price": "$120", + "status": "Out of Stock" + }, + { + "product": "Chakra Bracelet bib36pfvm", + "category": "Accessories", + "price": "$32", + "status": "Low Stock" + }, + { + "product": "Galaxy Earrings mbvjkgip5", + "category": "Accessories", + "price": "$34", + "status": "In Stock" + } + ], + "ignored_column_labels": [ + "Rating" + ] + } + } + }, + "artifact": "Product Category\n| Product | Category | Price | Rating | Status |\n| --- | --- | --- | --- | --- |\n| Bamboo Watchf230fh0g3 | Accessories | $99 | | In Stock |\nPREVIEW", + "ok": true, + "expected_task_pass": false + }, + { + "id": "dynamic22-037-next-section-excluded", + "category": "structured-extraction", + "root_cause": "adjacent_virtual_demo_scope_boundary", + "failure_mode": "false-negative", + "case": { + "url": "https://quasar.dev/vue-components/virtual-scroll/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 8, + "table_columns": [ + { + "name": "item", + "label": "Option", + "type": "text", + "pattern": "#?\\d+\\s*-\\s*Option\\s+\\d+" } - }, - "artifact": "
Item #0
Item #1
Item #2
", - "ok": true, - "expected_task_pass": false - }, - { - "id": "dynamic22-032-duplicate", - "category": "structured-extraction", - "root_cause": "dynamic_initial_scope_and_owned_records", - "failure_mode": "false-positive", - "case": { - "url": "https://primeng.dev/virtualscroller", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "table_row", - "minimum_records": 4, - "table_columns": [ - { - "name": "item", - "label": "Item", - "type": "text", - "pattern": "(?i)Item\\s*#\\s*\\d+" - } - ], - "record_identity_fields": [ - "item" - ], - "table_scope": { - "root_selector": ".doc-section-demo", - "root_index": 0, - "row_selector": ".p-virtualscroller-content > div", - "cell_selector": ":scope", - "start_pattern": "(?im)^(?=[ \\t|]*(?:[-*][ \\t]+)?Item[ \\t]*#[ \\t]*\\d+|[ \\t]*\\|[ \\t]*Item[ \\t]*\\|)", - "end_pattern": "(?im)^[ \\t#]*(?:PREVIEW|Usage|Examples|Horizontal)[ \\t]*$", - "plain_row_pattern": "[ \\t|]*(?:[-*][ \\t]+)?(?P(?i:Item)[ \\t]*#[ \\t]*\\d+)[ \\t|]*", - "reference_kind": "fixed_demo", - "reference_rows": [ - { - "item": "Item #0" - }, - { - "item": "Item #1" - }, - { - "item": "Item #2" - }, - { - "item": "Item #3" - } - ] - } + ], + "record_identity_fields": [ + "item" + ], + "table_scope": { + "root_selector": ".q-virtual-scroll", + "root_index": 0, + "row_selector": ".q-virtual-scroll__content > .q-item", + "cell_selector": ":scope", + "start_pattern": "(?m)^\\s*#{0,6}\\s*Basic\\s*$", + "end_pattern": "(?m)^[ \\t#]*Horizontal[ \\t]*$", + "plain_row_pattern": "(?P#?\\d+\\s*-\\s*Option\\s+\\d+)", + "reference_kind": "fixed_demo", + "reference_rows": [ + { + "item": "#0 - Option 1" + }, + { + "item": "#1 - Option 2" + }, + { + "item": "#2 - Option 3" + }, + { + "item": "#3 - Option 4" + }, + { + "item": "#4 - Option 5" + }, + { + "item": "#5 - Option 6" + }, + { + "item": "#6 - Option 7" + }, + { + "item": "#7 - Option 8" + }, + { + "item": "#8 - Option 9" + }, + { + "item": "#9 - Option 10" + } + ] + } + } + }, + "artifact": "### Basic\n\n#0 - Option 1\n#1 - Option 2\n#2 - Option 3\n#3 - Option 4\n#4 - Option 5\n#5 - Option 6\n#6 - Option 7\n#7 - Option 8\n\n### Horizontal\n\n#0 - Option 1\n#1 - Option 2\n#2 - Option 3\n#3 - Option 4\n#4 - Option 5\n#5 - Option 6\n#6 - Option 7\n#7 - Option 8", + "ok": true, + "expected_task_pass": true + }, + { + "id": "dynamic22-037-next-section-cannot-fill", + "category": "structured-extraction", + "root_cause": "adjacent_virtual_demo_scope_boundary", + "failure_mode": "false-positive", + "case": { + "url": "https://quasar.dev/vue-components/virtual-scroll/", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 8, + "table_columns": [ + { + "name": "item", + "label": "Option", + "type": "text", + "pattern": "#?\\d+\\s*-\\s*Option\\s+\\d+" } - }, - "artifact": "
Item #0
Item #0
Item #0
Item #0
", - "ok": true, - "expected_task_pass": false - }, - { - "id": "dynamic22-032-wrong-scope", - "category": "structured-extraction", - "root_cause": "dynamic_initial_scope_and_owned_records", - "failure_mode": "false-positive", - "case": { - "url": "https://primeng.dev/virtualscroller", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "table_row", - "minimum_records": 4, - "table_columns": [ - { - "name": "item", - "label": "Item", - "type": "text", - "pattern": "(?i)Item\\s*#\\s*\\d+" - } - ], - "record_identity_fields": [ - "item" - ], - "table_scope": { - "root_selector": ".doc-section-demo", - "root_index": 0, - "row_selector": ".p-virtualscroller-content > div", - "cell_selector": ":scope", - "start_pattern": "(?im)^(?=[ \\t|]*(?:[-*][ \\t]+)?Item[ \\t]*#[ \\t]*\\d+|[ \\t]*\\|[ \\t]*Item[ \\t]*\\|)", - "end_pattern": "(?im)^[ \\t#]*(?:PREVIEW|Usage|Examples|Horizontal)[ \\t]*$", - "plain_row_pattern": "[ \\t|]*(?:[-*][ \\t]+)?(?P(?i:Item)[ \\t]*#[ \\t]*\\d+)[ \\t|]*", - "reference_kind": "fixed_demo", - "reference_rows": [ - { - "item": "Item #0" - }, - { - "item": "Item #1" - }, - { - "item": "Item #2" - }, - { - "item": "Item #3" - } - ] - } + ], + "record_identity_fields": [ + "item" + ], + "table_scope": { + "root_selector": ".q-virtual-scroll", + "root_index": 0, + "row_selector": ".q-virtual-scroll__content > .q-item", + "cell_selector": ":scope", + "start_pattern": "(?m)^\\s*#{0,6}\\s*Basic\\s*$", + "end_pattern": "(?m)^[ \\t#]*Horizontal[ \\t]*$", + "plain_row_pattern": "(?P#?\\d+\\s*-\\s*Option\\s+\\d+)", + "reference_kind": "fixed_demo", + "reference_rows": [ + { + "item": "#0 - Option 1" + }, + { + "item": "#1 - Option 2" + }, + { + "item": "#2 - Option 3" + }, + { + "item": "#3 - Option 4" + }, + { + "item": "#4 - Option 5" + }, + { + "item": "#5 - Option 6" + }, + { + "item": "#6 - Option 7" + }, + { + "item": "#7 - Option 8" + }, + { + "item": "#8 - Option 9" + }, + { + "item": "#9 - Option 10" + } + ] + } + } + }, + "artifact": "### Basic\n\n#0 - Option 1\n#1 - Option 2\n\n### Horizontal\n\n#0 - Option 1\n#1 - Option 2\n#2 - Option 3\n#3 - Option 4\n#4 - Option 5\n#5 - Option 6\n#6 - Option 7\n#7 - Option 8", + "ok": true, + "expected_task_pass": false + }, + { + "id": "named-visible-text-visible-title", + "category": "structured-extraction", + "root_cause": "image_alt_is_not_visible_record_name", + "failure_mode": "false-negative", + "case": { + "url": "https://example.test/list", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 1, + "hosts": [ + "example.test" + ], + "entry_path": "/item/\\d+", + "name_validation": "nonempty" + } + }, + "artifact": "[![Decorative logo](/logo.png)Actual title](/item/1)", + "ok": true, + "expected_task_pass": true + }, + { + "id": "named-visible-text-image-only", + "category": "structured-extraction", + "root_cause": "image_alt_is_not_visible_record_name", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/list", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 1, + "hosts": [ + "example.test" + ], + "entry_path": "/item/\\d+", + "name_validation": "nonempty" + } + }, + "artifact": "[![Actual title](/logo.png)](/item/1)", + "ok": true, + "expected_task_pass": false + }, + { + "id": "named-visible-text-image-plus-action", + "category": "structured-extraction", + "root_cause": "image_alt_is_not_visible_record_name", + "failure_mode": "false-positive", + "case": { + "url": "https://example.test/list", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 1, + "hosts": [ + "example.test" + ], + "entry_path": "/item/\\d+", + "name_validation": "nonempty" + } + }, + "artifact": "[![Actual title](/logo.png)Read more](/item/1)", + "ok": true, + "expected_task_pass": false + }, + { + "id": "joyn-completed-window-67", + "category": "structured-extraction", + "root_cause": "premature_partial_dynamic_collection_denominator", + "failure_mode": "false-negative", + "case": { + "url": "https://www.joyn.de/filme", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 67, + "hosts": [ + "www.joyn.de" + ], + "entry_path": "/filme/(?!genre(?:/|$))[^/]+", + "name_validation": "nonempty", + "entry_scope": { + "html_selector": "main div[class*=\"CardStandard_StandardCardWrapper\"]", + "title_selector": "h3", + "link_selector": "a[data-testid=\"LL--movie\"]" + } + } + }, + "artifact": "

Movie 0

Movie 1

Movie 2

Movie 3

Movie 4

Movie 5

Movie 6

Movie 7

Movie 8

Movie 9

Movie 10

Movie 11

Movie 12

Movie 13

Movie 14

Movie 15

Movie 16

Movie 17

Movie 18

Movie 19

Movie 20

Movie 21

Movie 22

Movie 23

Movie 24

Movie 25

Movie 26

Movie 27

Movie 28

Movie 29

Movie 30

Movie 31

Movie 32

Movie 33

Movie 34

Movie 35

Movie 36

Movie 37

Movie 38

Movie 39

Movie 40

Movie 41

Movie 42

Movie 43

Movie 44

Movie 45

Movie 46

Movie 47

Movie 48

Movie 49

Movie 50

Movie 51

Movie 52

Movie 53

Movie 54

Movie 55

Movie 56

Movie 57

Movie 58

Movie 59

Movie 60

Movie 61

Movie 62

Movie 63

Movie 64

Movie 65

Movie 66

", + "ok": true, + "expected_task_pass": true + }, + { + "id": "joyn-completed-window-27", + "category": "structured-extraction", + "root_cause": "premature_partial_dynamic_collection_denominator", + "failure_mode": "false-positive", + "case": { + "url": "https://www.joyn.de/filme", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 67, + "hosts": [ + "www.joyn.de" + ], + "entry_path": "/filme/(?!genre(?:/|$))[^/]+", + "name_validation": "nonempty", + "entry_scope": { + "html_selector": "main div[class*=\"CardStandard_StandardCardWrapper\"]", + "title_selector": "h3", + "link_selector": "a[data-testid=\"LL--movie\"]" + } + } + }, + "artifact": "

Movie 0

Movie 1

Movie 2

Movie 3

Movie 4

Movie 5

Movie 6

Movie 7

Movie 8

Movie 9

Movie 10

Movie 11

Movie 12

Movie 13

Movie 14

Movie 15

Movie 16

Movie 17

Movie 18

Movie 19

Movie 20

Movie 21

Movie 22

Movie 23

Movie 24

Movie 25

Movie 26

", + "ok": true, + "expected_task_pass": false + }, + { + "id": "nintendo-owned-software-20", + "category": "structured-extraction", + "root_cause": "adjacent_search_result_regions", + "failure_mode": "false-negative", + "case": { + "url": "https://www.nintendo.com/jp/search/index.html?q=Nintendo", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 20, + "hosts": [ + "www.nintendo.com", + "store-jp.nintendo.com" + ], + "entry_path": "(?:/jp/games/(?:switch2|switch)/[A-Za-z0-9-]+/index\\.html|/item/software/[A-Za-z0-9]+)", + "name_validation": "nonempty", + "entry_scope": { + "html_selector": ".nc3-c-softCard", + "title_selector": ".nc3-c-softCard__name", + "link_selector": "a[href]:has(.nc3-c-softCard__name)", + "start_pattern": "(?m)^#{1,6}[ \\t]+ゲームソフトをさがす[ \\t]*$", + "end_pattern": "表示価格はすべて" + } + } + }, + "artifact": "### ページをさがす\n[Official website 0](https://www.nintendo.com/jp/games/switch2/game0/index.html)\n[Official website 1](https://www.nintendo.com/jp/games/switch2/game1/index.html)\n[Official website 2](https://www.nintendo.com/jp/games/switch2/game2/index.html)\n[Official website 3](https://www.nintendo.com/jp/games/switch2/game3/index.html)\n[Official website 4](https://www.nintendo.com/jp/games/switch2/game4/index.html)\n[Official website 5](https://www.nintendo.com/jp/games/switch2/game5/index.html)\n[Official website 6](https://www.nintendo.com/jp/games/switch2/game6/index.html)\n[Official website 7](https://www.nintendo.com/jp/games/switch2/game7/index.html)\n[Official website 8](https://www.nintendo.com/jp/games/switch2/game8/index.html)\n[Official website 9](https://www.nintendo.com/jp/games/switch2/game9/index.html)\n[Official website 10](https://www.nintendo.com/jp/games/switch2/game10/index.html)\n[Official website 11](https://www.nintendo.com/jp/games/switch2/game11/index.html)\n[Official website 12](https://www.nintendo.com/jp/games/switch2/game12/index.html)\n[Official website 13](https://www.nintendo.com/jp/games/switch2/game13/index.html)\n[Official website 14](https://www.nintendo.com/jp/games/switch2/game14/index.html)\n[Official website 15](https://www.nintendo.com/jp/games/switch2/game15/index.html)\n[Official website 16](https://www.nintendo.com/jp/games/switch2/game16/index.html)\n[Official website 17](https://www.nintendo.com/jp/games/switch2/game17/index.html)\n[Official website 18](https://www.nintendo.com/jp/games/switch2/game18/index.html)\n[Official website 19](https://www.nintendo.com/jp/games/switch2/game19/index.html)\n### ゲームソフトをさがす\n[Game 0](https://store-jp.nintendo.com/item/software/D70010000000000)\n[Game 1](https://store-jp.nintendo.com/item/software/D70010000000001)\n[Game 2](https://store-jp.nintendo.com/item/software/D70010000000002)\n[Game 3](https://store-jp.nintendo.com/item/software/D70010000000003)\n[Game 4](https://store-jp.nintendo.com/item/software/D70010000000004)\n[Game 5](https://store-jp.nintendo.com/item/software/D70010000000005)\n[Game 6](https://store-jp.nintendo.com/item/software/D70010000000006)\n[Game 7](https://store-jp.nintendo.com/item/software/D70010000000007)\n[Game 8](https://store-jp.nintendo.com/item/software/D70010000000008)\n[Game 9](https://store-jp.nintendo.com/item/software/D70010000000009)\n[Game 10](https://store-jp.nintendo.com/item/software/D70010000000010)\n[Game 11](https://store-jp.nintendo.com/item/software/D70010000000011)\n[Game 12](https://store-jp.nintendo.com/item/software/D70010000000012)\n[Game 13](https://store-jp.nintendo.com/item/software/D70010000000013)\n[Game 14](https://store-jp.nintendo.com/item/software/D70010000000014)\n[Game 15](https://store-jp.nintendo.com/item/software/D70010000000015)\n[Game 16](https://store-jp.nintendo.com/item/software/D70010000000016)\n[Game 17](https://store-jp.nintendo.com/item/software/D70010000000017)\n[Game 18](https://store-jp.nintendo.com/item/software/D70010000000018)\n[Game 19](https://store-jp.nintendo.com/item/software/D70010000000019)\n表示価格はすべて", + "ok": true, + "expected_task_pass": true + }, + { + "id": "nintendo-owned-software-2", + "category": "structured-extraction", + "root_cause": "adjacent_search_result_regions", + "failure_mode": "false-positive", + "case": { + "url": "https://www.nintendo.com/jp/search/index.html?q=Nintendo", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 20, + "hosts": [ + "www.nintendo.com", + "store-jp.nintendo.com" + ], + "entry_path": "(?:/jp/games/(?:switch2|switch)/[A-Za-z0-9-]+/index\\.html|/item/software/[A-Za-z0-9]+)", + "name_validation": "nonempty", + "entry_scope": { + "html_selector": ".nc3-c-softCard", + "title_selector": ".nc3-c-softCard__name", + "link_selector": "a[href]:has(.nc3-c-softCard__name)", + "start_pattern": "(?m)^#{1,6}[ \\t]+ゲームソフトをさがす[ \\t]*$", + "end_pattern": "表示価格はすべて" + } + } + }, + "artifact": "### ページをさがす\n[Official website 0](https://www.nintendo.com/jp/games/switch2/game0/index.html)\n[Official website 1](https://www.nintendo.com/jp/games/switch2/game1/index.html)\n[Official website 2](https://www.nintendo.com/jp/games/switch2/game2/index.html)\n[Official website 3](https://www.nintendo.com/jp/games/switch2/game3/index.html)\n[Official website 4](https://www.nintendo.com/jp/games/switch2/game4/index.html)\n[Official website 5](https://www.nintendo.com/jp/games/switch2/game5/index.html)\n[Official website 6](https://www.nintendo.com/jp/games/switch2/game6/index.html)\n[Official website 7](https://www.nintendo.com/jp/games/switch2/game7/index.html)\n[Official website 8](https://www.nintendo.com/jp/games/switch2/game8/index.html)\n[Official website 9](https://www.nintendo.com/jp/games/switch2/game9/index.html)\n[Official website 10](https://www.nintendo.com/jp/games/switch2/game10/index.html)\n[Official website 11](https://www.nintendo.com/jp/games/switch2/game11/index.html)\n[Official website 12](https://www.nintendo.com/jp/games/switch2/game12/index.html)\n[Official website 13](https://www.nintendo.com/jp/games/switch2/game13/index.html)\n[Official website 14](https://www.nintendo.com/jp/games/switch2/game14/index.html)\n[Official website 15](https://www.nintendo.com/jp/games/switch2/game15/index.html)\n[Official website 16](https://www.nintendo.com/jp/games/switch2/game16/index.html)\n[Official website 17](https://www.nintendo.com/jp/games/switch2/game17/index.html)\n[Official website 18](https://www.nintendo.com/jp/games/switch2/game18/index.html)\n[Official website 19](https://www.nintendo.com/jp/games/switch2/game19/index.html)\n### ゲームソフトをさがす\n[Game 0](https://store-jp.nintendo.com/item/software/D70010000000000)\n[Game 1](https://store-jp.nintendo.com/item/software/D70010000000001)\n表示価格はすべて", + "ok": true, + "expected_task_pass": false + }, + { + "id": "completion100-062-1", + "category": "structured-extraction", + "root_cause": "dynamic_membership_must_not_be_frozen", + "failure_mode": "false-negative", + "case": { + "url": "https://example.org", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "example.org" + ], + "entry_path": "/item/[0-9]+", + "name_validation": "nonempty" + } + }, + "artifact": "[Fresh title A](/item/91)\n[Fresh title B](/item/92)", + "ok": true, + "expected_task_pass": true + }, + { + "id": "completion100-062-2", + "category": "structured-extraction", + "root_cause": "dynamic_membership_must_not_be_frozen", + "failure_mode": "false-positive", + "case": { + "url": "https://example.org", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "example.org" + ], + "entry_path": "/item/[0-9]+", + "name_validation": "nonempty" + } + }, + "artifact": "[Fresh title A](/item/91)\n[Fresh title A](/item/91)\n[Invalid](/about)", + "ok": true, + "expected_task_pass": false + }, + { + "id": "completion100-067-3", + "category": "structured-extraction", + "root_cause": "dynamic_membership_must_not_be_frozen", + "failure_mode": "false-negative", + "case": { + "url": "https://example.org", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "example.org" + ], + "entry_path": "/item/[0-9]+", + "name_validation": "nonempty" + } + }, + "artifact": "[Fresh title A](/item/91)\n[Fresh title B](/item/92)", + "ok": true, + "expected_task_pass": true + }, + { + "id": "completion100-067-4", + "category": "structured-extraction", + "root_cause": "dynamic_membership_must_not_be_frozen", + "failure_mode": "false-positive", + "case": { + "url": "https://example.org", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "example.org" + ], + "entry_path": "/item/[0-9]+", + "name_validation": "nonempty" + } + }, + "artifact": "[Fresh title A](/item/91)\n[Fresh title A](/item/91)\n[Invalid](/about)", + "ok": true, + "expected_task_pass": false + }, + { + "id": "completion100-045-5", + "category": "structured-extraction", + "root_cause": "dynamic_membership_must_not_be_frozen", + "failure_mode": "false-negative", + "case": { + "url": "https://example.org", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "example.org" + ], + "entry_path": "/item/[0-9]+", + "name_validation": "nonempty" + } + }, + "artifact": "[Fresh title A](/item/91)\n[Fresh title B](/item/92)", + "ok": true, + "expected_task_pass": true + }, + { + "id": "completion100-045-6", + "category": "structured-extraction", + "root_cause": "dynamic_membership_must_not_be_frozen", + "failure_mode": "false-positive", + "case": { + "url": "https://example.org", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "example.org" + ], + "entry_path": "/item/[0-9]+", + "name_validation": "nonempty" + } + }, + "artifact": "[Fresh title A](/item/91)\n[Fresh title A](/item/91)\n[Invalid](/about)", + "ok": true, + "expected_task_pass": false + }, + { + "id": "completion100-095-7", + "category": "structured-extraction", + "root_cause": "dynamic_membership_must_not_be_frozen", + "failure_mode": "false-negative", + "case": { + "url": "https://example.org", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "example.org" + ], + "entry_path": "/item/[0-9]+", + "name_validation": "nonempty" + } + }, + "artifact": "[Fresh title A](/item/91)\n[Fresh title B](/item/92)", + "ok": true, + "expected_task_pass": true + }, + { + "id": "completion100-095-8", + "category": "structured-extraction", + "root_cause": "dynamic_membership_must_not_be_frozen", + "failure_mode": "false-positive", + "case": { + "url": "https://example.org", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "example.org" + ], + "entry_path": "/item/[0-9]+", + "name_validation": "nonempty" + } + }, + "artifact": "[Fresh title A](/item/91)\n[Fresh title A](/item/91)\n[Invalid](/about)", + "ok": true, + "expected_task_pass": false + }, + { + "id": "completion100-011-9", + "category": "structured-extraction", + "root_cause": "table_fields_and_example_boundaries", + "failure_mode": "false-negative", + "case": { + "url": "https://example.org/table", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 4, + "stable_text_anchors": [], + "duplicate_policy": "deduplicate_identical", + "table_columns": [ + { + "name": "f0", + "label": "Product", + "type": "text" + }, + { + "name": "f1", + "label": "Category", + "type": "text" + }, + { + "name": "f2", + "label": "Price", + "type": "number" } - }, - "artifact": "
Item #0
Item #1
Item #2
Item #3
", - "ok": true, - "expected_task_pass": false - }, - { - "id": "dynamic22-098-complete", - "category": "structured-extraction", - "root_cause": "dynamic_initial_scope_and_owned_records", - "failure_mode": "false-negative", - "case": { - "url": "https://shahid.mbc.net/ar", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 5, - "hosts": [ - "shahid.mbc.net" - ], - "entry_path": "/ar/player/episodes/[^/]+/id-[0-9]+", - "entry_identity_pattern": "/id-([0-9]+)$", - "name_validation": "nonempty", - "entry_scope": { - "html_selector": ".carousel-item", - "title_selector": "h3", - "link_selector": "a[href*=\"/ar/player/episodes/\"]", - "start_pattern": "الأكثر مشاهدة الأن", - "end_pattern": "اتصل بنا" - } + ], + "record_identity_fields": [ + "f0" + ], + "table_scope": { + "root_selector": "main table", + "root_index": 0, + "row_selector": "tr", + "cell_selector": "td", + "start_pattern": "(?=^\\s*\\|?\\s*Product(?:\\s|\\|))", + "end_pattern": "^\\s*#{1,6}\\s", + "plain_row_pattern": "(?P[^\t\n]+)\\t(?P[^\t\n]+)\\t(?P[^\t\n]+)", + "markdown_table_limit": 1 + } + } + }, + "artifact": "| Product | Category | Price |\n| --- | --- | --- |\n| Alpha | Alpha | 1.5 |\n| Beta | Beta | 1.5 |\n| Gamma | Gamma | 1.5 |\n| Delta | Delta | 1.5 |", + "ok": true, + "expected_task_pass": true + }, + { + "id": "completion100-011-10", + "category": "structured-extraction", + "root_cause": "table_fields_and_example_boundaries", + "failure_mode": "false-positive", + "case": { + "url": "https://example.org/table", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 4, + "stable_text_anchors": [], + "duplicate_policy": "deduplicate_identical", + "table_columns": [ + { + "name": "f0", + "label": "Product", + "type": "text" + }, + { + "name": "f1", + "label": "Category", + "type": "text" + }, + { + "name": "f2", + "label": "Price", + "type": "number" } - }, - "artifact": "

الموسم 2، الحلقة 1

الموسم 2، الحلقة 2

الموسم 2، الحلقة 3

الموسم 2، الحلقة 4

الموسم 2، الحلقة 5

", - "ok": true, - "expected_task_pass": true - }, - { - "id": "dynamic22-098-below80", - "category": "structured-extraction", - "root_cause": "dynamic_initial_scope_and_owned_records", - "failure_mode": "false-positive", - "case": { - "url": "https://shahid.mbc.net/ar", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 5, - "hosts": [ - "shahid.mbc.net" - ], - "entry_path": "/ar/player/episodes/[^/]+/id-[0-9]+", - "entry_identity_pattern": "/id-([0-9]+)$", - "name_validation": "nonempty", - "entry_scope": { - "html_selector": ".carousel-item", - "title_selector": "h3", - "link_selector": "a[href*=\"/ar/player/episodes/\"]", - "start_pattern": "الأكثر مشاهدة الأن", - "end_pattern": "اتصل بنا" - } + ], + "record_identity_fields": [ + "f0" + ], + "table_scope": { + "root_selector": "main table", + "root_index": 0, + "row_selector": "tr", + "cell_selector": "td", + "start_pattern": "(?=^\\s*\\|?\\s*Product(?:\\s|\\|))", + "end_pattern": "^\\s*#{1,6}\\s", + "plain_row_pattern": "(?P[^\t\n]+)\\t(?P[^\t\n]+)\\t(?P[^\t\n]+)", + "markdown_table_limit": 1 + } + } + }, + "artifact": "| Product | Category | Price |\n| --- | --- | --- |\n| Alpha | Alpha | |\n| Beta | Beta | |\n| Gamma | Gamma | |\n| Delta | Delta | |", + "ok": true, + "expected_task_pass": false + }, + { + "id": "completion100-011-11", + "category": "structured-extraction", + "root_cause": "table_fields_and_example_boundaries", + "failure_mode": "false-positive", + "case": { + "url": "https://example.org/table", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 4, + "stable_text_anchors": [], + "duplicate_policy": "deduplicate_identical", + "table_columns": [ + { + "name": "f0", + "label": "Product", + "type": "text" + }, + { + "name": "f1", + "label": "Category", + "type": "text" + }, + { + "name": "f2", + "label": "Price", + "type": "number" } - }, - "artifact": "

الموسم 2، الحلقة 1

الموسم 2، الحلقة 2

الموسم 2، الحلقة 3

الموسم 2، الحلقة 4

", - "ok": true, - "expected_task_pass": false - }, - { - "id": "dynamic22-098-duplicate", - "category": "structured-extraction", - "root_cause": "dynamic_initial_scope_and_owned_records", - "failure_mode": "false-positive", - "case": { - "url": "https://shahid.mbc.net/ar", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 5, - "hosts": [ - "shahid.mbc.net" - ], - "entry_path": "/ar/player/episodes/[^/]+/id-[0-9]+", - "entry_identity_pattern": "/id-([0-9]+)$", - "name_validation": "nonempty", - "entry_scope": { - "html_selector": ".carousel-item", - "title_selector": "h3", - "link_selector": "a[href*=\"/ar/player/episodes/\"]", - "start_pattern": "الأكثر مشاهدة الأن", - "end_pattern": "اتصل بنا" - } + ], + "record_identity_fields": [ + "f0" + ], + "table_scope": { + "root_selector": "main table", + "root_index": 0, + "row_selector": "tr", + "cell_selector": "td", + "start_pattern": "(?=^\\s*\\|?\\s*Product(?:\\s|\\|))", + "end_pattern": "^\\s*#{1,6}\\s", + "plain_row_pattern": "(?P[^\t\n]+)\\t(?P[^\t\n]+)\\t(?P[^\t\n]+)", + "markdown_table_limit": 1 + } + } + }, + "artifact": "| Product | Category | Price |\n| --- | --- | --- |\n| Alpha | Alpha | 1.5 |\n| Alpha | Alpha | 1.5 |\n| Alpha | Alpha | 1.5 |\n| Alpha | Alpha | 1.5 |", + "ok": true, + "expected_task_pass": false + }, + { + "id": "completion100-011-12", + "category": "structured-extraction", + "root_cause": "table_fields_and_example_boundaries", + "failure_mode": "false-positive", + "case": { + "url": "https://example.org/table", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 4, + "stable_text_anchors": [], + "duplicate_policy": "deduplicate_identical", + "table_columns": [ + { + "name": "f0", + "label": "Product", + "type": "text" + }, + { + "name": "f1", + "label": "Category", + "type": "text" + }, + { + "name": "f2", + "label": "Price", + "type": "number" } - }, - "artifact": "

الموسم 2، الحلقة 1

الموسم 2، الحلقة 2

الموسم 2، الحلقة 3

الموسم 2، الحلقة 4

الموسم 2، الحلقة 5

", - "ok": true, - "expected_task_pass": false - }, - { - "id": "dynamic22-098-wrong-scope", - "category": "structured-extraction", - "root_cause": "dynamic_initial_scope_and_owned_records", - "failure_mode": "false-positive", - "case": { - "url": "https://shahid.mbc.net/ar", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 5, - "hosts": [ - "shahid.mbc.net" - ], - "entry_path": "/ar/player/episodes/[^/]+/id-[0-9]+", - "entry_identity_pattern": "/id-([0-9]+)$", - "name_validation": "nonempty", - "entry_scope": { - "html_selector": ".carousel-item", - "title_selector": "h3", - "link_selector": "a[href*=\"/ar/player/episodes/\"]", - "start_pattern": "الأكثر مشاهدة الأن", - "end_pattern": "اتصل بنا" - } + ], + "record_identity_fields": [ + "f0" + ], + "table_scope": { + "root_selector": "main table", + "root_index": 0, + "row_selector": "tr", + "cell_selector": "td", + "start_pattern": "(?=^\\s*\\|?\\s*Product(?:\\s|\\|))", + "end_pattern": "^\\s*#{1,6}\\s", + "plain_row_pattern": "(?P[^\t\n]+)\\t(?P[^\t\n]+)\\t(?P[^\t\n]+)", + "markdown_table_limit": 1 + } + } + }, + "artifact": "| Product | Category | Price |\n| --- | --- | --- |\n| Alpha | Alpha | 1.5 |\n\n| Product | Category | Price |\n| --- | --- | --- |\n| Alpha | Alpha | 1.5 |\n| Beta | Beta | 1.5 |\n| Gamma | Gamma | 1.5 |\n| Delta | Delta | 1.5 |", + "ok": true, + "expected_task_pass": false + }, + { + "id": "completion100-024-13", + "category": "structured-extraction", + "root_cause": "table_fields_and_example_boundaries", + "failure_mode": "false-negative", + "case": { + "url": "https://example.org/table", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 2, + "stable_text_anchors": [], + "duplicate_policy": "deduplicate_identical", + "table_columns": [ + { + "name": "f0", + "label": "Name", + "type": "text" + }, + { + "name": "f1", + "label": "Email", + "type": "token", + "pattern": "[^\\s@]+@[^\\s@]+\\.[^\\s@]+" + }, + { + "name": "f2", + "label": "Phone Number", + "type": "token", + "pattern": "[+()\\d][\\d ()+.-]+" } - }, - "artifact": "

الموسم 2، الحلقة 1

الموسم 2، الحلقة 2

الموسم 2، الحلقة 3

الموسم 2، الحلقة 4

الموسم 2، الحلقة 5

", - "ok": true, - "expected_task_pass": false - }, - { - "id": "dynamic22-061-complete", - "category": "structured-extraction", - "root_cause": "dynamic_initial_scope_and_owned_records", - "failure_mode": "false-negative", - "case": { - "url": "https://juejin.cn/", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 16, - "hosts": [ - "juejin.cn" - ], - "entry_path": "/post/[0-9]+", - "name_validation": "nonempty", - "entry_scope": { - "html_selector": ".entry-list .entry a.title" - } + ], + "record_identity_fields": [ + "f0" + ], + "table_scope": { + "root_selector": "main table", + "root_index": 0, + "row_selector": "tr", + "cell_selector": "td", + "start_pattern": "(?=^\\s*\\|?\\s*Name(?:\\s|\\|))", + "end_pattern": "^\\s*#{1,6}\\s", + "plain_row_pattern": "(?P[^\t\n]+)\\t(?P[^\t\n]+)\\t(?P[^\t\n]+)", + "markdown_table_limit": 1 + } + } + }, + "artifact": "| Name | Email | Phone Number |\n| --- | --- | --- |\n| Alpha | person@example.org | (123) 456 789 |\n| Beta | person@example.org | (123) 456 789 |\n| Gamma | person@example.org | (123) 456 789 |", + "ok": true, + "expected_task_pass": true + }, + { + "id": "completion100-024-14", + "category": "structured-extraction", + "root_cause": "table_fields_and_example_boundaries", + "failure_mode": "false-positive", + "case": { + "url": "https://example.org/table", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 2, + "stable_text_anchors": [], + "duplicate_policy": "deduplicate_identical", + "table_columns": [ + { + "name": "f0", + "label": "Name", + "type": "text" + }, + { + "name": "f1", + "label": "Email", + "type": "token", + "pattern": "[^\\s@]+@[^\\s@]+\\.[^\\s@]+" + }, + { + "name": "f2", + "label": "Phone Number", + "type": "token", + "pattern": "[+()\\d][\\d ()+.-]+" } - }, - "artifact": "
新文章1
新文章2
新文章3
新文章4
新文章5
新文章6
新文章7
新文章8
新文章9
新文章10
新文章11
新文章12
新文章13
新文章14
新文章15
新文章16
", - "ok": true, - "expected_task_pass": true - }, - { - "id": "dynamic22-061-below80", - "category": "structured-extraction", - "root_cause": "dynamic_initial_scope_and_owned_records", - "failure_mode": "false-positive", - "case": { - "url": "https://juejin.cn/", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 16, - "hosts": [ - "juejin.cn" - ], - "entry_path": "/post/[0-9]+", - "name_validation": "nonempty", - "entry_scope": { - "html_selector": ".entry-list .entry a.title" - } + ], + "record_identity_fields": [ + "f0" + ], + "table_scope": { + "root_selector": "main table", + "root_index": 0, + "row_selector": "tr", + "cell_selector": "td", + "start_pattern": "(?=^\\s*\\|?\\s*Name(?:\\s|\\|))", + "end_pattern": "^\\s*#{1,6}\\s", + "plain_row_pattern": "(?P[^\t\n]+)\\t(?P[^\t\n]+)\\t(?P[^\t\n]+)", + "markdown_table_limit": 1 + } + } + }, + "artifact": "| Name | Email | Phone Number |\n| --- | --- | --- |\n| Alpha | person@example.org | |\n| Beta | person@example.org | |\n| Gamma | person@example.org | |", + "ok": true, + "expected_task_pass": false + }, + { + "id": "completion100-024-15", + "category": "structured-extraction", + "root_cause": "table_fields_and_example_boundaries", + "failure_mode": "false-positive", + "case": { + "url": "https://example.org/table", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 2, + "stable_text_anchors": [], + "duplicate_policy": "deduplicate_identical", + "table_columns": [ + { + "name": "f0", + "label": "Name", + "type": "text" + }, + { + "name": "f1", + "label": "Email", + "type": "token", + "pattern": "[^\\s@]+@[^\\s@]+\\.[^\\s@]+" + }, + { + "name": "f2", + "label": "Phone Number", + "type": "token", + "pattern": "[+()\\d][\\d ()+.-]+" } - }, - "artifact": "
新文章1
新文章2
新文章3
新文章4
新文章5
新文章6
新文章7
新文章8
新文章9
新文章10
新文章11
新文章12
新文章13
新文章14
新文章15
", - "ok": true, - "expected_task_pass": false - }, - { - "id": "dynamic22-061-duplicate", - "category": "structured-extraction", - "root_cause": "dynamic_initial_scope_and_owned_records", - "failure_mode": "false-positive", - "case": { - "url": "https://juejin.cn/", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 16, - "hosts": [ - "juejin.cn" - ], - "entry_path": "/post/[0-9]+", - "name_validation": "nonempty", - "entry_scope": { - "html_selector": ".entry-list .entry a.title" - } + ], + "record_identity_fields": [ + "f0" + ], + "table_scope": { + "root_selector": "main table", + "root_index": 0, + "row_selector": "tr", + "cell_selector": "td", + "start_pattern": "(?=^\\s*\\|?\\s*Name(?:\\s|\\|))", + "end_pattern": "^\\s*#{1,6}\\s", + "plain_row_pattern": "(?P[^\t\n]+)\\t(?P[^\t\n]+)\\t(?P[^\t\n]+)", + "markdown_table_limit": 1 + } + } + }, + "artifact": "| Name | Email | Phone Number |\n| --- | --- | --- |\n| Alpha | person@example.org | (123) 456 789 |\n| Alpha | person@example.org | (123) 456 789 |\n| Alpha | person@example.org | (123) 456 789 |", + "ok": true, + "expected_task_pass": false + }, + { + "id": "completion100-024-16", + "category": "structured-extraction", + "root_cause": "table_fields_and_example_boundaries", + "failure_mode": "false-positive", + "case": { + "url": "https://example.org/table", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 2, + "stable_text_anchors": [], + "duplicate_policy": "deduplicate_identical", + "table_columns": [ + { + "name": "f0", + "label": "Name", + "type": "text" + }, + { + "name": "f1", + "label": "Email", + "type": "token", + "pattern": "[^\\s@]+@[^\\s@]+\\.[^\\s@]+" + }, + { + "name": "f2", + "label": "Phone Number", + "type": "token", + "pattern": "[+()\\d][\\d ()+.-]+" } - }, - "artifact": "
新文章1
新文章2
新文章3
新文章4
新文章5
新文章6
新文章7
新文章8
新文章9
新文章10
新文章11
新文章12
新文章13
新文章14
新文章15
新文章16
", - "ok": true, - "expected_task_pass": false - }, - { - "id": "dynamic22-061-wrong-scope", - "category": "structured-extraction", - "root_cause": "dynamic_initial_scope_and_owned_records", - "failure_mode": "false-positive", - "case": { - "url": "https://juejin.cn/", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 16, - "hosts": [ - "juejin.cn" - ], - "entry_path": "/post/[0-9]+", - "name_validation": "nonempty", - "entry_scope": { - "html_selector": ".entry-list .entry a.title" - } + ], + "record_identity_fields": [ + "f0" + ], + "table_scope": { + "root_selector": "main table", + "root_index": 0, + "row_selector": "tr", + "cell_selector": "td", + "start_pattern": "(?=^\\s*\\|?\\s*Name(?:\\s|\\|))", + "end_pattern": "^\\s*#{1,6}\\s", + "plain_row_pattern": "(?P[^\t\n]+)\\t(?P[^\t\n]+)\\t(?P[^\t\n]+)", + "markdown_table_limit": 1 + } + } + }, + "artifact": "| Name | Email | Phone Number |\n| --- | --- | --- |\n| Alpha | person@example.org | (123) 456 789 |\n\n| Name | Email | Phone Number |\n| --- | --- | --- |\n| Alpha | person@example.org | (123) 456 789 |\n| Beta | person@example.org | (123) 456 789 |\n| Gamma | person@example.org | (123) 456 789 |", + "ok": true, + "expected_task_pass": false + }, + { + "id": "completion100-029-17", + "category": "structured-extraction", + "root_cause": "table_fields_and_example_boundaries", + "failure_mode": "false-negative", + "case": { + "url": "https://example.org/table", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 3, + "stable_text_anchors": [], + "duplicate_policy": "deduplicate_identical", + "table_columns": [ + { + "name": "f0", + "label": "Name", + "type": "text" + }, + { + "name": "f1", + "label": "Size", + "type": "token", + "pattern": "\\d+(?:\\.\\d+)?\\s*[kKmMgG]?[bB]" + }, + { + "name": "f2", + "label": "Type", + "type": "text" } - }, - "artifact": "
新文章1
新文章2
新文章3
新文章4
新文章5
新文章6
新文章7
新文章8
新文章9
新文章10
新文章11
新文章12
新文章13
新文章14
新文章15
新文章16
", - "ok": true, - "expected_task_pass": false - }, - { - "id": "dynamic22-037-prefetch-only", - "category": "structured-extraction", - "root_cause": "dynamic_initial_scope_and_owned_records", - "failure_mode": "false-positive", - "case": { - "url": "https://quasar.dev/vue-components/virtual-scroll/", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "table_row", - "minimum_records": 8, - "table_columns": [ - { - "name": "item", - "label": "Option", - "type": "text", - "pattern": "#?\\d+\\s*-\\s*Option\\s+\\d+" - } - ], - "record_identity_fields": [ - "item" - ], - "table_scope": { - "root_selector": ".q-virtual-scroll", - "root_index": 0, - "row_selector": ".q-virtual-scroll__content > .q-item", - "cell_selector": ":scope", - "start_pattern": "(?m)^\\s*#{0,6}\\s*Basic\\s*$", - "end_pattern": "(?m)^[ \\t#]*Horizontal[ \\t]*$", - "plain_row_pattern": "(?P#?\\d+\\s*-\\s*Option\\s+\\d+)", - "reference_kind": "fixed_demo", - "reference_rows": [ - { - "item": "#0 - Option 1" - }, - { - "item": "#1 - Option 2" - }, - { - "item": "#2 - Option 3" - }, - { - "item": "#3 - Option 4" - }, - { - "item": "#4 - Option 5" - }, - { - "item": "#5 - Option 6" - }, - { - "item": "#6 - Option 7" - }, - { - "item": "#7 - Option 8" - }, - { - "item": "#8 - Option 9" - }, - { - "item": "#9 - Option 10" - } - ] - } + ], + "record_identity_fields": [ + "f0" + ], + "table_scope": { + "root_selector": "table", + "root_index": 0, + "row_selector": "tr", + "cell_selector": "td", + "start_pattern": "(?=^\\s*\\|?\\s*Name(?:\\s|\\|))", + "end_pattern": "^\\s*#{1,6}\\s", + "plain_row_pattern": "(?P[^\t\n]+)\\t(?P[^\t\n]+)\\t(?P[^\t\n]+)", + "markdown_table_limit": 1 + } + } + }, + "artifact": "| Name | Size | Type |\n| --- | --- | --- |\n| Alpha | 25kb | Alpha |\n| Beta | 25kb | Beta |\n| Gamma | 25kb | Gamma |", + "ok": true, + "expected_task_pass": true + }, + { + "id": "completion100-029-18", + "category": "structured-extraction", + "root_cause": "table_fields_and_example_boundaries", + "failure_mode": "false-positive", + "case": { + "url": "https://example.org/table", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 3, + "stable_text_anchors": [], + "duplicate_policy": "deduplicate_identical", + "table_columns": [ + { + "name": "f0", + "label": "Name", + "type": "text" + }, + { + "name": "f1", + "label": "Size", + "type": "token", + "pattern": "\\d+(?:\\.\\d+)?\\s*[kKmMgG]?[bB]" + }, + { + "name": "f2", + "label": "Type", + "type": "text" } - }, - "artifact": "
#20 - Option 21
#21 - Option 22
#22 - Option 23
#23 - Option 24
#24 - Option 25
#25 - Option 26
#26 - Option 27
#27 - Option 28
", - "ok": true, - "expected_task_pass": false - }, - { - "id": "dynamic22-030-prefetch-only", - "category": "structured-extraction", - "root_cause": "dynamic_initial_scope_and_owned_records", - "failure_mode": "false-positive", - "case": { - "url": "https://primevue.dev/virtualscroller/", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "table_row", - "minimum_records": 4, - "table_columns": [ - { - "name": "item", - "label": "Item", - "type": "text", - "pattern": "(?i)Item\\s*#\\s*\\d+" - } - ], - "record_identity_fields": [ - "item" - ], - "table_scope": { - "root_selector": ".doc-section-demo", - "root_index": 0, - "row_selector": ".p-virtualscroller-content > div", - "cell_selector": ":scope", - "start_pattern": "(?im)^(?=[ \\t|]*(?:[-*][ \\t]+)?Item[ \\t]*#[ \\t]*\\d+|[ \\t]*\\|[ \\t]*Item[ \\t]*\\|)", - "end_pattern": "(?im)^[ \\t#]*(?:PREVIEW|Usage|Examples|Horizontal)[ \\t]*$", - "plain_row_pattern": "[ \\t|]*(?:[-*][ \\t]+)?(?P(?i:Item)[ \\t]*#[ \\t]*\\d+)[ \\t|]*", - "reference_kind": "fixed_demo", - "reference_rows": [ - { - "item": "Item #0" - }, - { - "item": "Item #1" - }, - { - "item": "Item #2" - }, - { - "item": "Item #3" - } - ] - } + ], + "record_identity_fields": [ + "f0" + ], + "table_scope": { + "root_selector": "table", + "root_index": 0, + "row_selector": "tr", + "cell_selector": "td", + "start_pattern": "(?=^\\s*\\|?\\s*Name(?:\\s|\\|))", + "end_pattern": "^\\s*#{1,6}\\s", + "plain_row_pattern": "(?P[^\t\n]+)\\t(?P[^\t\n]+)\\t(?P[^\t\n]+)", + "markdown_table_limit": 1 + } + } + }, + "artifact": "| Name | Size | Type |\n| --- | --- | --- |\n| Alpha | 25kb | |\n| Beta | 25kb | |\n| Gamma | 25kb | |", + "ok": true, + "expected_task_pass": false + }, + { + "id": "completion100-029-19", + "category": "structured-extraction", + "root_cause": "table_fields_and_example_boundaries", + "failure_mode": "false-positive", + "case": { + "url": "https://example.org/table", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 3, + "stable_text_anchors": [], + "duplicate_policy": "deduplicate_identical", + "table_columns": [ + { + "name": "f0", + "label": "Name", + "type": "text" + }, + { + "name": "f1", + "label": "Size", + "type": "token", + "pattern": "\\d+(?:\\.\\d+)?\\s*[kKmMgG]?[bB]" + }, + { + "name": "f2", + "label": "Type", + "type": "text" } - }, - "artifact": "
Item #20
Item #21
Item #22
Item #23
", - "ok": true, - "expected_task_pass": false - }, - { - "id": "dynamic22-032-prefetch-only", - "category": "structured-extraction", - "root_cause": "dynamic_initial_scope_and_owned_records", - "failure_mode": "false-positive", - "case": { - "url": "https://primeng.dev/virtualscroller", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "table_row", - "minimum_records": 4, - "table_columns": [ - { - "name": "item", - "label": "Item", - "type": "text", - "pattern": "(?i)Item\\s*#\\s*\\d+" - } - ], - "record_identity_fields": [ - "item" - ], - "table_scope": { - "root_selector": ".doc-section-demo", - "root_index": 0, - "row_selector": ".p-virtualscroller-content > div", - "cell_selector": ":scope", - "start_pattern": "(?im)^(?=[ \\t|]*(?:[-*][ \\t]+)?Item[ \\t]*#[ \\t]*\\d+|[ \\t]*\\|[ \\t]*Item[ \\t]*\\|)", - "end_pattern": "(?im)^[ \\t#]*(?:PREVIEW|Usage|Examples|Horizontal)[ \\t]*$", - "plain_row_pattern": "[ \\t|]*(?:[-*][ \\t]+)?(?P(?i:Item)[ \\t]*#[ \\t]*\\d+)[ \\t|]*", - "reference_kind": "fixed_demo", - "reference_rows": [ - { - "item": "Item #0" - }, - { - "item": "Item #1" - }, - { - "item": "Item #2" - }, - { - "item": "Item #3" - } - ] - } + ], + "record_identity_fields": [ + "f0" + ], + "table_scope": { + "root_selector": "table", + "root_index": 0, + "row_selector": "tr", + "cell_selector": "td", + "start_pattern": "(?=^\\s*\\|?\\s*Name(?:\\s|\\|))", + "end_pattern": "^\\s*#{1,6}\\s", + "plain_row_pattern": "(?P[^\t\n]+)\\t(?P[^\t\n]+)\\t(?P[^\t\n]+)", + "markdown_table_limit": 1 + } + } + }, + "artifact": "| Name | Size | Type |\n| --- | --- | --- |\n| Alpha | 25kb | Alpha |\n| Alpha | 25kb | Alpha |\n| Alpha | 25kb | Alpha |", + "ok": true, + "expected_task_pass": false + }, + { + "id": "completion100-029-20", + "category": "structured-extraction", + "root_cause": "table_fields_and_example_boundaries", + "failure_mode": "false-positive", + "case": { + "url": "https://example.org/table", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 3, + "stable_text_anchors": [], + "duplicate_policy": "deduplicate_identical", + "table_columns": [ + { + "name": "f0", + "label": "Name", + "type": "text" + }, + { + "name": "f1", + "label": "Size", + "type": "token", + "pattern": "\\d+(?:\\.\\d+)?\\s*[kKmMgG]?[bB]" + }, + { + "name": "f2", + "label": "Type", + "type": "text" } - }, - "artifact": "
Item #20
Item #21
Item #22
Item #23
", - "ok": true, - "expected_task_pass": false - }, - { - "id": "table-copy-marker-complete", - "category": "format-normalization", - "root_cause": "table_auxiliary_copy_markers", - "failure_mode": "false-negative", - "case": { - "url": "https://example.test/table", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "table_row", - "minimum_records": 1, - "table_columns": [ - { - "name": "name", - "label": "Name", - "type": "text" - }, - { - "name": "date", - "label": "Date", - "type": "date_ordinal_en" - }, - { - "name": "state", - "label": "State", - "type": "text", - "pattern": "Online|Offline" - } - ], - "record_identity_fields": [ - "name" - ], - "table_scope": { - "row_selector": "tbody tr", - "cell_selector": "td", - "start_pattern": "(?m)^(?=\\| Name \\|)", - "end_pattern": "END", - "plain_row_pattern": "(?P[A-Za-z]+) (?P\\d{1,2}(?:st|nd|rd|th) [A-Za-z]{3} \\d{2}) (?POnline|Offline)", - "plain_text_ignored_patterns": [ - "[↦↵](?=[ \\t]*(?:\\||$))", - "\\(external, opens in a new tab or window\\)" - ] - } + ], + "record_identity_fields": [ + "f0" + ], + "table_scope": { + "root_selector": "table", + "root_index": 0, + "row_selector": "tr", + "cell_selector": "td", + "start_pattern": "(?=^\\s*\\|?\\s*Name(?:\\s|\\|))", + "end_pattern": "^\\s*#{1,6}\\s", + "plain_row_pattern": "(?P[^\t\n]+)\\t(?P[^\t\n]+)\\t(?P[^\t\n]+)", + "markdown_table_limit": 1 + } + } + }, + "artifact": "| Name | Size | Type |\n| --- | --- | --- |\n| Alpha | 25kb | Alpha |\n\n| Name | Size | Type |\n| --- | --- | --- |\n| Alpha | 25kb | Alpha |\n| Beta | 25kb | Beta |\n| Gamma | 25kb | Gamma |", + "ok": true, + "expected_task_pass": false + }, + { + "id": "completion100-038-21", + "category": "structured-extraction", + "root_cause": "table_fields_and_example_boundaries", + "failure_mode": "false-negative", + "case": { + "url": "https://example.org/table", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 8, + "stable_text_anchors": [], + "duplicate_policy": "deduplicate_identical", + "table_columns": [ + { + "name": "f0", + "label": "No.", + "type": "integer" + }, + { + "name": "f1", + "label": "Name", + "type": "text" + }, + { + "name": "f2", + "label": "Weight", + "type": "number" + }, + { + "name": "f3", + "label": "Symbol", + "type": "token", + "pattern": "[A-Z][a-z]?" } - }, - "artifact": "| Name↦ | Date↦ | State↵ |\n| --- | --- | --- |\n| Ada(external, opens in a new tab or window)↦ | 29th Sep 25↦ | Online↵ |\nEND", - "ok": true, - "expected_task_pass": true - }, - { - "id": "table-copy-marker-bad-date", - "category": "format-normalization", - "root_cause": "table_auxiliary_copy_markers", - "failure_mode": "false-positive", - "case": { - "url": "https://example.test/table", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "table_row", - "minimum_records": 1, - "table_columns": [ - { - "name": "name", - "label": "Name", - "type": "text" - }, - { - "name": "date", - "label": "Date", - "type": "date_ordinal_en" - }, - { - "name": "state", - "label": "State", - "type": "text", - "pattern": "Online|Offline" - } - ], - "record_identity_fields": [ - "name" - ], - "table_scope": { - "row_selector": "tbody tr", - "cell_selector": "td", - "start_pattern": "(?m)^(?=\\| Name \\|)", - "end_pattern": "END", - "plain_row_pattern": "(?P[A-Za-z]+) (?P\\d{1,2}(?:st|nd|rd|th) [A-Za-z]{3} \\d{2}) (?POnline|Offline)", - "plain_text_ignored_patterns": [ - "[↦↵](?=[ \\t]*(?:\\||$))", - "\\(external, opens in a new tab or window\\)" - ] - } + ], + "record_identity_fields": [ + "f0" + ], + "table_scope": { + "root_selector": "main table", + "root_index": 0, + "row_selector": "tr", + "cell_selector": "td", + "start_pattern": "(?=^\\s*\\|?\\s*No\\.(?:\\s|\\|))", + "end_pattern": "^\\s*#{1,6}\\s", + "plain_row_pattern": "(?P[^\t\n]+)\\t(?P[^\t\n]+)\\t(?P[^\t\n]+)\\t(?P[^\t\n]+)", + "markdown_table_limit": 1 + } + } + }, + "artifact": "| No. | Name | Weight | Symbol |\n| --- | --- | --- | --- |\n| 1 | Alpha | 1.5 | He |\n| 2 | Beta | 1.5 | He |\n| 3 | Gamma | 1.5 | He |\n| 4 | Delta | 1.5 | He |\n| 5 | Epsilon | 1.5 | He |\n| 6 | Zeta | 1.5 | He |\n| 7 | Eta | 1.5 | He |\n| 8 | Theta | 1.5 | He |", + "ok": true, + "expected_task_pass": true + }, + { + "id": "completion100-038-22", + "category": "structured-extraction", + "root_cause": "table_fields_and_example_boundaries", + "failure_mode": "false-positive", + "case": { + "url": "https://example.org/table", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 8, + "stable_text_anchors": [], + "duplicate_policy": "deduplicate_identical", + "table_columns": [ + { + "name": "f0", + "label": "No.", + "type": "integer" + }, + { + "name": "f1", + "label": "Name", + "type": "text" + }, + { + "name": "f2", + "label": "Weight", + "type": "number" + }, + { + "name": "f3", + "label": "Symbol", + "type": "token", + "pattern": "[A-Z][a-z]?" } - }, - "artifact": "| Name↦ | Date↦ | State↵ |\n| --- | --- | --- |\n| Ada↦ | 32nd Sep 25↦ | Online↵ |\nEND", - "ok": true, - "expected_task_pass": false - }, - { - "id": "table-copy-marker-missing-field", - "category": "format-normalization", - "root_cause": "table_auxiliary_copy_markers", - "failure_mode": "false-positive", - "case": { - "url": "https://example.test/table", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "table_row", - "minimum_records": 1, - "table_columns": [ - { - "name": "name", - "label": "Name", - "type": "text" - }, - { - "name": "date", - "label": "Date", - "type": "date_ordinal_en" - }, - { - "name": "state", - "label": "State", - "type": "text", - "pattern": "Online|Offline" - } - ], - "record_identity_fields": [ - "name" - ], - "table_scope": { - "row_selector": "tbody tr", - "cell_selector": "td", - "start_pattern": "(?m)^(?=\\| Name \\|)", - "end_pattern": "END", - "plain_row_pattern": "(?P[A-Za-z]+) (?P\\d{1,2}(?:st|nd|rd|th) [A-Za-z]{3} \\d{2}) (?POnline|Offline)", - "plain_text_ignored_patterns": [ - "[↦↵](?=[ \\t]*(?:\\||$))", - "\\(external, opens in a new tab or window\\)" - ] - } + ], + "record_identity_fields": [ + "f0" + ], + "table_scope": { + "root_selector": "main table", + "root_index": 0, + "row_selector": "tr", + "cell_selector": "td", + "start_pattern": "(?=^\\s*\\|?\\s*No\\.(?:\\s|\\|))", + "end_pattern": "^\\s*#{1,6}\\s", + "plain_row_pattern": "(?P[^\t\n]+)\\t(?P[^\t\n]+)\\t(?P[^\t\n]+)\\t(?P[^\t\n]+)", + "markdown_table_limit": 1 + } + } + }, + "artifact": "| No. | Name | Weight | Symbol |\n| --- | --- | --- | --- |\n| 1 | Alpha | 1.5 | |\n| 2 | Beta | 1.5 | |\n| 3 | Gamma | 1.5 | |\n| 4 | Delta | 1.5 | |\n| 5 | Epsilon | 1.5 | |\n| 6 | Zeta | 1.5 | |\n| 7 | Eta | 1.5 | |\n| 8 | Theta | 1.5 | |", + "ok": true, + "expected_task_pass": false + }, + { + "id": "completion100-038-23", + "category": "structured-extraction", + "root_cause": "table_fields_and_example_boundaries", + "failure_mode": "false-positive", + "case": { + "url": "https://example.org/table", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 8, + "stable_text_anchors": [], + "duplicate_policy": "deduplicate_identical", + "table_columns": [ + { + "name": "f0", + "label": "No.", + "type": "integer" + }, + { + "name": "f1", + "label": "Name", + "type": "text" + }, + { + "name": "f2", + "label": "Weight", + "type": "number" + }, + { + "name": "f3", + "label": "Symbol", + "type": "token", + "pattern": "[A-Z][a-z]?" } - }, - "artifact": "| Name↦ | Date↦ | State↵ |\n| --- | --- | --- |\n| Ada↦ | ↦ | Online↵ |\nEND", - "ok": true, - "expected_task_pass": false - }, - { - "id": "table-copy-marker-literal-arrow", - "category": "format-normalization", - "root_cause": "table_auxiliary_copy_markers", - "failure_mode": "false-positive", - "case": { - "url": "https://example.test/table", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "table_row", - "minimum_records": 1, - "table_columns": [ - { - "name": "name", - "label": "Name", - "type": "text" - }, - { - "name": "date", - "label": "Date", - "type": "date_ordinal_en" - }, - { - "name": "state", - "label": "State", - "type": "text", - "pattern": "Online|Offline" - } - ], - "record_identity_fields": [ - "name" - ], - "table_scope": { - "row_selector": "tbody tr", - "cell_selector": "td", - "start_pattern": "(?m)^(?=\\| Name \\|)", - "end_pattern": "END", - "plain_row_pattern": "(?P[A-Za-z]+) (?P\\d{1,2}(?:st|nd|rd|th) [A-Za-z]{3} \\d{2}) (?POnline|Offline)", - "plain_text_ignored_patterns": [ - "[↦↵](?=[ \\t]*(?:\\||$))", - "\\(external, opens in a new tab or window\\)" - ] - } + ], + "record_identity_fields": [ + "f0" + ], + "table_scope": { + "root_selector": "main table", + "root_index": 0, + "row_selector": "tr", + "cell_selector": "td", + "start_pattern": "(?=^\\s*\\|?\\s*No\\.(?:\\s|\\|))", + "end_pattern": "^\\s*#{1,6}\\s", + "plain_row_pattern": "(?P[^\t\n]+)\\t(?P[^\t\n]+)\\t(?P[^\t\n]+)\\t(?P[^\t\n]+)", + "markdown_table_limit": 1 + } + } + }, + "artifact": "| No. | Name | Weight | Symbol |\n| --- | --- | --- | --- |\n| 1 | Alpha | 1.5 | He |\n| 1 | Alpha | 1.5 | He |\n| 1 | Alpha | 1.5 | He |\n| 1 | Alpha | 1.5 | He |\n| 1 | Alpha | 1.5 | He |\n| 1 | Alpha | 1.5 | He |\n| 1 | Alpha | 1.5 | He |\n| 1 | Alpha | 1.5 | He |", + "ok": true, + "expected_task_pass": false + }, + { + "id": "completion100-038-24", + "category": "structured-extraction", + "root_cause": "table_fields_and_example_boundaries", + "failure_mode": "false-positive", + "case": { + "url": "https://example.org/table", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 8, + "stable_text_anchors": [], + "duplicate_policy": "deduplicate_identical", + "table_columns": [ + { + "name": "f0", + "label": "No.", + "type": "integer" + }, + { + "name": "f1", + "label": "Name", + "type": "text" + }, + { + "name": "f2", + "label": "Weight", + "type": "number" + }, + { + "name": "f3", + "label": "Symbol", + "type": "token", + "pattern": "[A-Z][a-z]?" } - }, - "artifact": "| Name↦ | Date↦ | State↵ |\n| --- | --- | --- |\n| Ada↦ | 29th Sep 25↦ | On↦line↵ |\nEND", - "ok": true, - "expected_task_pass": false - }, - { - "id": "table-compact-cell-joined", - "category": "format-normalization", - "root_cause": "adjacent_cell_elements_and_empty_column_preservation", - "failure_mode": "false-negative", - "case": { - "url": "https://example.test/table", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "table_row", - "minimum_records": 1, - "table_columns": [ - { - "name": "product", - "label": "Product", - "type": "text", - "compact_whitespace": true - }, - { - "name": "category", - "label": "Category", - "type": "text" - }, - { - "name": "price", - "label": "Price", - "type": "currency_usd" - }, - { - "name": "status", - "label": "Status", - "type": "text" - } - ], - "record_identity_fields": [ - "product" - ], - "table_scope": { - "root_selector": ".doc-section-demo", - "root_index": 0, - "row_selector": "tbody tr", - "cell_selector": ":scope > td:not(:nth-child(4))", - "start_pattern": "(?im)^(?=[ \\t|]*Product[ \\t|]+Category)", - "end_pattern": "(?im)^[ \\t#]*(?:PREVIEW|Usage)[ \\t]*$", - "plain_row_pattern": "(?P[^\\n]+?\\s+[a-z0-9]{9})\\s+(?PAccessories|Fitness|Clothing)\\s+(?P\\$\\s*\\d+(?:\\.\\d{1,2})?)\\s+(?PIn Stock|Low Stock|Out of Stock)", - "plain_multiline_row_pattern": "(?m)^(?P[^\\n]+\\n[a-z0-9]{9})\\s*\\n[\\s]*(?PAccessories|Fitness|Clothing)[\\s]+(?P\\$\\s*\\d+(?:\\.\\d{1,2})?)[\\s]+(?PIn Stock|Low Stock|Out of Stock)(?=\\n|$)", - "reference_kind": "fixed_demo", - "reference_rows": [ - { - "product": "Bamboo Watch f230fh0g3", - "category": "Accessories", - "price": "$65", - "status": "In Stock" - }, - { - "product": "Black Watch nvklal433", - "category": "Accessories", - "price": "$72", - "status": "In Stock" - }, - { - "product": "Blue Band zz21cz3c1", - "category": "Fitness", - "price": "$79", - "status": "Low Stock" - }, - { - "product": "Blue T-Shirt 244wgerg2", - "category": "Clothing", - "price": "$29", - "status": "In Stock" - }, - { - "product": "Bracelet h456wer53", - "category": "Accessories", - "price": "$15", - "status": "In Stock" - }, - { - "product": "Brown Purse av2231fwg", - "category": "Accessories", - "price": "$120", - "status": "Out of Stock" - }, - { - "product": "Chakra Bracelet bib36pfvm", - "category": "Accessories", - "price": "$32", - "status": "Low Stock" - }, - { - "product": "Galaxy Earrings mbvjkgip5", - "category": "Accessories", - "price": "$34", - "status": "In Stock" - } - ], - "ignored_column_labels": [ - "Rating" - ] - } + ], + "record_identity_fields": [ + "f0" + ], + "table_scope": { + "root_selector": "main table", + "root_index": 0, + "row_selector": "tr", + "cell_selector": "td", + "start_pattern": "(?=^\\s*\\|?\\s*No\\.(?:\\s|\\|))", + "end_pattern": "^\\s*#{1,6}\\s", + "plain_row_pattern": "(?P[^\t\n]+)\\t(?P[^\t\n]+)\\t(?P[^\t\n]+)\\t(?P[^\t\n]+)", + "markdown_table_limit": 1 + } + } + }, + "artifact": "| No. | Name | Weight | Symbol |\n| --- | --- | --- | --- |\n| 1 | Alpha | 1.5 | He |\n\n| No. | Name | Weight | Symbol |\n| --- | --- | --- | --- |\n| 1 | Alpha | 1.5 | He |\n| 2 | Beta | 1.5 | He |\n| 3 | Gamma | 1.5 | He |\n| 4 | Delta | 1.5 | He |\n| 5 | Epsilon | 1.5 | He |\n| 6 | Zeta | 1.5 | He |\n| 7 | Eta | 1.5 | He |\n| 8 | Theta | 1.5 | He |", + "ok": true, + "expected_task_pass": false + }, + { + "id": "completion100-041-25", + "category": "structured-extraction", + "root_cause": "table_fields_and_example_boundaries", + "failure_mode": "false-negative", + "case": { + "url": "https://example.org/table", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 4, + "stable_text_anchors": [], + "duplicate_policy": "deduplicate_identical", + "table_columns": [ + { + "name": "f0", + "label": "Name", + "type": "text" + }, + { + "name": "f1", + "label": "Role", + "type": "text" + }, + { + "name": "f2", + "label": "Status", + "type": "text" + }, + { + "name": "f3", + "label": "Email", + "type": "token", + "pattern": "[^\\s@]+@[^\\s@]+\\.[^\\s@]+" } - }, - "artifact": "Product Category\n| Product | Category | Price | Rating | Status |\n| --- | --- | --- | --- | --- |\n| Bamboo Watchf230fh0g3 | Accessories | $65 | | In Stock |\nPREVIEW", - "ok": true, - "expected_task_pass": true - }, - { - "id": "table-compact-cell-wrong-id", - "category": "format-normalization", - "root_cause": "adjacent_cell_elements_and_empty_column_preservation", - "failure_mode": "false-positive", - "case": { - "url": "https://example.test/table", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "table_row", - "minimum_records": 1, - "table_columns": [ - { - "name": "product", - "label": "Product", - "type": "text", - "compact_whitespace": true - }, - { - "name": "category", - "label": "Category", - "type": "text" - }, - { - "name": "price", - "label": "Price", - "type": "currency_usd" - }, - { - "name": "status", - "label": "Status", - "type": "text" - } - ], - "record_identity_fields": [ - "product" - ], - "table_scope": { - "root_selector": ".doc-section-demo", - "root_index": 0, - "row_selector": "tbody tr", - "cell_selector": ":scope > td:not(:nth-child(4))", - "start_pattern": "(?im)^(?=[ \\t|]*Product[ \\t|]+Category)", - "end_pattern": "(?im)^[ \\t#]*(?:PREVIEW|Usage)[ \\t]*$", - "plain_row_pattern": "(?P[^\\n]+?\\s+[a-z0-9]{9})\\s+(?PAccessories|Fitness|Clothing)\\s+(?P\\$\\s*\\d+(?:\\.\\d{1,2})?)\\s+(?PIn Stock|Low Stock|Out of Stock)", - "plain_multiline_row_pattern": "(?m)^(?P[^\\n]+\\n[a-z0-9]{9})\\s*\\n[\\s]*(?PAccessories|Fitness|Clothing)[\\s]+(?P\\$\\s*\\d+(?:\\.\\d{1,2})?)[\\s]+(?PIn Stock|Low Stock|Out of Stock)(?=\\n|$)", - "reference_kind": "fixed_demo", - "reference_rows": [ - { - "product": "Bamboo Watch f230fh0g3", - "category": "Accessories", - "price": "$65", - "status": "In Stock" - }, - { - "product": "Black Watch nvklal433", - "category": "Accessories", - "price": "$72", - "status": "In Stock" - }, - { - "product": "Blue Band zz21cz3c1", - "category": "Fitness", - "price": "$79", - "status": "Low Stock" - }, - { - "product": "Blue T-Shirt 244wgerg2", - "category": "Clothing", - "price": "$29", - "status": "In Stock" - }, - { - "product": "Bracelet h456wer53", - "category": "Accessories", - "price": "$15", - "status": "In Stock" - }, - { - "product": "Brown Purse av2231fwg", - "category": "Accessories", - "price": "$120", - "status": "Out of Stock" - }, - { - "product": "Chakra Bracelet bib36pfvm", - "category": "Accessories", - "price": "$32", - "status": "Low Stock" - }, - { - "product": "Galaxy Earrings mbvjkgip5", - "category": "Accessories", - "price": "$34", - "status": "In Stock" - } - ], - "ignored_column_labels": [ - "Rating" - ] - } + ], + "record_identity_fields": [ + "f0" + ], + "table_scope": { + "root_selector": "table", + "root_index": 0, + "row_selector": "tr", + "cell_selector": "td", + "start_pattern": "(?=^\\s*\\|?\\s*Name(?:\\s|\\|))", + "end_pattern": "^\\s*#{1,6}\\s", + "plain_row_pattern": "(?P[^\t\n]+)\\t(?P[^\t\n]+)\\t(?P[^\t\n]+)\\t(?P[^\t\n]+)", + "markdown_table_limit": 1 + } + } + }, + "artifact": "| Name | Role | Status | Email |\n| --- | --- | --- | --- |\n| Alpha | Alpha | Alpha | person@example.org |\n| Beta | Beta | Beta | person@example.org |\n| Gamma | Gamma | Gamma | person@example.org |\n| Delta | Delta | Delta | person@example.org |", + "ok": true, + "expected_task_pass": true + }, + { + "id": "completion100-041-26", + "category": "structured-extraction", + "root_cause": "table_fields_and_example_boundaries", + "failure_mode": "false-positive", + "case": { + "url": "https://example.org/table", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 4, + "stable_text_anchors": [], + "duplicate_policy": "deduplicate_identical", + "table_columns": [ + { + "name": "f0", + "label": "Name", + "type": "text" + }, + { + "name": "f1", + "label": "Role", + "type": "text" + }, + { + "name": "f2", + "label": "Status", + "type": "text" + }, + { + "name": "f3", + "label": "Email", + "type": "token", + "pattern": "[^\\s@]+@[^\\s@]+\\.[^\\s@]+" } - }, - "artifact": "Product Category\n| Product | Category | Price | Rating | Status |\n| --- | --- | --- | --- | --- |\n| Bamboo Watchwrongid | Accessories | $65 | | In Stock |\nPREVIEW", - "ok": true, - "expected_task_pass": false - }, - { - "id": "table-compact-cell-wrong-price", - "category": "format-normalization", - "root_cause": "adjacent_cell_elements_and_empty_column_preservation", - "failure_mode": "false-positive", - "case": { - "url": "https://example.test/table", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "table_row", - "minimum_records": 1, - "table_columns": [ - { - "name": "product", - "label": "Product", - "type": "text", - "compact_whitespace": true - }, - { - "name": "category", - "label": "Category", - "type": "text" - }, - { - "name": "price", - "label": "Price", - "type": "currency_usd" - }, - { - "name": "status", - "label": "Status", - "type": "text" - } - ], - "record_identity_fields": [ - "product" - ], - "table_scope": { - "root_selector": ".doc-section-demo", - "root_index": 0, - "row_selector": "tbody tr", - "cell_selector": ":scope > td:not(:nth-child(4))", - "start_pattern": "(?im)^(?=[ \\t|]*Product[ \\t|]+Category)", - "end_pattern": "(?im)^[ \\t#]*(?:PREVIEW|Usage)[ \\t]*$", - "plain_row_pattern": "(?P[^\\n]+?\\s+[a-z0-9]{9})\\s+(?PAccessories|Fitness|Clothing)\\s+(?P\\$\\s*\\d+(?:\\.\\d{1,2})?)\\s+(?PIn Stock|Low Stock|Out of Stock)", - "plain_multiline_row_pattern": "(?m)^(?P[^\\n]+\\n[a-z0-9]{9})\\s*\\n[\\s]*(?PAccessories|Fitness|Clothing)[\\s]+(?P\\$\\s*\\d+(?:\\.\\d{1,2})?)[\\s]+(?PIn Stock|Low Stock|Out of Stock)(?=\\n|$)", - "reference_kind": "fixed_demo", - "reference_rows": [ - { - "product": "Bamboo Watch f230fh0g3", - "category": "Accessories", - "price": "$65", - "status": "In Stock" - }, - { - "product": "Black Watch nvklal433", - "category": "Accessories", - "price": "$72", - "status": "In Stock" - }, - { - "product": "Blue Band zz21cz3c1", - "category": "Fitness", - "price": "$79", - "status": "Low Stock" - }, - { - "product": "Blue T-Shirt 244wgerg2", - "category": "Clothing", - "price": "$29", - "status": "In Stock" - }, - { - "product": "Bracelet h456wer53", - "category": "Accessories", - "price": "$15", - "status": "In Stock" - }, - { - "product": "Brown Purse av2231fwg", - "category": "Accessories", - "price": "$120", - "status": "Out of Stock" - }, - { - "product": "Chakra Bracelet bib36pfvm", - "category": "Accessories", - "price": "$32", - "status": "Low Stock" - }, - { - "product": "Galaxy Earrings mbvjkgip5", - "category": "Accessories", - "price": "$34", - "status": "In Stock" - } - ], - "ignored_column_labels": [ - "Rating" - ] - } + ], + "record_identity_fields": [ + "f0" + ], + "table_scope": { + "root_selector": "table", + "root_index": 0, + "row_selector": "tr", + "cell_selector": "td", + "start_pattern": "(?=^\\s*\\|?\\s*Name(?:\\s|\\|))", + "end_pattern": "^\\s*#{1,6}\\s", + "plain_row_pattern": "(?P[^\t\n]+)\\t(?P[^\t\n]+)\\t(?P[^\t\n]+)\\t(?P[^\t\n]+)", + "markdown_table_limit": 1 + } + } + }, + "artifact": "| Name | Role | Status | Email |\n| --- | --- | --- | --- |\n| Alpha | Alpha | Alpha | |\n| Beta | Beta | Beta | |\n| Gamma | Gamma | Gamma | |\n| Delta | Delta | Delta | |", + "ok": true, + "expected_task_pass": false + }, + { + "id": "completion100-041-27", + "category": "structured-extraction", + "root_cause": "table_fields_and_example_boundaries", + "failure_mode": "false-positive", + "case": { + "url": "https://example.org/table", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 4, + "stable_text_anchors": [], + "duplicate_policy": "deduplicate_identical", + "table_columns": [ + { + "name": "f0", + "label": "Name", + "type": "text" + }, + { + "name": "f1", + "label": "Role", + "type": "text" + }, + { + "name": "f2", + "label": "Status", + "type": "text" + }, + { + "name": "f3", + "label": "Email", + "type": "token", + "pattern": "[^\\s@]+@[^\\s@]+\\.[^\\s@]+" } - }, - "artifact": "Product Category\n| Product | Category | Price | Rating | Status |\n| --- | --- | --- | --- | --- |\n| Bamboo Watchf230fh0g3 | Accessories | $99 | | In Stock |\nPREVIEW", - "ok": true, - "expected_task_pass": false - }, - { - "id": "dynamic22-037-next-section-excluded", - "category": "structured-extraction", - "root_cause": "adjacent_virtual_demo_scope_boundary", - "failure_mode": "false-negative", - "case": { - "url": "https://quasar.dev/vue-components/virtual-scroll/", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "table_row", - "minimum_records": 8, - "table_columns": [ - { - "name": "item", - "label": "Option", - "type": "text", - "pattern": "#?\\d+\\s*-\\s*Option\\s+\\d+" - } - ], - "record_identity_fields": [ - "item" - ], - "table_scope": { - "root_selector": ".q-virtual-scroll", - "root_index": 0, - "row_selector": ".q-virtual-scroll__content > .q-item", - "cell_selector": ":scope", - "start_pattern": "(?m)^\\s*#{0,6}\\s*Basic\\s*$", - "end_pattern": "(?m)^[ \\t#]*Horizontal[ \\t]*$", - "plain_row_pattern": "(?P#?\\d+\\s*-\\s*Option\\s+\\d+)", - "reference_kind": "fixed_demo", - "reference_rows": [ - { - "item": "#0 - Option 1" - }, - { - "item": "#1 - Option 2" - }, - { - "item": "#2 - Option 3" - }, - { - "item": "#3 - Option 4" - }, - { - "item": "#4 - Option 5" - }, - { - "item": "#5 - Option 6" - }, - { - "item": "#6 - Option 7" - }, - { - "item": "#7 - Option 8" - }, - { - "item": "#8 - Option 9" - }, - { - "item": "#9 - Option 10" - } - ] - } + ], + "record_identity_fields": [ + "f0" + ], + "table_scope": { + "root_selector": "table", + "root_index": 0, + "row_selector": "tr", + "cell_selector": "td", + "start_pattern": "(?=^\\s*\\|?\\s*Name(?:\\s|\\|))", + "end_pattern": "^\\s*#{1,6}\\s", + "plain_row_pattern": "(?P[^\t\n]+)\\t(?P[^\t\n]+)\\t(?P[^\t\n]+)\\t(?P[^\t\n]+)", + "markdown_table_limit": 1 + } + } + }, + "artifact": "| Name | Role | Status | Email |\n| --- | --- | --- | --- |\n| Alpha | Alpha | Alpha | person@example.org |\n| Alpha | Alpha | Alpha | person@example.org |\n| Alpha | Alpha | Alpha | person@example.org |\n| Alpha | Alpha | Alpha | person@example.org |", + "ok": true, + "expected_task_pass": false + }, + { + "id": "completion100-041-28", + "category": "structured-extraction", + "root_cause": "table_fields_and_example_boundaries", + "failure_mode": "false-positive", + "case": { + "url": "https://example.org/table", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 4, + "stable_text_anchors": [], + "duplicate_policy": "deduplicate_identical", + "table_columns": [ + { + "name": "f0", + "label": "Name", + "type": "text" + }, + { + "name": "f1", + "label": "Role", + "type": "text" + }, + { + "name": "f2", + "label": "Status", + "type": "text" + }, + { + "name": "f3", + "label": "Email", + "type": "token", + "pattern": "[^\\s@]+@[^\\s@]+\\.[^\\s@]+" } - }, - "artifact": "### Basic\n\n#0 - Option 1\n#1 - Option 2\n#2 - Option 3\n#3 - Option 4\n#4 - Option 5\n#5 - Option 6\n#6 - Option 7\n#7 - Option 8\n\n### Horizontal\n\n#0 - Option 1\n#1 - Option 2\n#2 - Option 3\n#3 - Option 4\n#4 - Option 5\n#5 - Option 6\n#6 - Option 7\n#7 - Option 8", - "ok": true, - "expected_task_pass": true - }, - { - "id": "dynamic22-037-next-section-cannot-fill", - "category": "structured-extraction", - "root_cause": "adjacent_virtual_demo_scope_boundary", - "failure_mode": "false-positive", - "case": { - "url": "https://quasar.dev/vue-components/virtual-scroll/", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "table_row", - "minimum_records": 8, - "table_columns": [ - { - "name": "item", - "label": "Option", - "type": "text", - "pattern": "#?\\d+\\s*-\\s*Option\\s+\\d+" - } - ], - "record_identity_fields": [ - "item" - ], - "table_scope": { - "root_selector": ".q-virtual-scroll", - "root_index": 0, - "row_selector": ".q-virtual-scroll__content > .q-item", - "cell_selector": ":scope", - "start_pattern": "(?m)^\\s*#{0,6}\\s*Basic\\s*$", - "end_pattern": "(?m)^[ \\t#]*Horizontal[ \\t]*$", - "plain_row_pattern": "(?P#?\\d+\\s*-\\s*Option\\s+\\d+)", - "reference_kind": "fixed_demo", - "reference_rows": [ - { - "item": "#0 - Option 1" - }, - { - "item": "#1 - Option 2" - }, - { - "item": "#2 - Option 3" - }, - { - "item": "#3 - Option 4" - }, - { - "item": "#4 - Option 5" - }, - { - "item": "#5 - Option 6" - }, - { - "item": "#6 - Option 7" - }, - { - "item": "#7 - Option 8" - }, - { - "item": "#8 - Option 9" - }, - { - "item": "#9 - Option 10" - } - ] - } + ], + "record_identity_fields": [ + "f0" + ], + "table_scope": { + "root_selector": "table", + "root_index": 0, + "row_selector": "tr", + "cell_selector": "td", + "start_pattern": "(?=^\\s*\\|?\\s*Name(?:\\s|\\|))", + "end_pattern": "^\\s*#{1,6}\\s", + "plain_row_pattern": "(?P[^\t\n]+)\\t(?P[^\t\n]+)\\t(?P[^\t\n]+)\\t(?P[^\t\n]+)", + "markdown_table_limit": 1 + } + } + }, + "artifact": "| Name | Role | Status | Email |\n| --- | --- | --- | --- |\n| Alpha | Alpha | Alpha | person@example.org |\n\n| Name | Role | Status | Email |\n| --- | --- | --- | --- |\n| Alpha | Alpha | Alpha | person@example.org |\n| Beta | Beta | Beta | person@example.org |\n| Gamma | Gamma | Gamma | person@example.org |\n| Delta | Delta | Delta | person@example.org |", + "ok": true, + "expected_task_pass": false + }, + { + "id": "completion100-058-29", + "category": "structured-extraction", + "root_cause": "table_fields_and_example_boundaries", + "failure_mode": "false-negative", + "case": { + "url": "https://example.org/table", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 3, + "stable_text_anchors": [], + "duplicate_policy": "deduplicate_identical", + "table_columns": [ + { + "name": "f0", + "label": "申请人", + "type": "text" + }, + { + "name": "f1", + "label": "审批状态", + "type": "text" + }, + { + "name": "f2", + "label": "签署方式", + "type": "text" + }, + { + "name": "f3", + "label": "电子邮件", + "type": "token", + "pattern": "[^\\s@]+@[^\\s@]+\\.[^\\s@]+" } - }, - "artifact": "### Basic\n\n#0 - Option 1\n#1 - Option 2\n\n### Horizontal\n\n#0 - Option 1\n#1 - Option 2\n#2 - Option 3\n#3 - Option 4\n#4 - Option 5\n#5 - Option 6\n#6 - Option 7\n#7 - Option 8", - "ok": true, - "expected_task_pass": false - }, - { - "id": "named-visible-text-visible-title", - "category": "structured-extraction", - "root_cause": "image_alt_is_not_visible_record_name", - "failure_mode": "false-negative", - "case": { - "url": "https://example.test/list", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 1, - "hosts": [ - "example.test" - ], - "entry_path": "/item/\\d+", - "name_validation": "nonempty" + ], + "record_identity_fields": [ + "f0" + ], + "table_scope": { + "root_selector": "table", + "root_index": 0, + "row_selector": "tr", + "cell_selector": "td", + "start_pattern": "(?=^\\s*\\|?\\s*申请人(?:\\s|\\|))", + "end_pattern": "^\\s*#{1,6}\\s", + "plain_row_pattern": "(?P[^\t\n]+)\\t(?P[^\t\n]+)\\t(?P[^\t\n]+)\\t(?P[^\t\n]+)", + "markdown_table_limit": 1 + } + } + }, + "artifact": "| 申请人 | 审批状态 | 签署方式 | 电子邮件 |\n| --- | --- | --- | --- |\n| Alpha | Alpha | Alpha | person@example.org |\n| Beta | Beta | Beta | person@example.org |\n| Gamma | Gamma | Gamma | person@example.org |", + "ok": true, + "expected_task_pass": true + }, + { + "id": "completion100-058-30", + "category": "structured-extraction", + "root_cause": "table_fields_and_example_boundaries", + "failure_mode": "false-positive", + "case": { + "url": "https://example.org/table", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 3, + "stable_text_anchors": [], + "duplicate_policy": "deduplicate_identical", + "table_columns": [ + { + "name": "f0", + "label": "申请人", + "type": "text" + }, + { + "name": "f1", + "label": "审批状态", + "type": "text" + }, + { + "name": "f2", + "label": "签署方式", + "type": "text" + }, + { + "name": "f3", + "label": "电子邮件", + "type": "token", + "pattern": "[^\\s@]+@[^\\s@]+\\.[^\\s@]+" } - }, - "artifact": "[![Decorative logo](/logo.png)Actual title](/item/1)", - "ok": true, - "expected_task_pass": true - }, - { - "id": "named-visible-text-image-only", - "category": "structured-extraction", - "root_cause": "image_alt_is_not_visible_record_name", - "failure_mode": "false-positive", - "case": { - "url": "https://example.test/list", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 1, - "hosts": [ - "example.test" - ], - "entry_path": "/item/\\d+", - "name_validation": "nonempty" + ], + "record_identity_fields": [ + "f0" + ], + "table_scope": { + "root_selector": "table", + "root_index": 0, + "row_selector": "tr", + "cell_selector": "td", + "start_pattern": "(?=^\\s*\\|?\\s*申请人(?:\\s|\\|))", + "end_pattern": "^\\s*#{1,6}\\s", + "plain_row_pattern": "(?P[^\t\n]+)\\t(?P[^\t\n]+)\\t(?P[^\t\n]+)\\t(?P[^\t\n]+)", + "markdown_table_limit": 1 + } + } + }, + "artifact": "| 申请人 | 审批状态 | 签署方式 | 电子邮件 |\n| --- | --- | --- | --- |\n| Alpha | Alpha | Alpha | |\n| Beta | Beta | Beta | |\n| Gamma | Gamma | Gamma | |", + "ok": true, + "expected_task_pass": false + }, + { + "id": "completion100-058-31", + "category": "structured-extraction", + "root_cause": "table_fields_and_example_boundaries", + "failure_mode": "false-positive", + "case": { + "url": "https://example.org/table", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 3, + "stable_text_anchors": [], + "duplicate_policy": "deduplicate_identical", + "table_columns": [ + { + "name": "f0", + "label": "申请人", + "type": "text" + }, + { + "name": "f1", + "label": "审批状态", + "type": "text" + }, + { + "name": "f2", + "label": "签署方式", + "type": "text" + }, + { + "name": "f3", + "label": "电子邮件", + "type": "token", + "pattern": "[^\\s@]+@[^\\s@]+\\.[^\\s@]+" } - }, - "artifact": "[![Actual title](/logo.png)](/item/1)", - "ok": true, - "expected_task_pass": false - }, - { - "id": "named-visible-text-image-plus-action", - "category": "structured-extraction", - "root_cause": "image_alt_is_not_visible_record_name", - "failure_mode": "false-positive", - "case": { - "url": "https://example.test/list", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 1, - "hosts": [ - "example.test" - ], - "entry_path": "/item/\\d+", - "name_validation": "nonempty" + ], + "record_identity_fields": [ + "f0" + ], + "table_scope": { + "root_selector": "table", + "root_index": 0, + "row_selector": "tr", + "cell_selector": "td", + "start_pattern": "(?=^\\s*\\|?\\s*申请人(?:\\s|\\|))", + "end_pattern": "^\\s*#{1,6}\\s", + "plain_row_pattern": "(?P[^\t\n]+)\\t(?P[^\t\n]+)\\t(?P[^\t\n]+)\\t(?P[^\t\n]+)", + "markdown_table_limit": 1 + } + } + }, + "artifact": "| 申请人 | 审批状态 | 签署方式 | 电子邮件 |\n| --- | --- | --- | --- |\n| Alpha | Alpha | Alpha | person@example.org |\n| Alpha | Alpha | Alpha | person@example.org |\n| Alpha | Alpha | Alpha | person@example.org |", + "ok": true, + "expected_task_pass": false + }, + { + "id": "completion100-058-32", + "category": "structured-extraction", + "root_cause": "table_fields_and_example_boundaries", + "failure_mode": "false-positive", + "case": { + "url": "https://example.org/table", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 3, + "stable_text_anchors": [], + "duplicate_policy": "deduplicate_identical", + "table_columns": [ + { + "name": "f0", + "label": "申请人", + "type": "text" + }, + { + "name": "f1", + "label": "审批状态", + "type": "text" + }, + { + "name": "f2", + "label": "签署方式", + "type": "text" + }, + { + "name": "f3", + "label": "电子邮件", + "type": "token", + "pattern": "[^\\s@]+@[^\\s@]+\\.[^\\s@]+" } - }, - "artifact": "[![Actual title](/logo.png)Read more](/item/1)", - "ok": true, - "expected_task_pass": false - }, - { - "id": "joyn-completed-window-67", - "category": "structured-extraction", - "root_cause": "premature_partial_dynamic_collection_denominator", - "failure_mode": "false-negative", - "case": { - "url": "https://www.joyn.de/filme", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 67, - "hosts": [ - "www.joyn.de" - ], - "entry_path": "/filme/(?!genre(?:/|$))[^/]+", - "name_validation": "nonempty", - "entry_scope": { - "html_selector": "main div[class*=\"CardStandard_StandardCardWrapper\"]", - "title_selector": "h3", - "link_selector": "a[data-testid=\"LL--movie\"]" - } + ], + "record_identity_fields": [ + "f0" + ], + "table_scope": { + "root_selector": "table", + "root_index": 0, + "row_selector": "tr", + "cell_selector": "td", + "start_pattern": "(?=^\\s*\\|?\\s*申请人(?:\\s|\\|))", + "end_pattern": "^\\s*#{1,6}\\s", + "plain_row_pattern": "(?P[^\t\n]+)\\t(?P[^\t\n]+)\\t(?P[^\t\n]+)\\t(?P[^\t\n]+)", + "markdown_table_limit": 1 + } + } + }, + "artifact": "| 申请人 | 审批状态 | 签署方式 | 电子邮件 |\n| --- | --- | --- | --- |\n| Alpha | Alpha | Alpha | person@example.org |\n\n| 申请人 | 审批状态 | 签署方式 | 电子邮件 |\n| --- | --- | --- | --- |\n| Alpha | Alpha | Alpha | person@example.org |\n| Beta | Beta | Beta | person@example.org |\n| Gamma | Gamma | Gamma | person@example.org |", + "ok": true, + "expected_task_pass": false + }, + { + "id": "completion100-016-33", + "category": "structured-extraction", + "root_cause": "independent_body_scope_omission_or_navigation", + "failure_mode": "false-negative", + "case": { + "url": "https://example.org/body", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "body_text", + "minimum_records": 0, + "stable_text_anchors": [ + { + "id": "heading", + "alternatives": [ + "Example tree" + ] } + ] }, - "artifact": "

Movie 0

Movie 1

Movie 2

Movie 3

Movie 4

Movie 5

Movie 6

Movie 7

Movie 8

Movie 9

Movie 10

Movie 11

Movie 12

Movie 13

Movie 14

Movie 15

Movie 16

Movie 17

Movie 18

Movie 19

Movie 20

Movie 21

Movie 22

Movie 23

Movie 24

Movie 25

Movie 26

Movie 27

Movie 28

Movie 29

Movie 30

Movie 31

Movie 32

Movie 33

Movie 34

Movie 35

Movie 36

Movie 37

Movie 38

Movie 39

Movie 40

Movie 41

Movie 42

Movie 43

Movie 44

Movie 45

Movie 46

Movie 47

Movie 48

Movie 49

Movie 50

Movie 51

Movie 52

Movie 53

Movie 54

Movie 55

Movie 56

Movie 57

Movie 58

Movie 59

Movie 60

Movie 61

Movie 62

Movie 63

Movie 64

Movie 65

Movie 66

", - "ok": true, - "expected_task_pass": true - }, - { - "id": "joyn-completed-window-27", - "category": "structured-extraction", - "root_cause": "premature_partial_dynamic_collection_denominator", - "failure_mode": "false-positive", - "case": { - "url": "https://www.joyn.de/filme", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 67, - "hosts": [ - "www.joyn.de" - ], - "entry_path": "/filme/(?!genre(?:/|$))[^/]+", - "name_validation": "nonempty", - "entry_scope": { - "html_selector": "main div[class*=\"CardStandard_StandardCardWrapper\"]", - "title_selector": "h3", - "link_selector": "a[data-testid=\"LL--movie\"]" - } + "body_requirements": { + "version": 3, + "unit": "region_compact_hexagram_recall", + "minimum_coverage": 0.8, + "minimum_floor": 0.6, + "minimum_region_passes": 1, + "segmentation_method": "sections", + "regions": [ + { + "id": "body-region-1", + "text": "example tree apples flowers", + "source_text": "Example tree\n\nApples\n\nFlowers", + "segmentation_method": "sections" } - }, - "artifact": "

Movie 0

Movie 1

Movie 2

Movie 3

Movie 4

Movie 5

Movie 6

Movie 7

Movie 8

Movie 9

Movie 10

Movie 11

Movie 12

Movie 13

Movie 14

Movie 15

Movie 16

Movie 17

Movie 18

Movie 19

Movie 20

Movie 21

Movie 22

Movie 23

Movie 24

Movie 25

Movie 26

", - "ok": true, - "expected_task_pass": false - }, - { - "id": "nintendo-owned-software-20", - "category": "structured-extraction", - "root_cause": "adjacent_search_result_regions", - "failure_mode": "false-negative", - "case": { - "url": "https://www.nintendo.com/jp/search/index.html?q=Nintendo", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 20, - "hosts": [ - "www.nintendo.com", - "store-jp.nintendo.com" - ], - "entry_path": "(?:/jp/games/(?:switch2|switch)/[A-Za-z0-9-]+/index\\.html|/item/software/[A-Za-z0-9]+)", - "name_validation": "nonempty", - "entry_scope": { - "html_selector": ".nc3-c-softCard", - "title_selector": ".nc3-c-softCard__name", - "link_selector": "a[href]:has(.nc3-c-softCard__name)", - "start_pattern": "(?m)^#{1,6}[ \\t]+ゲームソフトをさがす[ \\t]*$", - "end_pattern": "表示価格はすべて" - } + ] + } + }, + "artifact": "Example tree\nApples\nFlowers", + "ok": true, + "expected_task_pass": true + }, + { + "id": "completion100-016-34", + "category": "structured-extraction", + "root_cause": "independent_body_scope_omission_or_navigation", + "failure_mode": "false-positive", + "case": { + "url": "https://example.org/body", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "body_text", + "minimum_records": 0, + "stable_text_anchors": [ + { + "id": "heading", + "alternatives": [ + "Example tree" + ] } + ] }, - "artifact": "### ページをさがす\n[Official website 0](https://www.nintendo.com/jp/games/switch2/game0/index.html)\n[Official website 1](https://www.nintendo.com/jp/games/switch2/game1/index.html)\n[Official website 2](https://www.nintendo.com/jp/games/switch2/game2/index.html)\n[Official website 3](https://www.nintendo.com/jp/games/switch2/game3/index.html)\n[Official website 4](https://www.nintendo.com/jp/games/switch2/game4/index.html)\n[Official website 5](https://www.nintendo.com/jp/games/switch2/game5/index.html)\n[Official website 6](https://www.nintendo.com/jp/games/switch2/game6/index.html)\n[Official website 7](https://www.nintendo.com/jp/games/switch2/game7/index.html)\n[Official website 8](https://www.nintendo.com/jp/games/switch2/game8/index.html)\n[Official website 9](https://www.nintendo.com/jp/games/switch2/game9/index.html)\n[Official website 10](https://www.nintendo.com/jp/games/switch2/game10/index.html)\n[Official website 11](https://www.nintendo.com/jp/games/switch2/game11/index.html)\n[Official website 12](https://www.nintendo.com/jp/games/switch2/game12/index.html)\n[Official website 13](https://www.nintendo.com/jp/games/switch2/game13/index.html)\n[Official website 14](https://www.nintendo.com/jp/games/switch2/game14/index.html)\n[Official website 15](https://www.nintendo.com/jp/games/switch2/game15/index.html)\n[Official website 16](https://www.nintendo.com/jp/games/switch2/game16/index.html)\n[Official website 17](https://www.nintendo.com/jp/games/switch2/game17/index.html)\n[Official website 18](https://www.nintendo.com/jp/games/switch2/game18/index.html)\n[Official website 19](https://www.nintendo.com/jp/games/switch2/game19/index.html)\n### ゲームソフトをさがす\n[Game 0](https://store-jp.nintendo.com/item/software/D70010000000000)\n[Game 1](https://store-jp.nintendo.com/item/software/D70010000000001)\n[Game 2](https://store-jp.nintendo.com/item/software/D70010000000002)\n[Game 3](https://store-jp.nintendo.com/item/software/D70010000000003)\n[Game 4](https://store-jp.nintendo.com/item/software/D70010000000004)\n[Game 5](https://store-jp.nintendo.com/item/software/D70010000000005)\n[Game 6](https://store-jp.nintendo.com/item/software/D70010000000006)\n[Game 7](https://store-jp.nintendo.com/item/software/D70010000000007)\n[Game 8](https://store-jp.nintendo.com/item/software/D70010000000008)\n[Game 9](https://store-jp.nintendo.com/item/software/D70010000000009)\n[Game 10](https://store-jp.nintendo.com/item/software/D70010000000010)\n[Game 11](https://store-jp.nintendo.com/item/software/D70010000000011)\n[Game 12](https://store-jp.nintendo.com/item/software/D70010000000012)\n[Game 13](https://store-jp.nintendo.com/item/software/D70010000000013)\n[Game 14](https://store-jp.nintendo.com/item/software/D70010000000014)\n[Game 15](https://store-jp.nintendo.com/item/software/D70010000000015)\n[Game 16](https://store-jp.nintendo.com/item/software/D70010000000016)\n[Game 17](https://store-jp.nintendo.com/item/software/D70010000000017)\n[Game 18](https://store-jp.nintendo.com/item/software/D70010000000018)\n[Game 19](https://store-jp.nintendo.com/item/software/D70010000000019)\n表示価格はすべて", - "ok": true, - "expected_task_pass": true - }, - { - "id": "nintendo-owned-software-2", - "category": "structured-extraction", - "root_cause": "adjacent_search_result_regions", - "failure_mode": "false-positive", - "case": { - "url": "https://www.nintendo.com/jp/search/index.html?q=Nintendo", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 20, - "hosts": [ - "www.nintendo.com", - "store-jp.nintendo.com" - ], - "entry_path": "(?:/jp/games/(?:switch2|switch)/[A-Za-z0-9-]+/index\\.html|/item/software/[A-Za-z0-9]+)", - "name_validation": "nonempty", - "entry_scope": { - "html_selector": ".nc3-c-softCard", - "title_selector": ".nc3-c-softCard__name", - "link_selector": "a[href]:has(.nc3-c-softCard__name)", - "start_pattern": "(?m)^#{1,6}[ \\t]+ゲームソフトをさがす[ \\t]*$", - "end_pattern": "表示価格はすべて" - } + "body_requirements": { + "version": 3, + "unit": "region_compact_hexagram_recall", + "minimum_coverage": 0.8, + "minimum_floor": 0.6, + "minimum_region_passes": 1, + "segmentation_method": "sections", + "regions": [ + { + "id": "body-region-1", + "text": "example tree apples flowers", + "source_text": "Example tree\n\nApples\n\nFlowers", + "segmentation_method": "sections" } - }, - "artifact": "### ページをさがす\n[Official website 0](https://www.nintendo.com/jp/games/switch2/game0/index.html)\n[Official website 1](https://www.nintendo.com/jp/games/switch2/game1/index.html)\n[Official website 2](https://www.nintendo.com/jp/games/switch2/game2/index.html)\n[Official website 3](https://www.nintendo.com/jp/games/switch2/game3/index.html)\n[Official website 4](https://www.nintendo.com/jp/games/switch2/game4/index.html)\n[Official website 5](https://www.nintendo.com/jp/games/switch2/game5/index.html)\n[Official website 6](https://www.nintendo.com/jp/games/switch2/game6/index.html)\n[Official website 7](https://www.nintendo.com/jp/games/switch2/game7/index.html)\n[Official website 8](https://www.nintendo.com/jp/games/switch2/game8/index.html)\n[Official website 9](https://www.nintendo.com/jp/games/switch2/game9/index.html)\n[Official website 10](https://www.nintendo.com/jp/games/switch2/game10/index.html)\n[Official website 11](https://www.nintendo.com/jp/games/switch2/game11/index.html)\n[Official website 12](https://www.nintendo.com/jp/games/switch2/game12/index.html)\n[Official website 13](https://www.nintendo.com/jp/games/switch2/game13/index.html)\n[Official website 14](https://www.nintendo.com/jp/games/switch2/game14/index.html)\n[Official website 15](https://www.nintendo.com/jp/games/switch2/game15/index.html)\n[Official website 16](https://www.nintendo.com/jp/games/switch2/game16/index.html)\n[Official website 17](https://www.nintendo.com/jp/games/switch2/game17/index.html)\n[Official website 18](https://www.nintendo.com/jp/games/switch2/game18/index.html)\n[Official website 19](https://www.nintendo.com/jp/games/switch2/game19/index.html)\n### ゲームソフトをさがす\n[Game 0](https://store-jp.nintendo.com/item/software/D70010000000000)\n[Game 1](https://store-jp.nintendo.com/item/software/D70010000000001)\n表示価格はすべて", - "ok": true, - "expected_task_pass": false - }, - { - "id": "completion100-062-1", - "category": "structured-extraction", - "root_cause": "dynamic_membership_must_not_be_frozen", - "failure_mode": "false-negative", - "case": { - "url": "https://example.org", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 2, - "hosts": [ - "example.org" - ], - "entry_path": "/item/[0-9]+", - "name_validation": "nonempty" + ] + } + }, + "artifact": "Navigation\nExample tree", + "ok": true, + "expected_task_pass": false + }, + { + "id": "completion100-039-35", + "category": "structured-extraction", + "root_cause": "independent_body_scope_omission_or_navigation", + "failure_mode": "false-negative", + "case": { + "url": "https://example.org/body", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "body_text", + "minimum_records": 0, + "stable_text_anchors": [ + { + "id": "heading", + "alternatives": [ + "Example tree" + ] } + ] }, - "artifact": "[Fresh title A](/item/91)\n[Fresh title B](/item/92)", - "ok": true, - "expected_task_pass": true - }, - { - "id": "completion100-062-2", - "category": "structured-extraction", - "root_cause": "dynamic_membership_must_not_be_frozen", - "failure_mode": "false-positive", - "case": { - "url": "https://example.org", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 2, - "hosts": [ - "example.org" - ], - "entry_path": "/item/[0-9]+", - "name_validation": "nonempty" + "body_requirements": { + "version": 3, + "unit": "region_compact_hexagram_recall", + "minimum_coverage": 0.8, + "minimum_floor": 0.6, + "minimum_region_passes": 1, + "segmentation_method": "sections", + "regions": [ + { + "id": "body-region-1", + "text": "example tree apples flowers", + "source_text": "Example tree\n\nApples\n\nFlowers", + "segmentation_method": "sections" } - }, - "artifact": "[Fresh title A](/item/91)\n[Fresh title A](/item/91)\n[Invalid](/about)", - "ok": true, - "expected_task_pass": false - }, - { - "id": "completion100-067-3", - "category": "structured-extraction", - "root_cause": "dynamic_membership_must_not_be_frozen", - "failure_mode": "false-negative", - "case": { - "url": "https://example.org", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 2, - "hosts": [ - "example.org" - ], - "entry_path": "/item/[0-9]+", - "name_validation": "nonempty" + ] + } + }, + "artifact": "Example tree\nApples\nFlowers", + "ok": true, + "expected_task_pass": true + }, + { + "id": "completion100-039-36", + "category": "structured-extraction", + "root_cause": "independent_body_scope_omission_or_navigation", + "failure_mode": "false-positive", + "case": { + "url": "https://example.org/body", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "body_text", + "minimum_records": 0, + "stable_text_anchors": [ + { + "id": "heading", + "alternatives": [ + "Example tree" + ] } + ] }, - "artifact": "[Fresh title A](/item/91)\n[Fresh title B](/item/92)", - "ok": true, - "expected_task_pass": true - }, - { - "id": "completion100-067-4", - "category": "structured-extraction", - "root_cause": "dynamic_membership_must_not_be_frozen", - "failure_mode": "false-positive", - "case": { - "url": "https://example.org", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 2, - "hosts": [ - "example.org" - ], - "entry_path": "/item/[0-9]+", - "name_validation": "nonempty" + "body_requirements": { + "version": 3, + "unit": "region_compact_hexagram_recall", + "minimum_coverage": 0.8, + "minimum_floor": 0.6, + "minimum_region_passes": 1, + "segmentation_method": "sections", + "regions": [ + { + "id": "body-region-1", + "text": "example tree apples flowers", + "source_text": "Example tree\n\nApples\n\nFlowers", + "segmentation_method": "sections" } - }, - "artifact": "[Fresh title A](/item/91)\n[Fresh title A](/item/91)\n[Invalid](/about)", - "ok": true, - "expected_task_pass": false - }, - { - "id": "completion100-045-5", - "category": "structured-extraction", - "root_cause": "dynamic_membership_must_not_be_frozen", - "failure_mode": "false-negative", - "case": { - "url": "https://example.org", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 2, - "hosts": [ - "example.org" - ], - "entry_path": "/item/[0-9]+", - "name_validation": "nonempty" + ] + } + }, + "artifact": "Navigation\nExample tree", + "ok": true, + "expected_task_pass": false + }, + { + "id": "completion100-visible-html-image-only", + "category": "structured-extraction", + "root_cause": "image_metadata_is_not_visible_record_name", + "failure_mode": "false-positive", + "case": { + "url": "https://example.org", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 1, + "hosts": [ + "example.org" + ], + "entry_path": "/item/[0-9]+", + "name_validation": "nonempty" + } + }, + "artifact": "
\"Real
", + "ok": true, + "expected_task_pass": false + }, + { + "id": "completion100-visible-html-visible-name", + "category": "structured-extraction", + "root_cause": "image_metadata_is_not_visible_record_name", + "failure_mode": "false-negative", + "case": { + "url": "https://example.org", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 1, + "hosts": [ + "example.org" + ], + "entry_path": "/item/[0-9]+", + "name_validation": "nonempty" + } + }, + "artifact": "
\"Poster\"Real title
", + "ok": true, + "expected_task_pass": true + }, + { + "id": "completion100-nested-anchor-keep-navigation", + "category": "structured-extraction", + "root_cause": "decomposed_descendant_anchor_in_nonrecord_text", + "failure_mode": "false-negative", + "case": { + "url": "https://example.org", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 1, + "hosts": [ + "example.org" + ], + "entry_path": "/item/[0-9]+", + "name_validation": "nonempty", + "entry_scope": { + "html_selector": "article", + "title_selector": "h3", + "link_selector": "a[href]" + }, + "stable_text_anchors": [ + { + "id": "nav", + "alternatives": [ + "Navigation" + ], + "match_scope": "non_record_lines" } - }, - "artifact": "[Fresh title A](/item/91)\n[Fresh title B](/item/92)", - "ok": true, - "expected_task_pass": true - }, - { - "id": "completion100-045-6", - "category": "structured-extraction", - "root_cause": "dynamic_membership_must_not_be_frozen", - "failure_mode": "false-positive", - "case": { - "url": "https://example.org", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 2, - "hosts": [ - "example.org" - ], - "entry_path": "/item/[0-9]+", - "name_validation": "nonempty" + ] + } + }, + "artifact": "
", + "ok": true, + "expected_task_pass": true + }, + { + "id": "completion100-nested-anchor-record-is-not-navigation", + "category": "structured-extraction", + "root_cause": "decomposed_descendant_anchor_in_nonrecord_text", + "failure_mode": "false-positive", + "case": { + "url": "https://example.org", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 1, + "hosts": [ + "example.org" + ], + "entry_path": "/item/[0-9]+", + "name_validation": "nonempty", + "entry_scope": { + "html_selector": "article", + "title_selector": "h3", + "link_selector": "a[href]" + }, + "stable_text_anchors": [ + { + "id": "nav", + "alternatives": [ + "Navigation" + ], + "match_scope": "non_record_lines" } - }, - "artifact": "[Fresh title A](/item/91)\n[Fresh title A](/item/91)\n[Invalid](/about)", - "ok": true, - "expected_task_pass": false - }, - { - "id": "completion100-095-7", - "category": "structured-extraction", - "root_cause": "dynamic_membership_must_not_be_frozen", - "failure_mode": "false-negative", - "case": { - "url": "https://example.org", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 2, - "hosts": [ - "example.org" - ], - "entry_path": "/item/[0-9]+", - "name_validation": "nonempty" + ] + } + }, + "artifact": "
", + "ok": true, + "expected_task_pass": false + }, + { + "id": "completion100-tree-separated", + "category": "body-scope", + "root_cause": "separate_visible_nodes_not_cross_node_ngrams", + "failure_mode": "false-negative", + "case": { + "url": "https://example.org/tree", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "body_text", + "minimum_records": 0, + "stable_text_anchors": [ + { + "id": "heading", + "alternatives": [ + "Tree with dynamic data" + ] } + ] }, - "artifact": "[Fresh title A](/item/91)\n[Fresh title B](/item/92)", - "ok": true, - "expected_task_pass": true - }, - { - "id": "completion100-095-8", - "category": "structured-extraction", - "root_cause": "dynamic_membership_must_not_be_frozen", - "failure_mode": "false-positive", - "case": { - "url": "https://example.org", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "named_link", - "minimum_records": 2, - "hosts": [ - "example.org" - ], - "entry_path": "/item/[0-9]+", - "name_validation": "nonempty" + "body_requirements": { + "version": 3, + "unit": "region_compact_hexagram_recall", + "minimum_coverage": 0.8, + "minimum_floor": 0.6, + "minimum_region_passes": 3, + "segmentation_method": "balanced_tokens_fallback", + "regions": [ + { + "id": "visible-block-1", + "text": "tree with dynamic data", + "source_text": "tree with dynamic data", + "segmentation_method": "balanced_tokens_fallback" + }, + { + "id": "visible-block-2", + "text": "fruits", + "source_text": "fruits", + "segmentation_method": "balanced_tokens_fallback" + }, + { + "id": "visible-block-3", + "text": "vegetables", + "source_text": "vegetables", + "segmentation_method": "balanced_tokens_fallback" } - }, - "artifact": "[Fresh title A](/item/91)\n[Fresh title A](/item/91)\n[Invalid](/about)", - "ok": true, - "expected_task_pass": false - }, - { - "id": "completion100-011-9", - "category": "structured-extraction", - "root_cause": "table_fields_and_example_boundaries", - "failure_mode": "false-negative", - "case": { - "url": "https://example.org/table", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "table_row", - "minimum_records": 4, - "stable_text_anchors": [], - "duplicate_policy": "deduplicate_identical", - "table_columns": [ - { - "name": "f0", - "label": "Product", - "type": "text" - }, - { - "name": "f1", - "label": "Category", - "type": "text" - }, - { - "name": "f2", - "label": "Price", - "type": "number" - } - ], - "record_identity_fields": [ - "f0" - ], - "table_scope": { - "root_selector": "main table", - "root_index": 0, - "row_selector": "tr", - "cell_selector": "td", - "start_pattern": "(?=^\\s*\\|?\\s*Product(?:\\s|\\|))", - "end_pattern": "^\\s*#{1,6}\\s", - "plain_row_pattern": "(?P[^\t\n]+)\\t(?P[^\t\n]+)\\t(?P[^\t\n]+)", - "markdown_table_limit": 1 - } + ] + } + }, + "artifact": "Tree with dynamic data\nDecoration\nFruits\nDecoration\nVegetables", + "ok": true, + "expected_task_pass": true + }, + { + "id": "completion100-tree-missing-node", + "category": "body-scope", + "root_cause": "separate_visible_nodes_not_cross_node_ngrams", + "failure_mode": "false-positive", + "case": { + "url": "https://example.org/tree", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "body_text", + "minimum_records": 0, + "stable_text_anchors": [ + { + "id": "heading", + "alternatives": [ + "Tree with dynamic data" + ] } + ] }, - "artifact": "| Product | Category | Price |\n| --- | --- | --- |\n| Alpha | Alpha | 1.5 |\n| Beta | Beta | 1.5 |\n| Gamma | Gamma | 1.5 |\n| Delta | Delta | 1.5 |", - "ok": true, - "expected_task_pass": true - }, - { - "id": "completion100-011-10", - "category": "structured-extraction", - "root_cause": "table_fields_and_example_boundaries", - "failure_mode": "false-positive", - "case": { - "url": "https://example.org/table", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "table_row", - "minimum_records": 4, - "stable_text_anchors": [], - "duplicate_policy": "deduplicate_identical", - "table_columns": [ - { - "name": "f0", - "label": "Product", - "type": "text" - }, - { - "name": "f1", - "label": "Category", - "type": "text" - }, - { - "name": "f2", - "label": "Price", - "type": "number" - } - ], - "record_identity_fields": [ - "f0" - ], - "table_scope": { - "root_selector": "main table", - "root_index": 0, - "row_selector": "tr", - "cell_selector": "td", - "start_pattern": "(?=^\\s*\\|?\\s*Product(?:\\s|\\|))", - "end_pattern": "^\\s*#{1,6}\\s", - "plain_row_pattern": "(?P[^\t\n]+)\\t(?P[^\t\n]+)\\t(?P[^\t\n]+)", - "markdown_table_limit": 1 - } + "body_requirements": { + "version": 3, + "unit": "region_compact_hexagram_recall", + "minimum_coverage": 0.8, + "minimum_floor": 0.6, + "minimum_region_passes": 3, + "segmentation_method": "balanced_tokens_fallback", + "regions": [ + { + "id": "visible-block-1", + "text": "tree with dynamic data", + "source_text": "tree with dynamic data", + "segmentation_method": "balanced_tokens_fallback" + }, + { + "id": "visible-block-2", + "text": "fruits", + "source_text": "fruits", + "segmentation_method": "balanced_tokens_fallback" + }, + { + "id": "visible-block-3", + "text": "vegetables", + "source_text": "vegetables", + "segmentation_method": "balanced_tokens_fallback" } - }, - "artifact": "| Product | Category | Price |\n| --- | --- | --- |\n| Alpha | Alpha | |\n| Beta | Beta | |\n| Gamma | Gamma | |\n| Delta | Delta | |", - "ok": true, - "expected_task_pass": false - }, - { - "id": "completion100-011-11", - "category": "structured-extraction", - "root_cause": "table_fields_and_example_boundaries", - "failure_mode": "false-positive", - "case": { - "url": "https://example.org/table", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "table_row", - "minimum_records": 4, - "stable_text_anchors": [], - "duplicate_policy": "deduplicate_identical", - "table_columns": [ - { - "name": "f0", - "label": "Product", - "type": "text" - }, - { - "name": "f1", - "label": "Category", - "type": "text" - }, - { - "name": "f2", - "label": "Price", - "type": "number" - } - ], - "record_identity_fields": [ - "f0" - ], - "table_scope": { - "root_selector": "main table", - "root_index": 0, - "row_selector": "tr", - "cell_selector": "td", - "start_pattern": "(?=^\\s*\\|?\\s*Product(?:\\s|\\|))", - "end_pattern": "^\\s*#{1,6}\\s", - "plain_row_pattern": "(?P[^\t\n]+)\\t(?P[^\t\n]+)\\t(?P[^\t\n]+)", - "markdown_table_limit": 1 - } + ] + } + }, + "artifact": "Tree with dynamic data\nDecoration\nFruits", + "ok": true, + "expected_task_pass": false + }, + { + "id": "completion100-inline-html-real-markdown", + "category": "structured-extraction", + "root_cause": "inline_html_documentation_is_not_a_rendered_table", + "failure_mode": "false-negative", + "case": { + "url": "https://example.org/table", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 4, + "stable_text_anchors": [], + "duplicate_policy": "deduplicate_identical", + "table_columns": [ + { + "name": "f0", + "label": "Name", + "type": "text" + }, + { + "name": "f1", + "label": "Role", + "type": "text" + }, + { + "name": "f2", + "label": "Status", + "type": "text" + }, + { + "name": "f3", + "label": "Email", + "type": "token", + "pattern": "[^\\s@]+@[^\\s@]+\\.[^\\s@]+" } - }, - "artifact": "| Product | Category | Price |\n| --- | --- | --- |\n| Alpha | Alpha | 1.5 |\n| Alpha | Alpha | 1.5 |\n| Alpha | Alpha | 1.5 |\n| Alpha | Alpha | 1.5 |", - "ok": true, - "expected_task_pass": false - }, - { - "id": "completion100-011-12", - "category": "structured-extraction", - "root_cause": "table_fields_and_example_boundaries", - "failure_mode": "false-positive", - "case": { - "url": "https://example.org/table", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "table_row", - "minimum_records": 4, - "stable_text_anchors": [], - "duplicate_policy": "deduplicate_identical", - "table_columns": [ - { - "name": "f0", - "label": "Product", - "type": "text" - }, - { - "name": "f1", - "label": "Category", - "type": "text" - }, - { - "name": "f2", - "label": "Price", - "type": "number" - } - ], - "record_identity_fields": [ - "f0" - ], - "table_scope": { - "root_selector": "main table", - "root_index": 0, - "row_selector": "tr", - "cell_selector": "td", - "start_pattern": "(?=^\\s*\\|?\\s*Product(?:\\s|\\|))", - "end_pattern": "^\\s*#{1,6}\\s", - "plain_row_pattern": "(?P[^\t\n]+)\\t(?P[^\t\n]+)\\t(?P[^\t\n]+)", - "markdown_table_limit": 1 - } + ], + "record_identity_fields": [ + "f0" + ], + "table_scope": { + "root_selector": "table", + "root_index": 0, + "row_selector": "tr", + "cell_selector": "td", + "start_pattern": "(?=^\\s*\\|?\\s*Name(?:\\s|\\|))", + "end_pattern": "^\\s*#{1,6}\\s", + "plain_row_pattern": "(?P[^\t\n]+)\\t(?P[^\t\n]+)\\t(?P[^\t\n]+)\\t(?P[^\t\n]+)", + "markdown_table_limit": 1 + } + } + }, + "artifact": "Documentation\n| Name | Role | Status | Email |\n| --- | --- | --- | --- |\n| Alpha | Alpha | Alpha | person@example.org |\n| Beta | Beta | Beta | person@example.org |\n| Gamma | Gamma | Gamma | person@example.org |\n| Delta | Delta | Delta | person@example.org |\n\nThe `` contains a `` and a `
`.", + "ok": true, + "expected_task_pass": true + }, + { + "id": "completion100-inline-html-example-only", + "category": "structured-extraction", + "root_cause": "inline_html_documentation_is_not_a_rendered_table", + "failure_mode": "false-positive", + "case": { + "url": "https://example.org/table", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "table_row", + "minimum_records": 4, + "stable_text_anchors": [], + "duplicate_policy": "deduplicate_identical", + "table_columns": [ + { + "name": "f0", + "label": "Name", + "type": "text" + }, + { + "name": "f1", + "label": "Role", + "type": "text" + }, + { + "name": "f2", + "label": "Status", + "type": "text" + }, + { + "name": "f3", + "label": "Email", + "type": "token", + "pattern": "[^\\s@]+@[^\\s@]+\\.[^\\s@]+" } - }, - "artifact": "| Product | Category | Price |\n| --- | --- | --- |\n| Alpha | Alpha | 1.5 |\n\n| Product | Category | Price |\n| --- | --- | --- |\n| Alpha | Alpha | 1.5 |\n| Beta | Beta | 1.5 |\n| Gamma | Gamma | 1.5 |\n| Delta | Delta | 1.5 |", - "ok": true, - "expected_task_pass": false - }, - { - "id": "completion100-024-13", - "category": "structured-extraction", - "root_cause": "table_fields_and_example_boundaries", - "failure_mode": "false-negative", - "case": { - "url": "https://example.org/table", - "task_kind": "body", - "acceptance_contract": { - "version": 1, - "schema": "table_row", - "minimum_records": 2, - "stable_text_anchors": [], - "duplicate_policy": "deduplicate_identical", - "table_columns": [ - { - "name": "f0", - "label": "Name", - "type": "text" - }, - { - "name": "f1", - "label": "Email", - "type": "token", - "pattern": "[^\\s@]+@[^\\s@]+\\.[^\\s@]+" - }, - { - "name": "f2", - "label": "Phone Number", - "type": "token", - "pattern": "[+()\\d][\\d ()+.-]+" - } - ], - "record_identity_fields": [ - "f0" - ], - "table_scope": { - "root_selector": "main table", - "root_index": 0, - "row_selector": "tr", - "cell_selector": "td", - "start_pattern": "(?=^\\s*\\|?\\s*Name(?:\\s|\\|))", - "end_pattern": "^\\s*#{1,6}\\s", - "plain_row_pattern": "(?P[^\t\n]+)\\t(?P[^\t\n]+)\\t(?P[^\t\n]+)", - "markdown_table_limit": 1 - } + ], + "record_identity_fields": [ + "f0" + ], + "table_scope": { + "root_selector": "table", + "root_index": 0, + "row_selector": "tr", + "cell_selector": "td", + "start_pattern": "(?=^\\s*\\|?\\s*Name(?:\\s|\\|))", + "end_pattern": "^\\s*#{1,6}\\s", + "plain_row_pattern": "(?P[^\t\n]+)\\t(?P[^\t\n]+)\\t(?P[^\t\n]+)\\t(?P[^\t\n]+)", + "markdown_table_limit": 1 + } + } + }, + "artifact": "Documentation\nThe `` contains a `` and a `
`.", + "ok": true, + "expected_task_pass": false + }, + { + "id": "completion100-list-boundary-045-inside", + "category": "structured-extraction", + "root_cause": "markdown_section_ownership_matches_source_scope", + "failure_mode": "false-negative", + "case": { + "url": "https://example.org", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "example.org" + ], + "entry_path": "/item/[0-9]+", + "name_validation": "nonempty", + "entry_scope": { + "html_selector": "article", + "start_pattern": "(?m)^#{1,6}[ \\t]+NBA Standings[ \\t]*$", + "end_pattern": "(?m)^#{1,6}[ \\t]+Legend[ \\t]*$" + } + } + }, + "artifact": "# NBA Standings\n[New A](/item/1)\n[New B](/item/2)\n## Legend", + "ok": true, + "expected_task_pass": true + }, + { + "id": "completion100-list-boundary-045-outside", + "category": "structured-extraction", + "root_cause": "markdown_section_ownership_matches_source_scope", + "failure_mode": "false-positive", + "case": { + "url": "https://example.org", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "example.org" + ], + "entry_path": "/item/[0-9]+", + "name_validation": "nonempty", + "entry_scope": { + "html_selector": "article", + "start_pattern": "(?m)^#{1,6}[ \\t]+NBA Standings[ \\t]*$", + "end_pattern": "(?m)^#{1,6}[ \\t]+Legend[ \\t]*$" + } + } + }, + "artifact": "[Outside A](/item/1)\n[Outside B](/item/2)\n# NBA Standings\n[One only](/item/3)\n## Legend\n[Outside C](/item/4)", + "ok": true, + "expected_task_pass": false + }, + { + "id": "completion100-list-boundary-067-inside", + "category": "structured-extraction", + "root_cause": "markdown_section_ownership_matches_source_scope", + "failure_mode": "false-negative", + "case": { + "url": "https://example.org", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "example.org" + ], + "entry_path": "/item/[0-9]+", + "name_validation": "nonempty", + "entry_scope": { + "html_selector": "article", + "start_pattern": "(?m)^#{1,6}[ \\t]+トピックス[ \\t]*$", + "end_pattern": "(?m)^#{1,6}[ \\t]+ヤフコメ トレンド[ \\t]*$" + } + } + }, + "artifact": "## トピックス\n[New A](/item/1)\n[New B](/item/2)\n## ヤフコメ トレンド", + "ok": true, + "expected_task_pass": true + }, + { + "id": "completion100-list-boundary-067-outside", + "category": "structured-extraction", + "root_cause": "markdown_section_ownership_matches_source_scope", + "failure_mode": "false-positive", + "case": { + "url": "https://example.org", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "example.org" + ], + "entry_path": "/item/[0-9]+", + "name_validation": "nonempty", + "entry_scope": { + "html_selector": "article", + "start_pattern": "(?m)^#{1,6}[ \\t]+トピックス[ \\t]*$", + "end_pattern": "(?m)^#{1,6}[ \\t]+ヤフコメ トレンド[ \\t]*$" + } + } + }, + "artifact": "[Outside A](/item/1)\n[Outside B](/item/2)\n## トピックス\n[One only](/item/3)\n## ヤフコメ トレンド\n[Outside C](/item/4)", + "ok": true, + "expected_task_pass": false + }, + { + "id": "completion100-list-boundary-062-inside", + "category": "structured-extraction", + "root_cause": "markdown_section_ownership_matches_source_scope", + "failure_mode": "false-negative", + "case": { + "url": "https://example.org", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "example.org" + ], + "entry_path": "/item/[0-9]+", + "name_validation": "nonempty", + "entry_scope": { + "html_selector": "article", + "start_pattern": "(?m)^(?:推薦|推荐|\\[推荐\\]\\([^\\n]+\\))[ \\t]*$", + "end_pattern": "(?m)^(?:#{1,6}[ \\t]+)?派友在看[ \\t]*$" + } + } + }, + "artifact": "推荐\n[New A](/item/1)\n[New B](/item/2)\n派友在看", + "ok": true, + "expected_task_pass": true + }, + { + "id": "completion100-list-boundary-062-outside", + "category": "structured-extraction", + "root_cause": "markdown_section_ownership_matches_source_scope", + "failure_mode": "false-positive", + "case": { + "url": "https://example.org", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "example.org" + ], + "entry_path": "/item/[0-9]+", + "name_validation": "nonempty", + "entry_scope": { + "html_selector": "article", + "start_pattern": "(?m)^(?:推薦|推荐|\\[推荐\\]\\([^\\n]+\\))[ \\t]*$", + "end_pattern": "(?m)^(?:#{1,6}[ \\t]+)?派友在看[ \\t]*$" + } + } + }, + "artifact": "[Outside A](/item/1)\n[Outside B](/item/2)\n推荐\n[One only](/item/3)\n派友在看\n[Outside C](/item/4)", + "ok": true, + "expected_task_pass": false + }, + { + "id": "remaining7-caption-md", + "category": "structured-extraction", + "root_cause": "linked_card_caption_ownership", + "failure_mode": "false-negative", + "case": { + "url": "https://example.org", + "task_kind": "body", + "acceptance_contract": { + "version": 1, + "schema": "named_link", + "minimum_records": 2, + "hosts": [ + "example.org" + ], + "entry_path": "/(?:serien|filme|shows)/(?!genre(?:/|$))[^/]+", + "stable_text_anchors": [], + "name_validation": "nonempty", + "entry_scope": { + "html_selector": "main a:has(h3):has(p)", + "title_selector": "p", + "link_selector": "a[href]", + "start_pattern": "START", + "end_pattern": "END", + "markdown_label_pattern": "(?s)
\\s*(?P