From e0eafb87f952582f6e360a5081efb7d538fcf143 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 16 Nov 2026 06:50:29 +0000 Subject: [PATCH] fix(website): show the current Snowflake syntax figure and fail CI when it drifts The website still showed 2,388 of 3,351 forms from 6 November. The figures are now written from the Snowflake coverage summary, and check-ci fails while website/src/data/figures.json differs from it. --- Makefile | 12 ++- scripts/check_website_figures.py | 14 ++++ scripts/replay_scope/_helpers/decisions.py | 5 +- scripts/replay_scope/constants.py | 1 + scripts/website_figures/__init__.py | 1 + scripts/website_figures/_helpers/__init__.py | 0 scripts/website_figures/_helpers/figures.py | 32 ++++++++ scripts/website_figures/constants.py | 16 ++++ scripts/website_figures/main/__init__.py | 1 + scripts/website_figures/main/figures.py | 56 +++++++++++++ tests/unit/scripts/replay_scope/test_scope.py | 6 ++ .../unit/scripts/website_figures/__init__.py | 1 + .../scripts/website_figures/_test_types.py | 21 +++++ tests/unit/scripts/website_figures/helpers.py | 32 ++++++++ .../scripts/website_figures/test_figures.py | 78 +++++++++++++++++++ website/src/content/docs/docs/coverage.mdx | 9 ++- website/src/data/figures.json | 8 +- 17 files changed, 282 insertions(+), 11 deletions(-) create mode 100644 scripts/check_website_figures.py create mode 100644 scripts/website_figures/__init__.py create mode 100644 scripts/website_figures/_helpers/__init__.py create mode 100644 scripts/website_figures/_helpers/figures.py create mode 100644 scripts/website_figures/constants.py create mode 100644 scripts/website_figures/main/__init__.py create mode 100644 scripts/website_figures/main/figures.py create mode 100644 tests/unit/scripts/website_figures/__init__.py create mode 100644 tests/unit/scripts/website_figures/_test_types.py create mode 100644 tests/unit/scripts/website_figures/helpers.py create mode 100644 tests/unit/scripts/website_figures/test_figures.py diff --git a/Makefile b/Makefile index b88866acf..48d5bb671 100644 --- a/Makefile +++ b/Makefile @@ -30,7 +30,7 @@ PYTHON_TYPED := scripts tests/unit/scripts python/src $(PYTHON_API_TESTS) PYTHON_TESTS := tests/unit/scripts $(PYTHON_API_TESTS) endif -.PHONY: distribution-notices format rust-lint python-check fensu syntax-coverage depth-probe depth-probe-debug instruction-counts instruction-baseline gate-build engine-free bundled-extensions dialect-builds generate-cases generated-cases generate-syntax-cases syntax-obligations syntax-cases dialect-cases corpus-rerecord corpus-verdicts corpus-verdicts-baseline generate-referee-cases referee-cases referee-pending translation-files translation-results help pending check-syntax record-syntax check-convert record-convert sql-sizes sql-size-baseline sql-benchmark check-ci check-merge test test-rust test-python python-build python-module python-api python-wheel verify-recorded verify-changed replay-scope test-crate check-pr-metadata evidence-clone evidence-refresh evidence-pin evidence-pin-check clutter-guard census-duckshop fuzz-smoke gate gate-parallel census duckshop-compiled duckshop-measure +.PHONY: distribution-notices format rust-lint python-check fensu syntax-coverage website-figures website-figures-update depth-probe depth-probe-debug instruction-counts instruction-baseline gate-build engine-free bundled-extensions dialect-builds generate-cases generated-cases generate-syntax-cases syntax-obligations syntax-cases dialect-cases corpus-rerecord corpus-verdicts corpus-verdicts-baseline generate-referee-cases referee-cases referee-pending translation-files translation-results help pending check-syntax record-syntax check-convert record-convert sql-sizes sql-size-baseline sql-benchmark check-ci check-merge test test-rust test-python python-build python-module python-api python-wheel verify-recorded verify-changed replay-scope test-crate check-pr-metadata evidence-clone evidence-refresh evidence-pin evidence-pin-check clutter-guard census-duckshop fuzz-smoke gate gate-parallel census duckshop-compiled duckshop-measure format: cargo fmt --all @@ -68,6 +68,14 @@ syntax-coverage: gate-build $(MAKE) --no-print-directory check-syntax DIALECT=$$dialect REQUIRE_BASELINE=1; \ done +# Fails when the website's syntax figures (website/src/data/figures.json) differ from the Snowflake +# coverage summary; `website-figures-update` rewrites them from it. +SYNTAX_SUMMARY_FILE = target/syntax-summary.json +website-figures website-figures-update: gate-build + @mkdir -p target + $(GATE_DIR)/syntax-coverage summary --evidence $(call syntax_evidence_dir,snowflake) --json > $(SYNTAX_SUMMARY_FILE) + uv run python -m scripts.check_website_figures $(if $(filter website-figures-update,$@),write,check) $(SYNTAX_SUMMARY_FILE) + # Fails when a native parser crashes, panics or hangs on deeply nested SQL on a 2 MiB thread and # coverage/deep-nesting.toml does not list it, or when a listed failure no longer happens. depth-probe: gate-build @@ -236,7 +244,7 @@ referee-pending: uv run python -m scripts.write_referee_pending $(REFEREE_ARGS) # The static gate, the first step of `gate`, and the Python API's tests. -check-ci: rust-lint python-check fensu syntax-coverage depth-probe instruction-counts corpus-verdicts evidence-pin-check clutter-guard engine-free bundled-extensions dialect-builds generated-cases syntax-obligations syntax-cases dialect-cases referee-cases translation-files python-api distribution-notices replay-scope ## The static gate CI runs. +check-ci: rust-lint python-check fensu syntax-coverage website-figures depth-probe instruction-counts corpus-verdicts evidence-pin-check clutter-guard engine-free bundled-extensions dialect-builds generated-cases syntax-obligations syntax-cases dialect-cases referee-cases translation-files python-api distribution-notices replay-scope ## The static gate CI runs. # Each distribution ships the licence and third-party notices; its copies must match the root files. distribution-notices: diff --git a/scripts/check_website_figures.py b/scripts/check_website_figures.py new file mode 100644 index 000000000..987ebbb61 --- /dev/null +++ b/scripts/check_website_figures.py @@ -0,0 +1,14 @@ +"""Direct wrapper for the website figures.""" + +from __future__ import annotations + +from scripts.website_figures.main.figures import run_figures + + +def main(argv: list[str] | None = None) -> int: + """Write the website's syntax figures from the coverage summary, or fail when they differ.""" + return run_figures(argv) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/replay_scope/_helpers/decisions.py b/scripts/replay_scope/_helpers/decisions.py index 1f3397e9a..26f72624e 100644 --- a/scripts/replay_scope/_helpers/decisions.py +++ b/scripts/replay_scope/_helpers/decisions.py @@ -9,6 +9,7 @@ from scripts.replay_scope._helpers.git import changed, tracked from scripts.replay_scope.constants import ( BINARY_PACKAGE, + CHECKED_DOCUMENTATION_FILES, DIRECTORY_SUFFIX, DOCUMENTATION_PATHS, FULL_REPLAY_PATHS, @@ -40,7 +41,9 @@ def needs_code_checks(*, paths: list[str]) -> bool: def documentation(*, path: str) -> bool: - """Whether `path` is documentation: under a documentation path, or top-level Markdown.""" + """Whether `path` is documentation, other than the figures file check-ci compares.""" + if path in CHECKED_DOCUMENTATION_FILES: + return False top_level_markdown: bool = path.endswith(MARKDOWN_SUFFIX) and DIRECTORY_SUFFIX not in path return top_level_markdown or under(path=path, prefixes=DOCUMENTATION_PATHS) diff --git a/scripts/replay_scope/constants.py b/scripts/replay_scope/constants.py index cf5a14b4b..ef951747e 100644 --- a/scripts/replay_scope/constants.py +++ b/scripts/replay_scope/constants.py @@ -28,6 +28,7 @@ "tools/sqlquack-coverage-check/", ) DOCUMENTATION_PATHS: tuple[str, ...] = ("website/", "docs/") +CHECKED_DOCUMENTATION_FILES: tuple[str, ...] = ("website/src/data/figures.json",) SITE_PATHS: tuple[str, ...] = ( "website/", ".github/workflows/website-build.yml", diff --git a/scripts/website_figures/__init__.py b/scripts/website_figures/__init__.py new file mode 100644 index 000000000..aff621749 --- /dev/null +++ b/scripts/website_figures/__init__.py @@ -0,0 +1 @@ +"""The website's headline syntax figures, written from and checked against the coverage summary.""" diff --git a/scripts/website_figures/_helpers/__init__.py b/scripts/website_figures/_helpers/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/scripts/website_figures/_helpers/figures.py b/scripts/website_figures/_helpers/figures.py new file mode 100644 index 000000000..d581f6edb --- /dev/null +++ b/scripts/website_figures/_helpers/figures.py @@ -0,0 +1,32 @@ +"""The figures a coverage summary gives the website, and how the written ones differ.""" + +from __future__ import annotations + +from typing import Any + +from scripts.website_figures.constants import BASIS_KEY, FORMS_KEY, PERCENT_KEY, VERIFIED_KEY + + +def summary_figures(summary: dict[str, Any]) -> dict[str, str]: + """The verified count, the form count and the verified share, as the website writes them.""" + total: dict[str, int] = summary["total"] + return { + VERIFIED_KEY: f"{total['verified']:,}", + FORMS_KEY: f"{total['forms']:,}", + PERCENT_KEY: f"{summary['verified_percent']:.2f}%", + } + + +def figure_problems(*, figures: dict[str, Any], expected: dict[str, str]) -> list[str]: + """Each figure that differs from the summary, and a basis that does not state the count.""" + problems: list[str] = [ + f"{key} is {figures.get(key)!r}, but the summary gives {value!r}" + for key, value in expected.items() + if figures.get(key) != value + ] + basis: object = figures.get(BASIS_KEY) + if not isinstance(basis, str) or not basis.startswith(f"{expected[VERIFIED_KEY]} "): + problems.append( + f"{BASIS_KEY} does not start with the verified count {expected[VERIFIED_KEY]}" + ) + return problems diff --git a/scripts/website_figures/constants.py b/scripts/website_figures/constants.py new file mode 100644 index 000000000..999b361a8 --- /dev/null +++ b/scripts/website_figures/constants.py @@ -0,0 +1,16 @@ +"""Settings for keeping the website's syntax figures equal to the coverage summary.""" + +from __future__ import annotations + +FIGURES_FILE: str = "website/src/data/figures.json" +VERIFIED_KEY: str = "SYNTAX_VERIFIED" +FORMS_KEY: str = "SYNTAX_FORMS" +PERCENT_KEY: str = "SYNTAX_PCT" +BASIS_KEY: str = "SYNTAX_BASIS" +BASIS_TEMPLATE: str = ( + "{verified} forms verified against cases recorded on Snowflake and approved reviews, " + "as of {day}." +) +WRITE_COMMAND: str = "write" +CHECK_COMMAND: str = "check" +JSON_INDENT: str = "\t" diff --git a/scripts/website_figures/main/__init__.py b/scripts/website_figures/main/__init__.py new file mode 100644 index 000000000..37c77e3fd --- /dev/null +++ b/scripts/website_figures/main/__init__.py @@ -0,0 +1 @@ +"""Entry points for the website figures.""" diff --git a/scripts/website_figures/main/figures.py b/scripts/website_figures/main/figures.py new file mode 100644 index 000000000..e4e8780d1 --- /dev/null +++ b/scripts/website_figures/main/figures.py @@ -0,0 +1,56 @@ +"""Write the website's syntax figures from a coverage summary, or check that they match it.""" + +from __future__ import annotations + +import argparse +import datetime +import json +import sys +from pathlib import Path +from typing import Any + +from scripts.website_figures._helpers.figures import figure_problems, summary_figures +from scripts.website_figures.constants import ( + BASIS_KEY, + BASIS_TEMPLATE, + CHECK_COMMAND, + FIGURES_FILE, + FORMS_KEY, + JSON_INDENT, + VERIFIED_KEY, + WRITE_COMMAND, +) + + +def run_figures(argv: list[str] | None = None) -> int: + """Write or check the syntax figures against `syntax-coverage summary --json` output.""" + parser: argparse.ArgumentParser = argparse.ArgumentParser(description=run_figures.__doc__) + parser.add_argument("command", choices=(WRITE_COMMAND, CHECK_COMMAND)) + parser.add_argument("summary", type=Path, help="the JSON `syntax-coverage summary` printed") + parser.add_argument("--figures", type=Path, default=Path(FIGURES_FILE)) + parser.add_argument("--day", type=datetime.date.fromisoformat, default=datetime.date.today()) + arguments: argparse.Namespace = parser.parse_args(argv) + expected: dict[str, str] = summary_figures(json.loads(arguments.summary.read_text())) + figures: dict[str, Any] = json.loads(arguments.figures.read_text(encoding="utf-8")) + if arguments.command == WRITE_COMMAND: + day: datetime.date = arguments.day + basis: str = BASIS_TEMPLATE.format( + verified=expected[VERIFIED_KEY], day=f"{day.day} {day:%B %Y}" + ) + figures.update({**expected, BASIS_KEY: basis}) + text: str = json.dumps(figures, indent=JSON_INDENT, ensure_ascii=False) + "\n" + arguments.figures.write_text(text, encoding="utf-8") + print(f"website figures: wrote {basis}") + return 0 + problems: list[str] = figure_problems(figures=figures, expected=expected) + for problem in problems: + print(f"website figures: {problem}", file=sys.stderr) + if problems: + print( + f"website figures: rewrite them with `make website-figures-update`, then commit " + f"{FIGURES_FILE}", + file=sys.stderr, + ) + return 1 + print(f"website figures: passed, {expected[VERIFIED_KEY]} of {expected[FORMS_KEY]} forms") + return 0 diff --git a/tests/unit/scripts/replay_scope/test_scope.py b/tests/unit/scripts/replay_scope/test_scope.py index 651bd583b..03f89cb96 100644 --- a/tests/unit/scripts/replay_scope/test_scope.py +++ b/tests/unit/scripts/replay_scope/test_scope.py @@ -99,6 +99,12 @@ def test_given_changed_paths_when_scoping_then_the_replay_matches_what_they_feed expected_code=False, expected_site=True, ), + ChecksTestCase( + description="the website figures need the code checks that compare them", + changed=("website/src/data/figures.json",), + expected_code=True, + expected_site=True, + ), ChecksTestCase( description="the README needs neither the code checks nor the site", changed=("README.md",), diff --git a/tests/unit/scripts/website_figures/__init__.py b/tests/unit/scripts/website_figures/__init__.py new file mode 100644 index 000000000..84ef48c3c --- /dev/null +++ b/tests/unit/scripts/website_figures/__init__.py @@ -0,0 +1 @@ +"""Website figure tests.""" diff --git a/tests/unit/scripts/website_figures/_test_types.py b/tests/unit/scripts/website_figures/_test_types.py new file mode 100644 index 000000000..2a395a236 --- /dev/null +++ b/tests/unit/scripts/website_figures/_test_types.py @@ -0,0 +1,21 @@ +"""Test types for the website figures.""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any + + +@dataclass(frozen=True) +class WriteTestCase: + description: str + day: str + expected_figures: dict[str, str] + + +@dataclass(frozen=True) +class CheckTestCase: + description: str + edits: dict[str, Any] + expected_return_code: int + expected_fragment: str diff --git a/tests/unit/scripts/website_figures/helpers.py b/tests/unit/scripts/website_figures/helpers.py new file mode 100644 index 000000000..cb6c8e5c2 --- /dev/null +++ b/tests/unit/scripts/website_figures/helpers.py @@ -0,0 +1,32 @@ +"""A coverage summary and stale website figures, written into a temporary directory.""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any + +from scripts.website_figures.main.figures import run_figures + +SUMMARY: dict[str, Any] = { + "total": {"forms": 3351, "verified": 3199}, + "verified_percent": 95.4640405849, +} +STALE_FIGURES: dict[str, str] = { + "SYNTAX_VERIFIED": "2,388", + "SYNTAX_FORMS": "3,351", + "SYNTAX_PCT": "71.26%", + "SYNTAX_BASIS": "2,388 forms verified, as of 6 November 2026.", + "DUCKSHOP_MODELS": "1,000", +} + + +def written_files(*, tmp_path: Path, day: str) -> tuple[Path, Path]: + """The summary file and the figures file after `write` replaced the stale figures.""" + summary: Path = tmp_path / "summary.json" + figures: Path = tmp_path / "figures.json" + summary.write_text(json.dumps(SUMMARY)) + figures.write_text(json.dumps(STALE_FIGURES)) + written: int = run_figures(["write", str(summary), "--figures", str(figures), "--day", day]) + assert written == 0 + return summary, figures diff --git a/tests/unit/scripts/website_figures/test_figures.py b/tests/unit/scripts/website_figures/test_figures.py new file mode 100644 index 000000000..79413849a --- /dev/null +++ b/tests/unit/scripts/website_figures/test_figures.py @@ -0,0 +1,78 @@ +"""Writing the website's syntax figures from a coverage summary, then checking them against it.""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any + +import pytest + +from scripts.website_figures.main.figures import run_figures +from tests.unit.scripts.website_figures._test_types import CheckTestCase, WriteTestCase +from tests.unit.scripts.website_figures.helpers import written_files + + +@pytest.mark.parametrize( + "test_case", + ( + WriteTestCase( + description="the summary replaces the syntax figures and other figures stay", + day="2026-11-16", + expected_figures={ + "SYNTAX_VERIFIED": "3,199", + "SYNTAX_FORMS": "3,351", + "SYNTAX_PCT": "95.46%", + "SYNTAX_BASIS": ( + "3,199 forms verified against cases recorded on Snowflake and approved " + "reviews, as of 16 November 2026." + ), + "DUCKSHOP_MODELS": "1,000", + }, + ), + ), + ids=lambda case: case.description, +) +def test_given_stale_figures_when_writing_then_the_summary_figures_replace_them( + test_case: WriteTestCase, tmp_path: Path +) -> None: + _, figures = written_files(tmp_path=tmp_path, day=test_case.day) + assert json.loads(figures.read_text()) == test_case.expected_figures + + +@pytest.mark.parametrize( + "test_case", + ( + CheckTestCase( + description="freshly written figures pass", + edits={}, + expected_return_code=0, + expected_fragment="", + ), + CheckTestCase( + description="a stale share fails", + edits={"SYNTAX_PCT": "71.26%"}, + expected_return_code=1, + expected_fragment="SYNTAX_PCT is '71.26%', but the summary gives '95.46%'", + ), + CheckTestCase( + description="a basis that states an older count fails", + edits={"SYNTAX_BASIS": "2,388 forms verified, as of 6 November 2026."}, + expected_return_code=1, + expected_fragment="SYNTAX_BASIS does not start with the verified count 3,199", + ), + ), + ids=lambda case: case.description, +) +def test_given_written_figures_when_checking_then_only_a_mismatch_fails( + test_case: CheckTestCase, tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + summary, figures = written_files(tmp_path=tmp_path, day="2026-11-16") + edited: dict[str, Any] = {**json.loads(figures.read_text()), **test_case.edits} + figures.write_text(json.dumps(edited)) + capsys.readouterr() + return_code: int = run_figures(["check", str(summary), "--figures", str(figures)]) + assert (return_code, test_case.expected_fragment in capsys.readouterr().err) == ( + test_case.expected_return_code, + True, + ) diff --git a/website/src/content/docs/docs/coverage.mdx b/website/src/content/docs/docs/coverage.mdx index 3643c9849..a7488ffe5 100644 --- a/website/src/content/docs/docs/coverage.mdx +++ b/website/src/content/docs/docs/coverage.mdx @@ -39,14 +39,15 @@ A form counts as verified only when it has at least one required obligation, eve obligation is satisfied by a case recorded on Snowflake, and an approved review of its family is current. Anything missing, ambiguous or stale counts as not verified. -Reproduce it from a checkout: +Reproduce it from a checkout. `scripts/evidence.sh path` fetches the pinned recorded cases and +prints where they are: ```bash -cargo run --release -p sqlquack-coverage-check -- summary +cargo run --release -p sqlquack-coverage-check -- summary --evidence "$(scripts/evidence.sh path)/syntax/snowflake" ``` -Its `checked 3479 forms` line counts the whole catalogue. 128 of those forms no longer appear on any -pinned documentation page, so the figure counts the remaining {figures.SYNTAX_FORMS}. +Its `checked … forms` line counts the whole catalogue, including forms that no longer appear on any +pinned documentation page. The figure counts the remaining {figures.SYNTAX_FORMS}. ## Rules diff --git a/website/src/data/figures.json b/website/src/data/figures.json index c31a34b62..74b1490cd 100644 --- a/website/src/data/figures.json +++ b/website/src/data/figures.json @@ -1,10 +1,10 @@ { "_comment": "Headline coverage figures: the only file to edit for them. The homepage, docs/coverage and llms.txt read it at build time. A value left as its own {{NAME}} token renders as a visible placeholder; `npm run build` warns, and `npm run check:figures` fails, while any is left. A value of null hides the figure: the homepage drops the block marked for it, and docs pages render it only where shown(NAME) allows; used anywhere else, it still renders as its {{NAME}} token. Each NAME_PCT also yields NAME_PCT_WHOLE, the share rounded to a whole percent, for the homepage.", - "_sources": "The syntax figures come from `cargo run --release -p sqlquack-coverage-check -- summary` on main, which counts the catalogue against the review records in coverage/syntax/reviews, not from coverage/syntax/baseline.json, which is bound to an older catalogue. SYNTAX_FORMS is that count's denominator and SYNTAX_PCT its verified share, with SYNTAX_VERIFIED the verified count. The translation figures are hidden (null) until their definition is decided.", - "SYNTAX_PCT": "71.26%", + "_sources": "The syntax figures come from the Snowflake coverage summary on main, `syntax-coverage summary --evidence /syntax/snowflake`, which counts the catalogue against the pinned cases and receipts and the review records in coverage/syntax/reviews. `make website-figures-update` writes SYNTAX_VERIFIED, SYNTAX_FORMS, SYNTAX_PCT and SYNTAX_BASIS from it, and `make website-figures`, part of check-ci, fails while they differ. SYNTAX_FORMS is that count's denominator and SYNTAX_PCT its verified share, with SYNTAX_VERIFIED the verified count. The translation figures are hidden (null) until their definition is decided.", + "SYNTAX_PCT": "95.46%", "SYNTAX_FORMS": "3,351", - "SYNTAX_VERIFIED": "2,388", - "SYNTAX_BASIS": "2,388 forms verified against cases recorded on Snowflake and approved reviews, as of 6 November 2026.", + "SYNTAX_VERIFIED": "3,199", + "SYNTAX_BASIS": "3,199 forms verified against cases recorded on Snowflake and approved reviews, as of 16 November 2026.", "TRANSLATION_PCT": null, "TRANSLATION_BASIS": null, "TRANSLATION_STATEMENTS": null