Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 10 additions & 2 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ PYTHON_TYPED := scripts tests/unit/scripts python/src $(PYTHON_API_TESTS)
PYTHON_TESTS := tests/unit/scripts $(PYTHON_API_TESTS)
endif

.PHONY: distribution-notices format rust-lint python-check fensu syntax-coverage depth-probe depth-probe-debug instruction-counts instruction-baseline gate-build engine-free bundled-extensions dialect-builds generate-cases generated-cases generate-syntax-cases syntax-obligations syntax-cases dialect-cases corpus-rerecord corpus-verdicts corpus-verdicts-baseline generate-referee-cases referee-cases referee-pending translation-files translation-results help pending check-syntax record-syntax check-convert record-convert sql-sizes sql-size-baseline sql-benchmark check-ci check-merge test test-rust test-python python-build python-module python-api python-wheel verify-recorded verify-changed replay-scope test-crate check-pr-metadata evidence-clone evidence-refresh evidence-pin evidence-pin-check clutter-guard census-duckshop fuzz-smoke gate gate-parallel census duckshop-compiled duckshop-measure
.PHONY: distribution-notices format rust-lint python-check fensu syntax-coverage website-figures website-figures-update depth-probe depth-probe-debug instruction-counts instruction-baseline gate-build engine-free bundled-extensions dialect-builds generate-cases generated-cases generate-syntax-cases syntax-obligations syntax-cases dialect-cases corpus-rerecord corpus-verdicts corpus-verdicts-baseline generate-referee-cases referee-cases referee-pending translation-files translation-results help pending check-syntax record-syntax check-convert record-convert sql-sizes sql-size-baseline sql-benchmark check-ci check-merge test test-rust test-python python-build python-module python-api python-wheel verify-recorded verify-changed replay-scope test-crate check-pr-metadata evidence-clone evidence-refresh evidence-pin evidence-pin-check clutter-guard census-duckshop fuzz-smoke gate gate-parallel census duckshop-compiled duckshop-measure

format:
cargo fmt --all
Expand Down Expand Up @@ -68,6 +68,14 @@ syntax-coverage: gate-build
$(MAKE) --no-print-directory check-syntax DIALECT=$$dialect REQUIRE_BASELINE=1; \
done

# Fails when the website's syntax figures (website/src/data/figures.json) differ from the Snowflake
# coverage summary; `website-figures-update` rewrites them from it.
SYNTAX_SUMMARY_FILE = target/syntax-summary.json
website-figures website-figures-update: gate-build
@mkdir -p target
$(GATE_DIR)/syntax-coverage summary --evidence $(call syntax_evidence_dir,snowflake) --json > $(SYNTAX_SUMMARY_FILE)
uv run python -m scripts.check_website_figures $(if $(filter website-figures-update,$@),write,check) $(SYNTAX_SUMMARY_FILE)

# Fails when a native parser crashes, panics or hangs on deeply nested SQL on a 2 MiB thread and
# coverage/deep-nesting.toml does not list it, or when a listed failure no longer happens.
depth-probe: gate-build
Expand Down Expand Up @@ -236,7 +244,7 @@ referee-pending:
uv run python -m scripts.write_referee_pending $(REFEREE_ARGS)

# The static gate, the first step of `gate`, and the Python API's tests.
check-ci: rust-lint python-check fensu syntax-coverage depth-probe instruction-counts corpus-verdicts evidence-pin-check clutter-guard engine-free bundled-extensions dialect-builds generated-cases syntax-obligations syntax-cases dialect-cases referee-cases translation-files python-api distribution-notices replay-scope ## The static gate CI runs.
check-ci: rust-lint python-check fensu syntax-coverage website-figures depth-probe instruction-counts corpus-verdicts evidence-pin-check clutter-guard engine-free bundled-extensions dialect-builds generated-cases syntax-obligations syntax-cases dialect-cases referee-cases translation-files python-api distribution-notices replay-scope ## The static gate CI runs.

# Each distribution ships the licence and third-party notices; its copies must match the root files.
distribution-notices:
Expand Down
14 changes: 14 additions & 0 deletions scripts/check_website_figures.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
"""Direct wrapper for the website figures."""

from __future__ import annotations

from scripts.website_figures.main.figures import run_figures


def main(argv: list[str] | None = None) -> int:
"""Write the website's syntax figures from the coverage summary, or fail when they differ."""
return run_figures(argv)


if __name__ == "__main__":
raise SystemExit(main())
5 changes: 4 additions & 1 deletion scripts/replay_scope/_helpers/decisions.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
from scripts.replay_scope._helpers.git import changed, tracked
from scripts.replay_scope.constants import (
BINARY_PACKAGE,
CHECKED_DOCUMENTATION_FILES,
DIRECTORY_SUFFIX,
DOCUMENTATION_PATHS,
FULL_REPLAY_PATHS,
Expand Down Expand Up @@ -40,7 +41,9 @@ def needs_code_checks(*, paths: list[str]) -> bool:


def documentation(*, path: str) -> bool:
"""Whether `path` is documentation: under a documentation path, or top-level Markdown."""
"""Whether `path` is documentation, other than the figures file check-ci compares."""
if path in CHECKED_DOCUMENTATION_FILES:
return False
top_level_markdown: bool = path.endswith(MARKDOWN_SUFFIX) and DIRECTORY_SUFFIX not in path
return top_level_markdown or under(path=path, prefixes=DOCUMENTATION_PATHS)

Expand Down
1 change: 1 addition & 0 deletions scripts/replay_scope/constants.py
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@
"tools/sqlquack-coverage-check/",
)
DOCUMENTATION_PATHS: tuple[str, ...] = ("website/", "docs/")
CHECKED_DOCUMENTATION_FILES: tuple[str, ...] = ("website/src/data/figures.json",)
SITE_PATHS: tuple[str, ...] = (
"website/",
".github/workflows/website-build.yml",
Expand Down
1 change: 1 addition & 0 deletions scripts/website_figures/__init__.py
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
"""The website's headline syntax figures, written from and checked against the coverage summary."""
Empty file.
32 changes: 32 additions & 0 deletions scripts/website_figures/_helpers/figures.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,32 @@
"""The figures a coverage summary gives the website, and how the written ones differ."""

from __future__ import annotations

from typing import Any

from scripts.website_figures.constants import BASIS_KEY, FORMS_KEY, PERCENT_KEY, VERIFIED_KEY


def summary_figures(summary: dict[str, Any]) -> dict[str, str]:
"""The verified count, the form count and the verified share, as the website writes them."""
total: dict[str, int] = summary["total"]
return {
VERIFIED_KEY: f"{total['verified']:,}",
FORMS_KEY: f"{total['forms']:,}",
PERCENT_KEY: f"{summary['verified_percent']:.2f}%",
}


def figure_problems(*, figures: dict[str, Any], expected: dict[str, str]) -> list[str]:
"""Each figure that differs from the summary, and a basis that does not state the count."""
problems: list[str] = [
f"{key} is {figures.get(key)!r}, but the summary gives {value!r}"
for key, value in expected.items()
if figures.get(key) != value
]
basis: object = figures.get(BASIS_KEY)
if not isinstance(basis, str) or not basis.startswith(f"{expected[VERIFIED_KEY]} "):
problems.append(
f"{BASIS_KEY} does not start with the verified count {expected[VERIFIED_KEY]}"
)
return problems
16 changes: 16 additions & 0 deletions scripts/website_figures/constants.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
"""Settings for keeping the website's syntax figures equal to the coverage summary."""

from __future__ import annotations

FIGURES_FILE: str = "website/src/data/figures.json"
VERIFIED_KEY: str = "SYNTAX_VERIFIED"
FORMS_KEY: str = "SYNTAX_FORMS"
PERCENT_KEY: str = "SYNTAX_PCT"
BASIS_KEY: str = "SYNTAX_BASIS"
BASIS_TEMPLATE: str = (
"{verified} forms verified against cases recorded on Snowflake and approved reviews, "
"as of {day}."
)
WRITE_COMMAND: str = "write"
CHECK_COMMAND: str = "check"
JSON_INDENT: str = "\t"
1 change: 1 addition & 0 deletions scripts/website_figures/main/__init__.py
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
"""Entry points for the website figures."""
56 changes: 56 additions & 0 deletions scripts/website_figures/main/figures.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,56 @@
"""Write the website's syntax figures from a coverage summary, or check that they match it."""

from __future__ import annotations

import argparse
import datetime
import json
import sys
from pathlib import Path
from typing import Any

from scripts.website_figures._helpers.figures import figure_problems, summary_figures
from scripts.website_figures.constants import (
BASIS_KEY,
BASIS_TEMPLATE,
CHECK_COMMAND,
FIGURES_FILE,
FORMS_KEY,
JSON_INDENT,
VERIFIED_KEY,
WRITE_COMMAND,
)


def run_figures(argv: list[str] | None = None) -> int:
"""Write or check the syntax figures against `syntax-coverage summary --json` output."""
parser: argparse.ArgumentParser = argparse.ArgumentParser(description=run_figures.__doc__)
parser.add_argument("command", choices=(WRITE_COMMAND, CHECK_COMMAND))
parser.add_argument("summary", type=Path, help="the JSON `syntax-coverage summary` printed")
parser.add_argument("--figures", type=Path, default=Path(FIGURES_FILE))
parser.add_argument("--day", type=datetime.date.fromisoformat, default=datetime.date.today())
arguments: argparse.Namespace = parser.parse_args(argv)
expected: dict[str, str] = summary_figures(json.loads(arguments.summary.read_text()))
figures: dict[str, Any] = json.loads(arguments.figures.read_text(encoding="utf-8"))
if arguments.command == WRITE_COMMAND:
day: datetime.date = arguments.day
basis: str = BASIS_TEMPLATE.format(
verified=expected[VERIFIED_KEY], day=f"{day.day} {day:%B %Y}"
)
figures.update({**expected, BASIS_KEY: basis})
text: str = json.dumps(figures, indent=JSON_INDENT, ensure_ascii=False) + "\n"
arguments.figures.write_text(text, encoding="utf-8")
print(f"website figures: wrote {basis}")
return 0
problems: list[str] = figure_problems(figures=figures, expected=expected)
for problem in problems:
print(f"website figures: {problem}", file=sys.stderr)
if problems:
print(
f"website figures: rewrite them with `make website-figures-update`, then commit "
f"{FIGURES_FILE}",
file=sys.stderr,
)
return 1
print(f"website figures: passed, {expected[VERIFIED_KEY]} of {expected[FORMS_KEY]} forms")
return 0
6 changes: 6 additions & 0 deletions tests/unit/scripts/replay_scope/test_scope.py
Original file line number Diff line number Diff line change
Expand Up @@ -99,6 +99,12 @@ def test_given_changed_paths_when_scoping_then_the_replay_matches_what_they_feed
expected_code=False,
expected_site=True,
),
ChecksTestCase(
description="the website figures need the code checks that compare them",
changed=("website/src/data/figures.json",),
expected_code=True,
expected_site=True,
),
ChecksTestCase(
description="the README needs neither the code checks nor the site",
changed=("README.md",),
Expand Down
1 change: 1 addition & 0 deletions tests/unit/scripts/website_figures/__init__.py
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
"""Website figure tests."""
21 changes: 21 additions & 0 deletions tests/unit/scripts/website_figures/_test_types.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
"""Test types for the website figures."""

from __future__ import annotations

from dataclasses import dataclass
from typing import Any


@dataclass(frozen=True)
class WriteTestCase:
description: str
day: str
expected_figures: dict[str, str]


@dataclass(frozen=True)
class CheckTestCase:
description: str
edits: dict[str, Any]
expected_return_code: int
expected_fragment: str
32 changes: 32 additions & 0 deletions tests/unit/scripts/website_figures/helpers.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,32 @@
"""A coverage summary and stale website figures, written into a temporary directory."""

from __future__ import annotations

import json
from pathlib import Path
from typing import Any

from scripts.website_figures.main.figures import run_figures

SUMMARY: dict[str, Any] = {
"total": {"forms": 3351, "verified": 3199},
"verified_percent": 95.4640405849,
}
STALE_FIGURES: dict[str, str] = {
"SYNTAX_VERIFIED": "2,388",
"SYNTAX_FORMS": "3,351",
"SYNTAX_PCT": "71.26%",
"SYNTAX_BASIS": "2,388 forms verified, as of 6 November 2026.",
"DUCKSHOP_MODELS": "1,000",
}


def written_files(*, tmp_path: Path, day: str) -> tuple[Path, Path]:
"""The summary file and the figures file after `write` replaced the stale figures."""
summary: Path = tmp_path / "summary.json"
figures: Path = tmp_path / "figures.json"
summary.write_text(json.dumps(SUMMARY))
figures.write_text(json.dumps(STALE_FIGURES))
written: int = run_figures(["write", str(summary), "--figures", str(figures), "--day", day])
assert written == 0
return summary, figures
78 changes: 78 additions & 0 deletions tests/unit/scripts/website_figures/test_figures.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,78 @@
"""Writing the website's syntax figures from a coverage summary, then checking them against it."""

from __future__ import annotations

import json
from pathlib import Path
from typing import Any

import pytest

from scripts.website_figures.main.figures import run_figures
from tests.unit.scripts.website_figures._test_types import CheckTestCase, WriteTestCase
from tests.unit.scripts.website_figures.helpers import written_files


@pytest.mark.parametrize(
"test_case",
(
WriteTestCase(
description="the summary replaces the syntax figures and other figures stay",
day="2026-11-16",
expected_figures={
"SYNTAX_VERIFIED": "3,199",
"SYNTAX_FORMS": "3,351",
"SYNTAX_PCT": "95.46%",
"SYNTAX_BASIS": (
"3,199 forms verified against cases recorded on Snowflake and approved "
"reviews, as of 16 November 2026."
),
"DUCKSHOP_MODELS": "1,000",
},
),
),
ids=lambda case: case.description,
)
def test_given_stale_figures_when_writing_then_the_summary_figures_replace_them(
test_case: WriteTestCase, tmp_path: Path
) -> None:
_, figures = written_files(tmp_path=tmp_path, day=test_case.day)
assert json.loads(figures.read_text()) == test_case.expected_figures


@pytest.mark.parametrize(
"test_case",
(
CheckTestCase(
description="freshly written figures pass",
edits={},
expected_return_code=0,
expected_fragment="",
),
CheckTestCase(
description="a stale share fails",
edits={"SYNTAX_PCT": "71.26%"},
expected_return_code=1,
expected_fragment="SYNTAX_PCT is '71.26%', but the summary gives '95.46%'",
),
CheckTestCase(
description="a basis that states an older count fails",
edits={"SYNTAX_BASIS": "2,388 forms verified, as of 6 November 2026."},
expected_return_code=1,
expected_fragment="SYNTAX_BASIS does not start with the verified count 3,199",
),
),
ids=lambda case: case.description,
)
def test_given_written_figures_when_checking_then_only_a_mismatch_fails(
test_case: CheckTestCase, tmp_path: Path, capsys: pytest.CaptureFixture[str]
) -> None:
summary, figures = written_files(tmp_path=tmp_path, day="2026-11-16")
edited: dict[str, Any] = {**json.loads(figures.read_text()), **test_case.edits}
figures.write_text(json.dumps(edited))
capsys.readouterr()
return_code: int = run_figures(["check", str(summary), "--figures", str(figures)])
assert (return_code, test_case.expected_fragment in capsys.readouterr().err) == (
test_case.expected_return_code,
True,
)
9 changes: 5 additions & 4 deletions website/src/content/docs/docs/coverage.mdx
Original file line number Diff line number Diff line change
Expand Up @@ -39,14 +39,15 @@ A form counts as verified only when it has at least one required obligation, eve
obligation is satisfied by a case recorded on Snowflake, and an approved review of its family is
current. Anything missing, ambiguous or stale counts as not verified.

Reproduce it from a checkout:
Reproduce it from a checkout. `scripts/evidence.sh path` fetches the pinned recorded cases and
prints where they are:

```bash
cargo run --release -p sqlquack-coverage-check -- summary
cargo run --release -p sqlquack-coverage-check -- summary --evidence "$(scripts/evidence.sh path)/syntax/snowflake"
```

Its `checked 3479 forms` line counts the whole catalogue. 128 of those forms no longer appear on any
pinned documentation page, so the figure counts the remaining {figures.SYNTAX_FORMS}.
Its `checked … forms` line counts the whole catalogue, including forms that no longer appear on any
pinned documentation page. The figure counts the remaining {figures.SYNTAX_FORMS}.

## Rules

Expand Down
8 changes: 4 additions & 4 deletions website/src/data/figures.json
Original file line number Diff line number Diff line change
@@ -1,10 +1,10 @@
{
"_comment": "Headline coverage figures: the only file to edit for them. The homepage, docs/coverage and llms.txt read it at build time. A value left as its own {{NAME}} token renders as a visible placeholder; `npm run build` warns, and `npm run check:figures` fails, while any is left. A value of null hides the figure: the homepage drops the block marked <!-- figure:NAME --> for it, and docs pages render it only where shown(NAME) allows; used anywhere else, it still renders as its {{NAME}} token. Each NAME_PCT also yields NAME_PCT_WHOLE, the share rounded to a whole percent, for the homepage.",
"_sources": "The syntax figures come from `cargo run --release -p sqlquack-coverage-check -- summary` on main, which counts the catalogue against the review records in coverage/syntax/reviews, not from coverage/syntax/baseline.json, which is bound to an older catalogue. SYNTAX_FORMS is that count's denominator and SYNTAX_PCT its verified share, with SYNTAX_VERIFIED the verified count. The translation figures are hidden (null) until their definition is decided.",
"SYNTAX_PCT": "71.26%",
"_sources": "The syntax figures come from the Snowflake coverage summary on main, `syntax-coverage summary --evidence <pinned evidence>/syntax/snowflake`, which counts the catalogue against the pinned cases and receipts and the review records in coverage/syntax/reviews. `make website-figures-update` writes SYNTAX_VERIFIED, SYNTAX_FORMS, SYNTAX_PCT and SYNTAX_BASIS from it, and `make website-figures`, part of check-ci, fails while they differ. SYNTAX_FORMS is that count's denominator and SYNTAX_PCT its verified share, with SYNTAX_VERIFIED the verified count. The translation figures are hidden (null) until their definition is decided.",
"SYNTAX_PCT": "95.46%",
"SYNTAX_FORMS": "3,351",
"SYNTAX_VERIFIED": "2,388",
"SYNTAX_BASIS": "2,388 forms verified against cases recorded on Snowflake and approved reviews, as of 6 November 2026.",
"SYNTAX_VERIFIED": "3,199",
"SYNTAX_BASIS": "3,199 forms verified against cases recorded on Snowflake and approved reviews, as of 16 November 2026.",
"TRANSLATION_PCT": null,
"TRANSLATION_BASIS": null,
"TRANSLATION_STATEMENTS": null
Expand Down
Loading