Skip to content

Commit e0eafb8

Browse files
committed
fix(website): show the current Snowflake syntax figure and fail CI when it drifts
The website still showed 2,388 of 3,351 forms from 6 November. The figures are now written from the Snowflake coverage summary, and check-ci fails while website/src/data/figures.json differs from it.
1 parent 69edbca commit e0eafb8

17 files changed

Lines changed: 282 additions & 11 deletions

File tree

‎Makefile‎

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@ PYTHON_TYPED := scripts tests/unit/scripts python/src $(PYTHON_API_TESTS)
3030
PYTHON_TESTS := tests/unit/scripts $(PYTHON_API_TESTS)
3131
endif
3232

33-
.PHONY: distribution-notices format rust-lint python-check fensu syntax-coverage depth-probe depth-probe-debug instruction-counts instruction-baseline gate-build engine-free bundled-extensions dialect-builds generate-cases generated-cases generate-syntax-cases syntax-obligations syntax-cases dialect-cases corpus-rerecord corpus-verdicts corpus-verdicts-baseline generate-referee-cases referee-cases referee-pending translation-files translation-results help pending check-syntax record-syntax check-convert record-convert sql-sizes sql-size-baseline sql-benchmark check-ci check-merge test test-rust test-python python-build python-module python-api python-wheel verify-recorded verify-changed replay-scope test-crate check-pr-metadata evidence-clone evidence-refresh evidence-pin evidence-pin-check clutter-guard census-duckshop fuzz-smoke gate gate-parallel census duckshop-compiled duckshop-measure
33+
.PHONY: distribution-notices format rust-lint python-check fensu syntax-coverage website-figures website-figures-update depth-probe depth-probe-debug instruction-counts instruction-baseline gate-build engine-free bundled-extensions dialect-builds generate-cases generated-cases generate-syntax-cases syntax-obligations syntax-cases dialect-cases corpus-rerecord corpus-verdicts corpus-verdicts-baseline generate-referee-cases referee-cases referee-pending translation-files translation-results help pending check-syntax record-syntax check-convert record-convert sql-sizes sql-size-baseline sql-benchmark check-ci check-merge test test-rust test-python python-build python-module python-api python-wheel verify-recorded verify-changed replay-scope test-crate check-pr-metadata evidence-clone evidence-refresh evidence-pin evidence-pin-check clutter-guard census-duckshop fuzz-smoke gate gate-parallel census duckshop-compiled duckshop-measure
3434

3535
format:
3636
cargo fmt --all
@@ -68,6 +68,14 @@ syntax-coverage: gate-build
6868
$(MAKE) --no-print-directory check-syntax DIALECT=$$dialect REQUIRE_BASELINE=1; \
6969
done
7070

71+
# Fails when the website's syntax figures (website/src/data/figures.json) differ from the Snowflake
72+
# coverage summary; `website-figures-update` rewrites them from it.
73+
SYNTAX_SUMMARY_FILE = target/syntax-summary.json
74+
website-figures website-figures-update: gate-build
75+
@mkdir -p target
76+
$(GATE_DIR)/syntax-coverage summary --evidence $(call syntax_evidence_dir,snowflake) --json > $(SYNTAX_SUMMARY_FILE)
77+
uv run python -m scripts.check_website_figures $(if $(filter website-figures-update,$@),write,check) $(SYNTAX_SUMMARY_FILE)
78+
7179
# Fails when a native parser crashes, panics or hangs on deeply nested SQL on a 2 MiB thread and
7280
# coverage/deep-nesting.toml does not list it, or when a listed failure no longer happens.
7381
depth-probe: gate-build
@@ -236,7 +244,7 @@ referee-pending:
236244
uv run python -m scripts.write_referee_pending $(REFEREE_ARGS)
237245

238246
# The static gate, the first step of `gate`, and the Python API's tests.
239-
check-ci: rust-lint python-check fensu syntax-coverage depth-probe instruction-counts corpus-verdicts evidence-pin-check clutter-guard engine-free bundled-extensions dialect-builds generated-cases syntax-obligations syntax-cases dialect-cases referee-cases translation-files python-api distribution-notices replay-scope ## The static gate CI runs.
247+
check-ci: rust-lint python-check fensu syntax-coverage website-figures depth-probe instruction-counts corpus-verdicts evidence-pin-check clutter-guard engine-free bundled-extensions dialect-builds generated-cases syntax-obligations syntax-cases dialect-cases referee-cases translation-files python-api distribution-notices replay-scope ## The static gate CI runs.
240248

241249
# Each distribution ships the licence and third-party notices; its copies must match the root files.
242250
distribution-notices:

‎scripts/check_website_figures.py‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
"""Direct wrapper for the website figures."""
2+
3+
from __future__ import annotations
4+
5+
from scripts.website_figures.main.figures import run_figures
6+
7+
8+
def main(argv: list[str] | None = None) -> int:
9+
"""Write the website's syntax figures from the coverage summary, or fail when they differ."""
10+
return run_figures(argv)
11+
12+
13+
if __name__ == "__main__":
14+
raise SystemExit(main())

‎scripts/replay_scope/_helpers/decisions.py‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@
99
from scripts.replay_scope._helpers.git import changed, tracked
1010
from scripts.replay_scope.constants import (
1111
BINARY_PACKAGE,
12+
CHECKED_DOCUMENTATION_FILES,
1213
DIRECTORY_SUFFIX,
1314
DOCUMENTATION_PATHS,
1415
FULL_REPLAY_PATHS,
@@ -40,7 +41,9 @@ def needs_code_checks(*, paths: list[str]) -> bool:
4041

4142

4243
def documentation(*, path: str) -> bool:
43-
"""Whether `path` is documentation: under a documentation path, or top-level Markdown."""
44+
"""Whether `path` is documentation, other than the figures file check-ci compares."""
45+
if path in CHECKED_DOCUMENTATION_FILES:
46+
return False
4447
top_level_markdown: bool = path.endswith(MARKDOWN_SUFFIX) and DIRECTORY_SUFFIX not in path
4548
return top_level_markdown or under(path=path, prefixes=DOCUMENTATION_PATHS)
4649

‎scripts/replay_scope/constants.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@
2828
"tools/sqlquack-coverage-check/",
2929
)
3030
DOCUMENTATION_PATHS: tuple[str, ...] = ("website/", "docs/")
31+
CHECKED_DOCUMENTATION_FILES: tuple[str, ...] = ("website/src/data/figures.json",)
3132
SITE_PATHS: tuple[str, ...] = (
3233
"website/",
3334
".github/workflows/website-build.yml",
Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
"""The website's headline syntax figures, written from and checked against the coverage summary."""

‎scripts/website_figures/_helpers/__init__.py‎

Whitespace-only changes.
Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
"""The figures a coverage summary gives the website, and how the written ones differ."""
2+
3+
from __future__ import annotations
4+
5+
from typing import Any
6+
7+
from scripts.website_figures.constants import BASIS_KEY, FORMS_KEY, PERCENT_KEY, VERIFIED_KEY
8+
9+
10+
def summary_figures(summary: dict[str, Any]) -> dict[str, str]:
11+
"""The verified count, the form count and the verified share, as the website writes them."""
12+
total: dict[str, int] = summary["total"]
13+
return {
14+
VERIFIED_KEY: f"{total['verified']:,}",
15+
FORMS_KEY: f"{total['forms']:,}",
16+
PERCENT_KEY: f"{summary['verified_percent']:.2f}%",
17+
}
18+
19+
20+
def figure_problems(*, figures: dict[str, Any], expected: dict[str, str]) -> list[str]:
21+
"""Each figure that differs from the summary, and a basis that does not state the count."""
22+
problems: list[str] = [
23+
f"{key} is {figures.get(key)!r}, but the summary gives {value!r}"
24+
for key, value in expected.items()
25+
if figures.get(key) != value
26+
]
27+
basis: object = figures.get(BASIS_KEY)
28+
if not isinstance(basis, str) or not basis.startswith(f"{expected[VERIFIED_KEY]} "):
29+
problems.append(
30+
f"{BASIS_KEY} does not start with the verified count {expected[VERIFIED_KEY]}"
31+
)
32+
return problems
Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
"""Settings for keeping the website's syntax figures equal to the coverage summary."""
2+
3+
from __future__ import annotations
4+
5+
FIGURES_FILE: str = "website/src/data/figures.json"
6+
VERIFIED_KEY: str = "SYNTAX_VERIFIED"
7+
FORMS_KEY: str = "SYNTAX_FORMS"
8+
PERCENT_KEY: str = "SYNTAX_PCT"
9+
BASIS_KEY: str = "SYNTAX_BASIS"
10+
BASIS_TEMPLATE: str = (
11+
"{verified} forms verified against cases recorded on Snowflake and approved reviews, "
12+
"as of {day}."
13+
)
14+
WRITE_COMMAND: str = "write"
15+
CHECK_COMMAND: str = "check"
16+
JSON_INDENT: str = "\t"
Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
"""Entry points for the website figures."""
Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,56 @@
1+
"""Write the website's syntax figures from a coverage summary, or check that they match it."""
2+
3+
from __future__ import annotations
4+
5+
import argparse
6+
import datetime
7+
import json
8+
import sys
9+
from pathlib import Path
10+
from typing import Any
11+
12+
from scripts.website_figures._helpers.figures import figure_problems, summary_figures
13+
from scripts.website_figures.constants import (
14+
BASIS_KEY,
15+
BASIS_TEMPLATE,
16+
CHECK_COMMAND,
17+
FIGURES_FILE,
18+
FORMS_KEY,
19+
JSON_INDENT,
20+
VERIFIED_KEY,
21+
WRITE_COMMAND,
22+
)
23+
24+
25+
def run_figures(argv: list[str] | None = None) -> int:
26+
"""Write or check the syntax figures against `syntax-coverage summary --json` output."""
27+
parser: argparse.ArgumentParser = argparse.ArgumentParser(description=run_figures.__doc__)
28+
parser.add_argument("command", choices=(WRITE_COMMAND, CHECK_COMMAND))
29+
parser.add_argument("summary", type=Path, help="the JSON `syntax-coverage summary` printed")
30+
parser.add_argument("--figures", type=Path, default=Path(FIGURES_FILE))
31+
parser.add_argument("--day", type=datetime.date.fromisoformat, default=datetime.date.today())
32+
arguments: argparse.Namespace = parser.parse_args(argv)
33+
expected: dict[str, str] = summary_figures(json.loads(arguments.summary.read_text()))
34+
figures: dict[str, Any] = json.loads(arguments.figures.read_text(encoding="utf-8"))
35+
if arguments.command == WRITE_COMMAND:
36+
day: datetime.date = arguments.day
37+
basis: str = BASIS_TEMPLATE.format(
38+
verified=expected[VERIFIED_KEY], day=f"{day.day} {day:%B %Y}"
39+
)
40+
figures.update({**expected, BASIS_KEY: basis})
41+
text: str = json.dumps(figures, indent=JSON_INDENT, ensure_ascii=False) + "\n"
42+
arguments.figures.write_text(text, encoding="utf-8")
43+
print(f"website figures: wrote {basis}")
44+
return 0
45+
problems: list[str] = figure_problems(figures=figures, expected=expected)
46+
for problem in problems:
47+
print(f"website figures: {problem}", file=sys.stderr)
48+
if problems:
49+
print(
50+
f"website figures: rewrite them with `make website-figures-update`, then commit "
51+
f"{FIGURES_FILE}",
52+
file=sys.stderr,
53+
)
54+
return 1
55+
print(f"website figures: passed, {expected[VERIFIED_KEY]} of {expected[FORMS_KEY]} forms")
56+
return 0

0 commit comments

Comments
 (0)