From 4690fe43d69b7a7a7626c4206e85bb56da1db8f7 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 12 Nov 2026 22:01:04 +0000 Subject: [PATCH 1/3] chore(syntax-coverage): withdraw the DDL rejections the corrected documented cases also accept Correcting the documented Iceberg PARTITION BY and semantic view REFERENCES rows made 21 rejection obligations render SQL an accepted row also renders, but the DDL obligations were not rebuilt. Rebuilding them withdraws those 21; none was satisfied, so coverage is unchanged. --- coverage/syntax/obligations/ddl.toml | 168 ------------------------ coverage/syntax/withdrawn/ddl.toml | 189 +++++++++++++++++++++++++++ 2 files changed, 189 insertions(+), 168 deletions(-) diff --git a/coverage/syntax/obligations/ddl.toml b/coverage/syntax/obligations/ddl.toml index 20cb1102c..452105f80 100644 --- a/coverage/syntax/obligations/ddl.toml +++ b/coverage/syntax/obligations/ddl.toml @@ -176060,158 +176060,6 @@ required = true anchor = "doc.sql.create-iceberg-table / Syntax" note = "a string literal in place of (DAY ( 'v1' )), observed" -[[obligation]] -id = "sf.ddl.create-iceberg-table.table.iceberg-rest-in-a-catalog-linked-database.o299" -kind = "option_present" -required = true -target = "[ PARTITION BY ( { | BUCKET ( , ) | TRUNCATE ( , ) | YEAR (…" -anchor = "doc.sql.create-iceberg-table / Syntax / External Iceberg catalog / Iceberg REST in a catalog-linked database" -note = "option present [ PARTITION BY ( { | BUCKET ( , ) | TRUNCATE ( , ) | YEAR ( Date: Thu, 12 Nov 2026 22:17:49 +0000 Subject: [PATCH 2/3] ci(syntax-coverage): fail the gate when Snowflake obligations are stale The Snowflake obligations builder had no --check, so a limits.toml edit that changed a family's withdrawals could land without regenerating them. The builder now takes --check, which writes nothing and fails on each obligations, withdrawn or elisions file that differs from what it would write, and --since, which builds only the families whose catalogue, rows in limits.toml, generated files or builder changed since a revision. make syntax-obligations runs both against EVIDENCE_BASE as part of check-ci, and passes when no family changed. --- Makefile | 15 +- coverage/syntax/README.md | 6 + .../grammar_catalogue/_helpers/toml_out.py | 7 +- .../grammar_obligations/_helpers/elisions.py | 18 +- .../_helpers/__init__.py | 0 .../_helpers/changed.py | 84 ++++++++ .../syntax_obligations_snowflake/constants.py | 16 ++ .../main/build.py | 93 ++++++--- .../syntax_obligations_snowflake/__init__.py | 0 .../_test_types.py | 64 ++++++ .../syntax_obligations_snowflake/helpers.py | 128 ++++++++++++ .../test_changed.py | 190 ++++++++++++++++++ .../test_check.py | 177 ++++++++++++++++ 13 files changed, 754 insertions(+), 44 deletions(-) create mode 100644 scripts/syntax_obligations_snowflake/_helpers/__init__.py create mode 100644 scripts/syntax_obligations_snowflake/_helpers/changed.py create mode 100644 tests/unit/scripts/syntax_obligations_snowflake/__init__.py create mode 100644 tests/unit/scripts/syntax_obligations_snowflake/_test_types.py create mode 100644 tests/unit/scripts/syntax_obligations_snowflake/helpers.py create mode 100644 tests/unit/scripts/syntax_obligations_snowflake/test_changed.py create mode 100644 tests/unit/scripts/syntax_obligations_snowflake/test_check.py diff --git a/Makefile b/Makefile index 8cc31f4d3..82f0ac0aa 100644 --- a/Makefile +++ b/Makefile @@ -30,7 +30,7 @@ PYTHON_TYPED := scripts tests/unit/scripts python/src $(PYTHON_API_TESTS) PYTHON_TESTS := tests/unit/scripts $(PYTHON_API_TESTS) endif -.PHONY: distribution-notices format rust-lint python-check fensu syntax-coverage depth-probe depth-probe-debug instruction-counts instruction-baseline gate-build engine-free bundled-extensions dialect-builds generate-cases generated-cases generate-syntax-cases syntax-cases dialect-cases corpus-rerecord corpus-verdicts corpus-verdicts-baseline generate-referee-cases referee-cases referee-pending translation-files translation-results help pending check-syntax record-syntax check-convert record-convert sql-sizes sql-size-baseline sql-benchmark check-ci check-merge test test-rust test-python python-build python-module python-api python-wheel verify-recorded verify-changed replay-scope test-crate check-pr-metadata evidence-clone evidence-refresh evidence-pin evidence-pin-check clutter-guard census-duckshop fuzz-smoke gate gate-parallel census duckshop-compiled duckshop-measure +.PHONY: distribution-notices format rust-lint python-check fensu syntax-coverage depth-probe depth-probe-debug instruction-counts instruction-baseline gate-build engine-free bundled-extensions dialect-builds generate-cases generated-cases generate-syntax-cases syntax-obligations syntax-cases dialect-cases corpus-rerecord corpus-verdicts corpus-verdicts-baseline generate-referee-cases referee-cases referee-pending translation-files translation-results help pending check-syntax record-syntax check-convert record-convert sql-sizes sql-size-baseline sql-benchmark check-ci check-merge test test-rust test-python python-build python-module python-api python-wheel verify-recorded verify-changed replay-scope test-crate check-pr-metadata evidence-clone evidence-refresh evidence-pin evidence-pin-check clutter-guard census-duckshop fuzz-smoke gate gate-parallel census duckshop-compiled duckshop-measure format: cargo fmt --all @@ -183,10 +183,11 @@ generated-cases: # dialect's suite folder until then. syntax_evidence_dir = $(if $(EVIDENCE_PINNED),$(EVIDENCE_ROOT)/syntax/$(1),$(call suite_dir,syntax,$(1))) -# The syntax-coverage families whose cases and marks scripts/build_syntax_cases.py writes. +# The Snowflake syntax-coverage families, whose obligations, cases and marks the builders write. +SYNTAX_FAMILIES = classes ddl dml expressions functions query scripting security_admin transaction_session utility SYNTAX_CASE_ARGS = --evidence $(call syntax_evidence_dir,snowflake) \ --receipts $(call syntax_evidence_dir,snowflake)/receipts --held coverage/syntax/held.jsonl \ - classes ddl dml expressions functions query scripting security_admin transaction_session utility + $(SYNTAX_FAMILIES) # Rewrites the Snowflake syntax cases, marks and arity from the obligations and the recorded receipts. # Once the evidence is pinned the cases go to the clone's working tree, to commit there. @@ -194,6 +195,12 @@ generate-syntax-cases: EVIDENCE = working generate-syntax-cases: uv run python -m scripts.build_syntax_cases $(SYNTAX_CASE_ARGS) +# Fails when a Snowflake obligations, withdrawn or elisions file differs from what the builder writes. +# Only the families whose inputs changed since EVIDENCE_BASE are rebuilt (every family when the +# builder changed); with none changed the target prints that it skipped them all and passes. +syntax-obligations: + uv run python -m scripts.build_syntax_obligations_snowflake --check --since $(EVIDENCE_BASE) $(SYNTAX_FAMILIES) + # Fails when a syntax case, mark or arity file differs from what the committed receipts regenerate. syntax-cases: uv run python -m scripts.build_syntax_cases --check $(SYNTAX_CASE_ARGS) @@ -225,7 +232,7 @@ referee-pending: uv run python -m scripts.write_referee_pending $(REFEREE_ARGS) # The static gate, the first step of `gate`, and the Python API's tests. -check-ci: rust-lint python-check fensu syntax-coverage depth-probe instruction-counts corpus-verdicts evidence-pin-check clutter-guard engine-free bundled-extensions dialect-builds generated-cases syntax-cases dialect-cases referee-cases translation-files python-api distribution-notices replay-scope ## The static gate CI runs. +check-ci: rust-lint python-check fensu syntax-coverage depth-probe instruction-counts corpus-verdicts evidence-pin-check clutter-guard engine-free bundled-extensions dialect-builds generated-cases syntax-obligations syntax-cases dialect-cases referee-cases translation-files python-api distribution-notices replay-scope ## The static gate CI runs. # Each distribution ships the licence and third-party notices; its copies must match the root files. distribution-notices: diff --git a/coverage/syntax/README.md b/coverage/syntax/README.md index f2f9e8470..fff295974 100644 --- a/coverage/syntax/README.md +++ b/coverage/syntax/README.md @@ -39,8 +39,14 @@ Families: `query`, `dml`, `ddl`, `transaction_session`, `security_admin`, `utili ```sh python -m scripts.fetch_syntax_docs_snowflake CACHE CACHE/idx/* # private cache, never committed python -m scripts.build_syntax_catalogue_snowflake --cache CACHE --out coverage/syntax +python -m scripts.build_syntax_obligations_snowflake ddl query # obligations, withdrawn, elisions ``` +Rebuild a family's obligations whenever its catalogue, its rows in `limits.toml` or the builder +changes. `make syntax-obligations`, part of `make check-ci`, fails when a committed obligations, +withdrawn or elisions file differs from what the builder writes; it rebuilds only the families +whose inputs or generated files changed since `EVIDENCE_BASE`, and passes when none did. + Pages with no Markdown export (the per-category command hubs) are fetched as HTML and catalogue no form. A rebuild keeps every existing id whose (page, grammar) is unchanged. A form whose grammar changed keeps its id when its natural id is unchanged or when all its earlier signatures diff --git a/scripts/grammar_catalogue/_helpers/toml_out.py b/scripts/grammar_catalogue/_helpers/toml_out.py index f2dfa4a98..d6c7d075f 100644 --- a/scripts/grammar_catalogue/_helpers/toml_out.py +++ b/scripts/grammar_catalogue/_helpers/toml_out.py @@ -28,8 +28,13 @@ def row_lines(*, table: str, row: Row) -> list[str]: def write_tables(*, path: Path, header: str, tables: list[tuple[str, list[Row]]]) -> None: """Write a header comment and the given arrays of tables.""" + path.write_text(tables_text(header=header, tables=tables)) + + +def tables_text(*, header: str, tables: list[tuple[str, list[Row]]]) -> str: + """A header comment and the given arrays of tables, as write_tables writes them.""" lines: list[str] = [header.rstrip(), ""] for table, rows in tables: for row in rows: lines += row_lines(table=table, row=row) - path.write_text("\n".join(lines)) + return "\n".join(lines) diff --git a/scripts/grammar_obligations/_helpers/elisions.py b/scripts/grammar_obligations/_helpers/elisions.py index e63c50ac7..4b4155a91 100644 --- a/scripts/grammar_obligations/_helpers/elisions.py +++ b/scripts/grammar_obligations/_helpers/elisions.py @@ -6,7 +6,7 @@ from collections.abc import Mapping from pathlib import Path -from scripts.grammar_catalogue._helpers.toml_out import write_tables +from scripts.grammar_catalogue._helpers.toml_out import tables_text from scripts.grammar_obligations.constants import ( ELISION_START, FAMILY_ELISIONS, @@ -66,10 +66,12 @@ def _is_repetition_branch(branch: tuple[Node, ...]) -> bool: return TOKEN_ELISION in kinds and TOKEN_KEYWORD not in kinds and len(branch) > 1 -def write_elisions(*, root: Path, family: str, trees: dict[str, list[tuple[Node, ...]]]) -> None: - """Write elisions/.toml: each elision, its fill and the forms it defers to.""" +def elision_outputs( + *, root: Path, family: str, trees: dict[str, list[tuple[Node, ...]]] +) -> list[tuple[Path, str | None]]: + """elisions/.toml and its text (None: no file); nothing for a family without rules.""" if family not in FAMILY_ELISIONS: - return + return [] rules: dict[str, tuple[str, tuple[str, ...], str]] = FAMILY_ELISIONS[family] catalogued: set[str] = _catalogued(root=root) rows: list[Row] = [] @@ -94,16 +96,14 @@ def write_elisions(*, root: Path, family: str, trees: dict[str, list[tuple[Node, ) path: Path = root / "elisions" / f"{family}.toml" if not rows: - path.unlink(missing_ok=True) - return - path.parent.mkdir(parents=True, exist_ok=True) - write_tables( - path=path, + return [(path, None)] + text: str = tables_text( header=f"# Elisions (`...`) in the grammar of family {family}: what each renders as and " "the catalogued\n# forms that carry the elided syntax; generated by " "scripts/build_syntax_obligations.py.", tables=[("elision", rows)], ) + return [(path, text)] def _catalogued(*, root: Path) -> set[str]: diff --git a/scripts/syntax_obligations_snowflake/_helpers/__init__.py b/scripts/syntax_obligations_snowflake/_helpers/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/scripts/syntax_obligations_snowflake/_helpers/changed.py b/scripts/syntax_obligations_snowflake/_helpers/changed.py new file mode 100644 index 000000000..d6853ba2b --- /dev/null +++ b/scripts/syntax_obligations_snowflake/_helpers/changed.py @@ -0,0 +1,84 @@ +"""The families whose generated obligations can differ since a base revision.""" + +from __future__ import annotations + +import subprocess +import tomllib +from pathlib import Path + +from scripts.grammar_obligations.constants import FAMILY_ELISIONS +from scripts.grammar_obligations.types import Row +from scripts.replay_scope._helpers.git import changed +from scripts.syntax_obligations_snowflake.constants import ( + BUILDER_PATHS, + CATALOGUE_DIR, + FAMILY_DIRS, + FAMILY_FILE_DEPTH, + FAMILY_FILE_SUFFIX, + LIMITS_FILE, + LIMITS_TABLES, +) + + +def changed_families(*, root: Path, base: str, families: list[str]) -> list[str]: + """The named families whose catalogue, limits rows, generated files or builder changed.""" + top: Path = Path(_git(cwd=root, arguments=("rev-parse", "--show-toplevel")).strip()) + prefix: str = root.resolve().relative_to(top.resolve()).as_posix() + "/" + paths: list[str] = changed(root=top, base=base) + if any(path.startswith(BUILDER_PATHS) for path in paths): + return list(families) + selected: set[str] = set() + for path in paths: + selected |= ( + _file_families(relative=path.removeprefix(prefix)) if path.startswith(prefix) else set() + ) + if prefix + LIMITS_FILE in paths: + selected |= _limits_families(root=root, top=top, base=base, relative=prefix + LIMITS_FILE) + return [family for family in families if family in selected] + + +def _file_families(*, relative: str) -> set[str]: + """The families a changed file under the root feeds: its own, plus the elisions' readers.""" + parts: list[str] = relative.split("/") + if ( + len(parts) != FAMILY_FILE_DEPTH + or parts[0] not in FAMILY_DIRS + or not parts[1].endswith(FAMILY_FILE_SUFFIX) + ): + return set() + family: set[str] = {parts[1].removesuffix(FAMILY_FILE_SUFFIX)} + return family | FAMILY_ELISIONS.keys() if parts[0] == CATALOGUE_DIR else family + + +def _limits_families(*, root: Path, top: Path, base: str, relative: str) -> set[str]: + """The families whose rows differ between the merge base's limits.toml and the root's.""" + merge_base: str = _git(cwd=top, arguments=("merge-base", base, "HEAD")).strip() + before: dict[str, list[tuple[str, Row]]] = _limits_by_family( + _git(cwd=top, arguments=("show", f"{merge_base}:{relative}"), missing_ok=True) + ) + path: Path = root / LIMITS_FILE + after: dict[str, list[tuple[str, Row]]] = _limits_by_family( + path.read_text() if path.exists() else "" + ) + return { + family for family in before.keys() | after.keys() if before.get(family) != after.get(family) + } + + +def _limits_by_family(text: str) -> dict[str, list[tuple[str, Row]]]: + """limits.toml's rows grouped by the family their form id names, in file order per table.""" + grouped: dict[str, list[tuple[str, Row]]] = {} + document: dict[str, list[Row]] = tomllib.loads(text) + for table in LIMITS_TABLES: + for row in document.get(table, []): + family: str = str(row["form"]).split(".")[1] + grouped.setdefault(family, []).append((table, row)) + return grouped + + +def _git(*, cwd: Path, arguments: tuple[str, ...], missing_ok: bool = False) -> str: + """A Git command's output; with missing_ok, empty when it fails, as for a file not there.""" + result: subprocess.CompletedProcess[str] = subprocess.run( + ("git", *arguments), cwd=cwd, check=not missing_ok, capture_output=True, text=True + ) + return result.stdout if result.returncode == 0 else "" diff --git a/scripts/syntax_obligations_snowflake/constants.py b/scripts/syntax_obligations_snowflake/constants.py index 1a30a931f..3d343ab7a 100644 --- a/scripts/syntax_obligations_snowflake/constants.py +++ b/scripts/syntax_obligations_snowflake/constants.py @@ -2,6 +2,8 @@ from __future__ import annotations +from pathlib import Path + WITHDRAWN_HEADER: str = ( "# Derived rejections of family {family} withdrawn because the documented grammar also\n" "# renders their SQL as valid; generated by scripts/build_syntax_obligations.py." @@ -11,3 +13,17 @@ "\n# Each [[withdrawn_class]] withdraws one class of derived negatives by decision, once," " with its count." ) + +DEFAULT_ROOT: Path = Path("coverage/syntax") +BUILDER_PATHS: tuple[str, ...] = ( + "scripts/build_syntax_obligations_snowflake.py", + "scripts/syntax_obligations_snowflake/", + "scripts/grammar_obligations/", + "scripts/grammar_catalogue/", +) +CATALOGUE_DIR: str = "catalogue" +FAMILY_DIRS: tuple[str, ...] = (CATALOGUE_DIR, "obligations", "withdrawn", "elisions") +FAMILY_FILE_SUFFIX: str = ".toml" +FAMILY_FILE_DEPTH: int = 2 +LIMITS_FILE: str = "limits.toml" +LIMITS_TABLES: tuple[str, ...] = ("limit", "suppress", "reading") diff --git a/scripts/syntax_obligations_snowflake/main/build.py b/scripts/syntax_obligations_snowflake/main/build.py index f02bddec3..fd9b024e5 100644 --- a/scripts/syntax_obligations_snowflake/main/build.py +++ b/scripts/syntax_obligations_snowflake/main/build.py @@ -1,49 +1,82 @@ -"""Write coverage/syntax/obligations/.toml from the catalogue grammar and limits.toml.""" +"""Write, or with --check compare, the Snowflake obligations, withdrawn and elisions files.""" from __future__ import annotations import argparse +import sys from pathlib import Path -from scripts.grammar_catalogue._helpers.toml_out import write_tables -from scripts.grammar_obligations._helpers.elisions import write_elisions +from scripts.grammar_catalogue._helpers.toml_out import tables_text +from scripts.grammar_obligations._helpers.cases import sync_outputs +from scripts.grammar_obligations._helpers.elisions import elision_outputs from scripts.grammar_obligations._helpers.obligations import family_obligations, obligation_rows -from scripts.grammar_obligations.types import Row +from scripts.syntax_obligations_snowflake._helpers.changed import changed_families from scripts.syntax_obligations_snowflake.constants import ( + DEFAULT_ROOT, WITHDRAWN_CLASS_HEADER, WITHDRAWN_HEADER, ) def run_build(argv: list[str] | None = None) -> int: - """Regenerate the obligations of the named families.""" + """Regenerate the obligations of the named families, or with --check compare them.""" parser: argparse.ArgumentParser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--root", type=Path, default=Path("coverage/syntax")) + parser.add_argument("--root", type=Path, default=DEFAULT_ROOT) + parser.add_argument( + "--check", action="store_true", help="write nothing; fail when a committed file differs" + ) + parser.add_argument( + "--since", + metavar="REVISION", + help="build only the families whose inputs changed since the merge base with REVISION", + ) parser.add_argument("families", nargs="+") arguments: argparse.Namespace = parser.parse_args(argv) root: Path = arguments.root - for family in arguments.families: - path: Path = root / "obligations" / f"{family}.toml" - numbered, trees, withdrawn, classes, _ = family_obligations(root=root, family=family) - write_elisions(root=root, family=family, trees=trees) - rows: list[Row] = obligation_rows(numbered) - path.parent.mkdir(parents=True, exist_ok=True) - write_tables( - path=path, - header=f"# Obligations of family {family}; generated by " - "scripts/build_syntax_obligations.py from the catalogue grammar and limits.toml.\n" - "# See docs/syntax-coverage.md.", - tables=[("obligation", rows)], + families: list[str] = arguments.families + if arguments.since: + families = changed_families(root=root, base=arguments.since, families=families) + skipped: list[str] = [f for f in arguments.families if f not in families] + print( + f"syntax obligations: {len(families)} families changed since {arguments.since}" + f"{': ' + ' '.join(families) if families else ''}; skipped {len(skipped)}" ) - withdrawn_path: Path = root / "withdrawn" / f"{family}.toml" - if withdrawn or classes: - withdrawn_path.parent.mkdir(parents=True, exist_ok=True) - write_tables( - path=withdrawn_path, - header=WITHDRAWN_HEADER.format(family=family) - + (WITHDRAWN_CLASS_HEADER if classes else ""), - tables=[("withdrawn_class", classes), ("withdrawn", withdrawn)], - ) - elif withdrawn_path.exists(): - withdrawn_path.unlink() - return 0 + differing: list[Path] = [] + for family in families: + differing += sync_outputs( + outputs=_family_outputs(root=root, family=family), check=arguments.check + ) + if not arguments.check: + return 0 + for path in differing: + print(f"stale: {path} differs from what the build writes", file=sys.stderr) + print( + f"syntax obligations: {'FAILED' if differing else 'passed'}, {len(differing)} stale files;" + " run without --check to regenerate" + ) + return 1 if differing else 0 + + +def _family_outputs(*, root: Path, family: str) -> list[tuple[Path, str | None]]: + """The text of each file the build writes for one family (None: no file).""" + numbered, trees, withdrawn, classes, _ = family_obligations(root=root, family=family) + obligations: str = tables_text( + header=f"# Obligations of family {family}; generated by " + "scripts/build_syntax_obligations.py from the catalogue grammar and limits.toml.\n" + "# See docs/syntax-coverage.md.", + tables=[("obligation", obligation_rows(numbered))], + ) + withdrawals: str | None = ( + tables_text( + header=WITHDRAWN_HEADER.format(family=family) + + (WITHDRAWN_CLASS_HEADER if classes else ""), + tables=[("withdrawn_class", classes), ("withdrawn", withdrawn)], + ) + if withdrawn or classes + else None + ) + return [ + *elision_outputs(root=root, family=family, trees=trees), + (root / "obligations" / f"{family}.toml", obligations), + (root / "withdrawn" / f"{family}.toml", withdrawals), + ] diff --git a/tests/unit/scripts/syntax_obligations_snowflake/__init__.py b/tests/unit/scripts/syntax_obligations_snowflake/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/unit/scripts/syntax_obligations_snowflake/_test_types.py b/tests/unit/scripts/syntax_obligations_snowflake/_test_types.py new file mode 100644 index 000000000..75bfbb133 --- /dev/null +++ b/tests/unit/scripts/syntax_obligations_snowflake/_test_types.py @@ -0,0 +1,64 @@ +"""Test types for the Snowflake obligation build's --check and its family selection.""" + +from __future__ import annotations + +from dataclasses import dataclass + + +@dataclass(frozen=True) +class CheckTestCase: + description: str + family: str + relative: str + old: str + new: str + expected_code: int + expected_stale: tuple[str, ...] + + +@dataclass(frozen=True) +class MissingFileTestCase: + description: str + family: str + relative: str + expected_code: int + + +@dataclass(frozen=True) +class SelectionTestCase: + description: str + relative: str + old: str + new: str + expected_families: tuple[str, ...] + + +@dataclass(frozen=True) +class SkipTestCase: + description: str + families: tuple[str, ...] + expected_lines: tuple[str, ...] + + +@dataclass(frozen=True) +class ClosureTestCase: + description: str + module: str + selection_only: tuple[str, ...] + expected_outside: tuple[str, ...] + + +@dataclass(frozen=True) +class BranchTestCase: + description: str + branch_edit: tuple[str, str, str] + main_edit: tuple[str, str, str] + expected_families: tuple[str, ...] + + +@dataclass(frozen=True) +class StrayElisionTestCase: + description: str + arguments: tuple[str, ...] + expected_code: int + expected_text: str diff --git a/tests/unit/scripts/syntax_obligations_snowflake/helpers.py b/tests/unit/scripts/syntax_obligations_snowflake/helpers.py new file mode 100644 index 000000000..9851b21d1 --- /dev/null +++ b/tests/unit/scripts/syntax_obligations_snowflake/helpers.py @@ -0,0 +1,128 @@ +"""Small roots and repositories for the Snowflake obligation build's tests.""" + +from __future__ import annotations + +import subprocess +import sys +from itertools import filterfalse +from pathlib import Path + +from scripts.syntax_obligations_snowflake.main.build import run_build + +REPOSITORY: Path = Path(__file__).parents[4] +FAMILY: str = "transaction_session" +CATALOGUE: str = """ +[[form]] +id = "sf.transaction_session.use-database" +family = "transaction_session" +kind = "statement" +grammar = ["USE [ DATABASE ] "] +uses = [] +status = "catalogued" + +[[form]] +id = "sf.transaction_session.use-role" +family = "transaction_session" +kind = "statement" +grammar = ["USE ROLE "] +uses = [] +status = "catalogued" +""" +WITHDRAWN_ROW: str = 'form = "sf.transaction_session.use-role"' +ELISION_FAMILY: str = "expressions" +ELISION_CATALOGUE: str = """ +[[form]] +id = "sf.expressions.commands-model-function.model-default-version" +family = "expressions" +kind = "statement" +grammar = ["SELECT MODEL( )!( ... ) FROM "] +uses = [] +status = "catalogued" +""" +CATALOGUES: dict[str, str] = {FAMILY: CATALOGUE, ELISION_FAMILY: ELISION_CATALOGUE} +SELECTION_ROOT: str = "coverage/syntax" +SELECTION_FAMILIES: tuple[str, ...] = ("ddl", "expressions", "query", "utility") +SELECTION_LIMITS: str = """[[limit]] +form = "sf.ddl.create-table" +kind = "negative" +note = "a ddl limit" + +[[suppress]] +form = "sf.query.select" +kind = "positive" +reason = "a query suppression" +""" +BUILDER_MODULE: str = "scripts/grammar_obligations/x.py" +LOADED_MODULES: str = ( + "import importlib, sys\n" + "importlib.import_module(sys.argv[1])\n" + "print('\\n'.join(m.__file__ for n, m in sys.modules.items() if n.split('.')[0] == 'scripts'))" +) + + +def built_root(*, directory: Path, family: str) -> Path: + """A root holding one family's small catalogue, built once, so its files are as built.""" + root: Path = directory / "root" + (root / "catalogue").mkdir(parents=True) + (root / "catalogue" / f"{family}.toml").write_text(CATALOGUES[family]) + (root / "limits.toml").write_text("") + assert run_build(["--root", str(root), family]) == 0 + return root + + +def committed_repository(directory: Path) -> Path: + """A committed repository with each selection family's files, another dialect's and a module.""" + files: dict[str, str] = { + f"{SELECTION_ROOT}/limits.toml": SELECTION_LIMITS, + BUILDER_MODULE: "X = 0\n", + } + for part in ("catalogue", "obligations", "withdrawn"): + files |= _family_files(prefix=f"{SELECTION_ROOT}/{part}", part=part) + files |= _family_files(prefix=f"coverage/syntax-databricks/{part}", part=part) + for relative, text in files.items(): + (directory / relative).parent.mkdir(parents=True, exist_ok=True) + (directory / relative).write_text(text) + _git(directory, "init", "-q", "-b", "main") + commit(repository=directory, message="base") + return directory + + +def edit(*, repository: Path, relative: str, old: str, new: str) -> None: + """Replace the first old with new in one file of the repository.""" + path: Path = repository / relative + path.write_text(path.read_text().replace(old, new, 1)) + + +def commit(*, repository: Path, message: str) -> None: + """Commit every change in the repository.""" + _git(repository, "add", "-A") + _git( + repository, "-c", "user.name=t", "-c", "user.email=t@example.com", "commit", "-qm", message + ) + + +def checkout(*, repository: Path, arguments: tuple[str, ...]) -> None: + """Run git checkout with arguments, as to create or switch a branch.""" + _git(repository, "checkout", "-q", *arguments) + + +def scripts_outside(*, module: str, allowed: tuple[str, ...]) -> tuple[str, ...]: + """The files of scripts modules a fresh import of module loads that no allowed prefix covers.""" + output: str = subprocess.run( + (sys.executable, "-c", LOADED_MODULES, module), + cwd=REPOSITORY, + check=True, + capture_output=True, + text=True, + ).stdout + loaded: list[str] = [Path(line).relative_to(REPOSITORY).as_posix() for line in output.split()] + assert loaded + return tuple(filterfalse(lambda path: path.startswith(allowed), loaded)) + + +def _family_files(*, prefix: str, part: str) -> dict[str, str]: + return {f"{prefix}/{family}.toml": f"# {part} {family}\n" for family in SELECTION_FAMILIES} + + +def _git(repository: Path, *arguments: str) -> None: + subprocess.run(("git", *arguments), cwd=repository, check=True, capture_output=True) diff --git a/tests/unit/scripts/syntax_obligations_snowflake/test_changed.py b/tests/unit/scripts/syntax_obligations_snowflake/test_changed.py new file mode 100644 index 000000000..da652930b --- /dev/null +++ b/tests/unit/scripts/syntax_obligations_snowflake/test_changed.py @@ -0,0 +1,190 @@ +"""--since selects only the families whose inputs or generated files changed.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from scripts.syntax_obligations_snowflake._helpers.changed import changed_families +from scripts.syntax_obligations_snowflake.constants import BUILDER_PATHS +from scripts.syntax_obligations_snowflake.main.build import run_build +from tests.unit.scripts.syntax_obligations_snowflake._test_types import ( + BranchTestCase, + ClosureTestCase, + SelectionTestCase, + SkipTestCase, +) +from tests.unit.scripts.syntax_obligations_snowflake.helpers import ( + BUILDER_MODULE, + SELECTION_FAMILIES, + SELECTION_ROOT, + checkout, + commit, + committed_repository, + edit, + scripts_outside, +) + + +@pytest.mark.parametrize( + "test_case", + ( + SelectionTestCase( + description="nothing changed", + relative=f"{SELECTION_ROOT}/limits.toml", + old="a ddl limit", + new="a ddl limit", + expected_families=(), + ), + SelectionTestCase( + description="a ddl limit edited in the shared limits.toml", + relative=f"{SELECTION_ROOT}/limits.toml", + old='note = "a ddl limit"', + new='note = "an edited ddl limit"', + expected_families=("ddl",), + ), + SelectionTestCase( + description="a limits.toml comment that changes no row", + relative=f"{SELECTION_ROOT}/limits.toml", + old="[[limit]]", + new="# a comment\n[[limit]]", + expected_families=(), + ), + SelectionTestCase( + description="one family's withdrawn file edited", + relative=f"{SELECTION_ROOT}/withdrawn/utility.toml", + old="# withdrawn utility", + new="# withdrawn utility, edited", + expected_families=("utility",), + ), + SelectionTestCase( + description="a catalogue edited, which the elisions' families also read", + relative=f"{SELECTION_ROOT}/catalogue/ddl.toml", + old="# catalogue ddl", + new="# catalogue ddl, edited", + expected_families=("ddl", "expressions", "query"), + ), + SelectionTestCase( + description="another dialect's file of the same family", + relative="coverage/syntax-databricks/withdrawn/utility.toml", + old="# withdrawn utility", + new="# withdrawn utility, edited", + expected_families=(), + ), + SelectionTestCase( + description="a module of the builder edited", + relative=BUILDER_MODULE, + old="X = 0", + new="X = 1", + expected_families=SELECTION_FAMILIES, + ), + ), + ids=lambda case: case.description, +) +def test_given_a_change_since_the_base_when_selecting_then_only_affected_families_are_built( + test_case: SelectionTestCase, tmp_path: Path +) -> None: + repository: Path = committed_repository(tmp_path) + path: Path = repository / test_case.relative + path.write_text(path.read_text().replace(test_case.old, test_case.new, 1)) + + families: list[str] = changed_families( + root=repository / SELECTION_ROOT, base="main", families=list(SELECTION_FAMILIES) + ) + + assert families == list(test_case.expected_families) + + +@pytest.mark.parametrize( + "test_case", + ( + BranchTestCase( + description="a committed ddl edit while main moved on with a query edit", + branch_edit=(f"{SELECTION_ROOT}/limits.toml", "a ddl limit", "an edited ddl limit"), + main_edit=(f"{SELECTION_ROOT}/limits.toml", "a query suppression", "an edited one"), + expected_families=("ddl",), + ), + BranchTestCase( + description="a committed withdrawn edit while main moved on with a catalogue edit", + branch_edit=(f"{SELECTION_ROOT}/withdrawn/utility.toml", "utility", "utility, edited"), + main_edit=(f"{SELECTION_ROOT}/catalogue/ddl.toml", "ddl", "ddl, edited"), + expected_families=("utility",), + ), + ), + ids=lambda case: case.description, +) +def test_given_a_committed_branch_behind_its_base_when_selecting_then_only_its_own_changes_count( + test_case: BranchTestCase, tmp_path: Path +) -> None: + repository: Path = committed_repository(tmp_path) + checkout(repository=repository, arguments=("-b", "feature")) + edit( + repository=repository, + relative=test_case.branch_edit[0], + old=test_case.branch_edit[1], + new=test_case.branch_edit[2], + ) + commit(repository=repository, message="branch") + checkout(repository=repository, arguments=("main",)) + edit( + repository=repository, + relative=test_case.main_edit[0], + old=test_case.main_edit[1], + new=test_case.main_edit[2], + ) + commit(repository=repository, message="main") + checkout(repository=repository, arguments=("feature",)) + + families: list[str] = changed_families( + root=repository / SELECTION_ROOT, base="main", families=list(SELECTION_FAMILIES) + ) + + assert families == list(test_case.expected_families) + + +@pytest.mark.parametrize( + "test_case", + ( + SkipTestCase( + description="no family changed since the base", + families=SELECTION_FAMILIES, + expected_lines=( + "syntax obligations: 0 families changed since main; skipped 4", + "syntax obligations: passed, 0 stale files; run without --check to regenerate", + ), + ), + ), + ids=lambda case: case.description, +) +def test_given_no_family_changed_when_checking_since_the_base_then_the_check_skips_and_passes( + test_case: SkipTestCase, tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + root: Path = committed_repository(tmp_path) / SELECTION_ROOT + + code: int = run_build(["--root", str(root), "--check", "--since", "main", *test_case.families]) + + assert code == 0 + assert capsys.readouterr().out.splitlines() == list(test_case.expected_lines) + + +@pytest.mark.parametrize( + "test_case", + ( + ClosureTestCase( + description="the Snowflake obligation builder", + module="scripts.build_syntax_obligations_snowflake", + selection_only=("scripts/__init__.py", "scripts/replay_scope/"), + expected_outside=(), + ), + ), + ids=lambda case: case.description, +) +def test_given_the_builder_when_importing_it_then_every_module_it_loads_is_a_builder_path( + test_case: ClosureTestCase, +) -> None: + outside: tuple[str, ...] = scripts_outside( + module=test_case.module, allowed=BUILDER_PATHS + test_case.selection_only + ) + + assert outside == test_case.expected_outside diff --git a/tests/unit/scripts/syntax_obligations_snowflake/test_check.py b/tests/unit/scripts/syntax_obligations_snowflake/test_check.py new file mode 100644 index 000000000..31653df8b --- /dev/null +++ b/tests/unit/scripts/syntax_obligations_snowflake/test_check.py @@ -0,0 +1,177 @@ +"""The build's --check fails when a committed obligations, withdrawn or elisions file is stale.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from scripts.syntax_obligations_snowflake.main.build import run_build +from tests.unit.scripts.syntax_obligations_snowflake._test_types import ( + CheckTestCase, + MissingFileTestCase, + StrayElisionTestCase, +) +from tests.unit.scripts.syntax_obligations_snowflake.helpers import ( + ELISION_FAMILY, + FAMILY, + WITHDRAWN_ROW, + built_root, +) + +STALE_ROW: str = ( + '[[withdrawn]]\nform = "sf.transaction_session.use-database"\nkind = "negative_neighbour"\n' + 'target = ""\nnote = "a rejection the build no longer withdraws"\nsql = "USE ;"\n' +) + + +@pytest.mark.parametrize( + "test_case", + ( + CheckTestCase( + description="committed files as the build writes them", + family=FAMILY, + relative=f"withdrawn/{FAMILY}.toml", + old=WITHDRAWN_ROW, + new=WITHDRAWN_ROW, + expected_code=0, + expected_stale=(), + ), + CheckTestCase( + description="a withdrawn file holding a withdrawal the build no longer makes", + family=FAMILY, + relative=f"withdrawn/{FAMILY}.toml", + old="[[withdrawn]]\n", + new=STALE_ROW + "\n[[withdrawn]]\n", + expected_code=1, + expected_stale=(f"withdrawn/{FAMILY}.toml",), + ), + CheckTestCase( + description="a withdrawn file missing a withdrawal the build makes", + family=FAMILY, + relative=f"withdrawn/{FAMILY}.toml", + old=WITHDRAWN_ROW, + new='form = "sf.transaction_session.use-other"', + expected_code=1, + expected_stale=(f"withdrawn/{FAMILY}.toml",), + ), + CheckTestCase( + description="an obligation made optional", + family=FAMILY, + relative=f"obligations/{FAMILY}.toml", + old="required = true", + new="required = false", + expected_code=1, + expected_stale=(f"obligations/{FAMILY}.toml",), + ), + CheckTestCase( + description="an elision family's files as the build writes them", + family=ELISION_FAMILY, + relative=f"elisions/{ELISION_FAMILY}.toml", + old='fill = "1"', + new='fill = "1"', + expected_code=0, + expected_stale=(), + ), + CheckTestCase( + description="an elisions file with its fill edited", + family=ELISION_FAMILY, + relative=f"elisions/{ELISION_FAMILY}.toml", + old='fill = "1"', + new='fill = "2"', + expected_code=1, + expected_stale=(f"elisions/{ELISION_FAMILY}.toml",), + ), + ), + ids=lambda case: case.description, +) +def test_given_committed_files_when_checking_then_only_a_stale_file_fails_and_none_is_written( + test_case: CheckTestCase, tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + root: Path = built_root(directory=tmp_path, family=test_case.family) + path: Path = root / test_case.relative + edited: str = path.read_text().replace(test_case.old, test_case.new, 1) + path.write_text(edited) + capsys.readouterr() + + code: int = run_build(["--root", str(root), "--check", test_case.family]) + + assert code == test_case.expected_code + assert capsys.readouterr().err.splitlines() == [ + f"stale: {root / relative} differs from what the build writes" + for relative in test_case.expected_stale + ] + assert path.read_text() == edited + + +@pytest.mark.parametrize( + "test_case", + ( + MissingFileTestCase( + description="the withdrawn file deleted", + family=FAMILY, + relative=f"withdrawn/{FAMILY}.toml", + expected_code=1, + ), + MissingFileTestCase( + description="the obligations file deleted", + family=FAMILY, + relative=f"obligations/{FAMILY}.toml", + expected_code=1, + ), + MissingFileTestCase( + description="an elision family's elisions file deleted", + family=ELISION_FAMILY, + relative=f"elisions/{ELISION_FAMILY}.toml", + expected_code=1, + ), + ), + ids=lambda case: case.description, +) +def test_given_a_file_the_build_writes_is_missing_when_checking_then_it_fails_and_stays_missing( + test_case: MissingFileTestCase, tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + root: Path = built_root(directory=tmp_path, family=test_case.family) + path: Path = root / test_case.relative + path.unlink() + capsys.readouterr() + + code: int = run_build(["--root", str(root), "--check", test_case.family]) + + assert code == test_case.expected_code + assert capsys.readouterr().err.splitlines() == [ + f"stale: {path} differs from what the build writes" + ] + assert not path.exists() + + +@pytest.mark.parametrize( + "test_case", + ( + StrayElisionTestCase( + description="regenerating", + arguments=(FAMILY,), + expected_code=0, + expected_text="# kept\n", + ), + StrayElisionTestCase( + description="checking", + arguments=("--check", FAMILY), + expected_code=0, + expected_text="# kept\n", + ), + ), + ids=lambda case: case.description, +) +def test_given_an_elisions_file_of_a_family_without_rules_when_building_then_it_is_left_alone( + test_case: StrayElisionTestCase, tmp_path: Path +) -> None: + root: Path = built_root(directory=tmp_path, family=FAMILY) + stray: Path = root / "elisions" / f"{FAMILY}.toml" + stray.parent.mkdir() + stray.write_text("# kept\n") + + code: int = run_build(["--root", str(root), *test_case.arguments]) + + assert code == test_case.expected_code + assert stray.read_text() == test_case.expected_text From 6b477b18b63d6f6cb3bd221f3616bf55b6386876 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 12 Nov 2026 22:57:14 +0000 Subject: [PATCH 3/3] ci: check every family's Snowflake obligations nightly and report failed nightly runs Pull requests and queue groups check only the families their change touches, so a scheduled run, whose base is the commit it tests, checked none. SYNTAX_OBLIGATIONS_SINCE, empty outside pull requests and queue groups, now makes those runs check every family. A failed scheduled run opens an issue labelled nightly-ci-failure, or comments on the open one. --- .github/workflows/ci.yml | 32 ++++++++++++++++++++++++++++++-- Makefile | 10 +++++++--- coverage/syntax/README.md | 3 ++- 3 files changed, 39 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4ea7d0126..f9734bed8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -6,8 +6,9 @@ name: CI # - code changes run check-ci, the Rust and Python tests, and the replay of the recorded cases, # which reuses main's judgements for every case whose conversion and recording are unchanged; # - a change under a full-replay path replays every case. -# The nightly run on main replays every case, refreshes the replay cache pull requests reuse, -# converts DuckShop within its time budget and fuzzes. Release Please dispatches a full run on +# The nightly run on main replays every case, checks every family's syntax obligations, refreshes +# the replay cache pull requests reuse, converts DuckShop within its time budget and fuzzes; when +# it fails it opens an issue labelled nightly-ci-failure, or comments on the one already open. Release Please dispatches a full run on # each release pull request's branch, so every release replays every case first. # A merge queue group runs the same checks as a pull request, scoped against the group's base. on: @@ -79,6 +80,9 @@ jobs: UV_NO_SYNC: "1" # The evidence pin check and the result baselines compare with the pull request's base. EVIDENCE_BASE: ${{ github.event.pull_request.base.sha || github.event.merge_group.base_sha || 'origin/main' }} + # Pull requests and queue groups check the families their change touches; other runs, which + # have no base to compare with, check every family's syntax obligations. + SYNTAX_OBLIGATIONS_SINCE: ${{ github.event.pull_request.base.sha || github.event.merge_group.base_sha || '' }} REPLAY: ${{ needs.changes.outputs.replay }} # The Python tests and the replay take the module and binary from check-merge's gate build. PYTHON_MODULE_BUILD: gate-build @@ -243,3 +247,27 @@ jobs: name: fuzz-artifacts path: fuzz/artifacts if-no-files-found: ignore + + nightly-report: + name: Report a failed nightly run + if: failure() && github.event_name == 'schedule' + needs: [changes, site, checks, verify, duckshop, fuzz] + runs-on: ubuntu-24.04 + permissions: + issues: write + steps: + - name: Open an issue, or comment on the one already open + env: + GH_TOKEN: ${{ github.token }} + GH_REPO: ${{ github.repository }} + LABEL: nightly-ci-failure + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + run: | + body="The nightly CI run on ${GITHUB_SHA} failed: ${RUN_URL}" + open=$(gh issue list --label "${LABEL}" --state open --json number --jq '.[0].number // empty') + if [[ -n "${open}" ]]; then + gh issue comment "${open}" --body "${body}" + else + gh label create "${LABEL}" --color B60205 --description "The nightly CI run failed" --force + gh issue create --title "Nightly CI failed" --label "${LABEL}" --body "${body}" + fi diff --git a/Makefile b/Makefile index 82f0ac0aa..ba010e7ad 100644 --- a/Makefile +++ b/Makefile @@ -196,10 +196,14 @@ generate-syntax-cases: uv run python -m scripts.build_syntax_cases $(SYNTAX_CASE_ARGS) # Fails when a Snowflake obligations, withdrawn or elisions file differs from what the builder writes. -# Only the families whose inputs changed since EVIDENCE_BASE are rebuilt (every family when the -# builder changed); with none changed the target prints that it skipped them all and passes. +# Only the families whose inputs changed since SYNTAX_OBLIGATIONS_SINCE are rebuilt (every family +# when the builder changed); with none changed the target prints that it skipped them all and +# passes. An empty SYNTAX_OBLIGATIONS_SINCE checks every family, as CI's nightly run does. +SYNTAX_OBLIGATIONS_SINCE ?= $(EVIDENCE_BASE) + syntax-obligations: - uv run python -m scripts.build_syntax_obligations_snowflake --check --since $(EVIDENCE_BASE) $(SYNTAX_FAMILIES) + uv run python -m scripts.build_syntax_obligations_snowflake --check \ + $(if $(SYNTAX_OBLIGATIONS_SINCE),--since $(SYNTAX_OBLIGATIONS_SINCE)) $(SYNTAX_FAMILIES) # Fails when a syntax case, mark or arity file differs from what the committed receipts regenerate. syntax-cases: diff --git a/coverage/syntax/README.md b/coverage/syntax/README.md index fff295974..7072ddc7d 100644 --- a/coverage/syntax/README.md +++ b/coverage/syntax/README.md @@ -45,7 +45,8 @@ python -m scripts.build_syntax_obligations_snowflake ddl query # obligations, w Rebuild a family's obligations whenever its catalogue, its rows in `limits.toml` or the builder changes. `make syntax-obligations`, part of `make check-ci`, fails when a committed obligations, withdrawn or elisions file differs from what the builder writes; it rebuilds only the families -whose inputs or generated files changed since `EVIDENCE_BASE`, and passes when none did. +whose inputs or generated files changed since `EVIDENCE_BASE`, and passes when none did. With +`SYNTAX_OBLIGATIONS_SINCE=` (empty) it checks every family, as CI's nightly run does. Pages with no Markdown export (the per-category command hubs) are fetched as HTML and catalogue no form. A rebuild keeps every existing id whose (page, grammar) is unchanged. A form whose