diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4ea7d0126..f9734bed8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -6,8 +6,9 @@ name: CI # - code changes run check-ci, the Rust and Python tests, and the replay of the recorded cases, # which reuses main's judgements for every case whose conversion and recording are unchanged; # - a change under a full-replay path replays every case. -# The nightly run on main replays every case, refreshes the replay cache pull requests reuse, -# converts DuckShop within its time budget and fuzzes. Release Please dispatches a full run on +# The nightly run on main replays every case, checks every family's syntax obligations, refreshes +# the replay cache pull requests reuse, converts DuckShop within its time budget and fuzzes; when +# it fails it opens an issue labelled nightly-ci-failure, or comments on the one already open. Release Please dispatches a full run on # each release pull request's branch, so every release replays every case first. # A merge queue group runs the same checks as a pull request, scoped against the group's base. on: @@ -79,6 +80,9 @@ jobs: UV_NO_SYNC: "1" # The evidence pin check and the result baselines compare with the pull request's base. EVIDENCE_BASE: ${{ github.event.pull_request.base.sha || github.event.merge_group.base_sha || 'origin/main' }} + # Pull requests and queue groups check the families their change touches; other runs, which + # have no base to compare with, check every family's syntax obligations. + SYNTAX_OBLIGATIONS_SINCE: ${{ github.event.pull_request.base.sha || github.event.merge_group.base_sha || '' }} REPLAY: ${{ needs.changes.outputs.replay }} # The Python tests and the replay take the module and binary from check-merge's gate build. PYTHON_MODULE_BUILD: gate-build @@ -243,3 +247,27 @@ jobs: name: fuzz-artifacts path: fuzz/artifacts if-no-files-found: ignore + + nightly-report: + name: Report a failed nightly run + if: failure() && github.event_name == 'schedule' + needs: [changes, site, checks, verify, duckshop, fuzz] + runs-on: ubuntu-24.04 + permissions: + issues: write + steps: + - name: Open an issue, or comment on the one already open + env: + GH_TOKEN: ${{ github.token }} + GH_REPO: ${{ github.repository }} + LABEL: nightly-ci-failure + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + run: | + body="The nightly CI run on ${GITHUB_SHA} failed: ${RUN_URL}" + open=$(gh issue list --label "${LABEL}" --state open --json number --jq '.[0].number // empty') + if [[ -n "${open}" ]]; then + gh issue comment "${open}" --body "${body}" + else + gh label create "${LABEL}" --color B60205 --description "The nightly CI run failed" --force + gh issue create --title "Nightly CI failed" --label "${LABEL}" --body "${body}" + fi diff --git a/Makefile b/Makefile index 8cc31f4d3..ba010e7ad 100644 --- a/Makefile +++ b/Makefile @@ -30,7 +30,7 @@ PYTHON_TYPED := scripts tests/unit/scripts python/src $(PYTHON_API_TESTS) PYTHON_TESTS := tests/unit/scripts $(PYTHON_API_TESTS) endif -.PHONY: distribution-notices format rust-lint python-check fensu syntax-coverage depth-probe depth-probe-debug instruction-counts instruction-baseline gate-build engine-free bundled-extensions dialect-builds generate-cases generated-cases generate-syntax-cases syntax-cases dialect-cases corpus-rerecord corpus-verdicts corpus-verdicts-baseline generate-referee-cases referee-cases referee-pending translation-files translation-results help pending check-syntax record-syntax check-convert record-convert sql-sizes sql-size-baseline sql-benchmark check-ci check-merge test test-rust test-python python-build python-module python-api python-wheel verify-recorded verify-changed replay-scope test-crate check-pr-metadata evidence-clone evidence-refresh evidence-pin evidence-pin-check clutter-guard census-duckshop fuzz-smoke gate gate-parallel census duckshop-compiled duckshop-measure +.PHONY: distribution-notices format rust-lint python-check fensu syntax-coverage depth-probe depth-probe-debug instruction-counts instruction-baseline gate-build engine-free bundled-extensions dialect-builds generate-cases generated-cases generate-syntax-cases syntax-obligations syntax-cases dialect-cases corpus-rerecord corpus-verdicts corpus-verdicts-baseline generate-referee-cases referee-cases referee-pending translation-files translation-results help pending check-syntax record-syntax check-convert record-convert sql-sizes sql-size-baseline sql-benchmark check-ci check-merge test test-rust test-python python-build python-module python-api python-wheel verify-recorded verify-changed replay-scope test-crate check-pr-metadata evidence-clone evidence-refresh evidence-pin evidence-pin-check clutter-guard census-duckshop fuzz-smoke gate gate-parallel census duckshop-compiled duckshop-measure format: cargo fmt --all @@ -183,10 +183,11 @@ generated-cases: # dialect's suite folder until then. syntax_evidence_dir = $(if $(EVIDENCE_PINNED),$(EVIDENCE_ROOT)/syntax/$(1),$(call suite_dir,syntax,$(1))) -# The syntax-coverage families whose cases and marks scripts/build_syntax_cases.py writes. +# The Snowflake syntax-coverage families, whose obligations, cases and marks the builders write. +SYNTAX_FAMILIES = classes ddl dml expressions functions query scripting security_admin transaction_session utility SYNTAX_CASE_ARGS = --evidence $(call syntax_evidence_dir,snowflake) \ --receipts $(call syntax_evidence_dir,snowflake)/receipts --held coverage/syntax/held.jsonl \ - classes ddl dml expressions functions query scripting security_admin transaction_session utility + $(SYNTAX_FAMILIES) # Rewrites the Snowflake syntax cases, marks and arity from the obligations and the recorded receipts. # Once the evidence is pinned the cases go to the clone's working tree, to commit there. @@ -194,6 +195,16 @@ generate-syntax-cases: EVIDENCE = working generate-syntax-cases: uv run python -m scripts.build_syntax_cases $(SYNTAX_CASE_ARGS) +# Fails when a Snowflake obligations, withdrawn or elisions file differs from what the builder writes. +# Only the families whose inputs changed since SYNTAX_OBLIGATIONS_SINCE are rebuilt (every family +# when the builder changed); with none changed the target prints that it skipped them all and +# passes. An empty SYNTAX_OBLIGATIONS_SINCE checks every family, as CI's nightly run does. +SYNTAX_OBLIGATIONS_SINCE ?= $(EVIDENCE_BASE) + +syntax-obligations: + uv run python -m scripts.build_syntax_obligations_snowflake --check \ + $(if $(SYNTAX_OBLIGATIONS_SINCE),--since $(SYNTAX_OBLIGATIONS_SINCE)) $(SYNTAX_FAMILIES) + # Fails when a syntax case, mark or arity file differs from what the committed receipts regenerate. syntax-cases: uv run python -m scripts.build_syntax_cases --check $(SYNTAX_CASE_ARGS) @@ -225,7 +236,7 @@ referee-pending: uv run python -m scripts.write_referee_pending $(REFEREE_ARGS) # The static gate, the first step of `gate`, and the Python API's tests. -check-ci: rust-lint python-check fensu syntax-coverage depth-probe instruction-counts corpus-verdicts evidence-pin-check clutter-guard engine-free bundled-extensions dialect-builds generated-cases syntax-cases dialect-cases referee-cases translation-files python-api distribution-notices replay-scope ## The static gate CI runs. +check-ci: rust-lint python-check fensu syntax-coverage depth-probe instruction-counts corpus-verdicts evidence-pin-check clutter-guard engine-free bundled-extensions dialect-builds generated-cases syntax-obligations syntax-cases dialect-cases referee-cases translation-files python-api distribution-notices replay-scope ## The static gate CI runs. # Each distribution ships the licence and third-party notices; its copies must match the root files. distribution-notices: diff --git a/coverage/syntax/README.md b/coverage/syntax/README.md index f2f9e8470..7072ddc7d 100644 --- a/coverage/syntax/README.md +++ b/coverage/syntax/README.md @@ -39,8 +39,15 @@ Families: `query`, `dml`, `ddl`, `transaction_session`, `security_admin`, `utili ```sh python -m scripts.fetch_syntax_docs_snowflake CACHE CACHE/idx/* # private cache, never committed python -m scripts.build_syntax_catalogue_snowflake --cache CACHE --out coverage/syntax +python -m scripts.build_syntax_obligations_snowflake ddl query # obligations, withdrawn, elisions ``` +Rebuild a family's obligations whenever its catalogue, its rows in `limits.toml` or the builder +changes. `make syntax-obligations`, part of `make check-ci`, fails when a committed obligations, +withdrawn or elisions file differs from what the builder writes; it rebuilds only the families +whose inputs or generated files changed since `EVIDENCE_BASE`, and passes when none did. With +`SYNTAX_OBLIGATIONS_SINCE=` (empty) it checks every family, as CI's nightly run does. + Pages with no Markdown export (the per-category command hubs) are fetched as HTML and catalogue no form. A rebuild keeps every existing id whose (page, grammar) is unchanged. A form whose grammar changed keeps its id when its natural id is unchanged or when all its earlier signatures diff --git a/coverage/syntax/obligations/ddl.toml b/coverage/syntax/obligations/ddl.toml index 20cb1102c..452105f80 100644 --- a/coverage/syntax/obligations/ddl.toml +++ b/coverage/syntax/obligations/ddl.toml @@ -176060,158 +176060,6 @@ required = true anchor = "doc.sql.create-iceberg-table / Syntax" note = "a string literal in place of (DAY ( 'v1' )), observed" -[[obligation]] -id = "sf.ddl.create-iceberg-table.table.iceberg-rest-in-a-catalog-linked-database.o299" -kind = "option_present" -required = true -target = "[ PARTITION BY ( { | BUCKET ( , ) | TRUNCATE ( , ) | YEAR (…" -anchor = "doc.sql.create-iceberg-table / Syntax / External Iceberg catalog / Iceberg REST in a catalog-linked database" -note = "option present [ PARTITION BY ( { | BUCKET ( , ) | TRUNCATE ( , ) | YEAR ( list[str]: def write_tables(*, path: Path, header: str, tables: list[tuple[str, list[Row]]]) -> None: """Write a header comment and the given arrays of tables.""" + path.write_text(tables_text(header=header, tables=tables)) + + +def tables_text(*, header: str, tables: list[tuple[str, list[Row]]]) -> str: + """A header comment and the given arrays of tables, as write_tables writes them.""" lines: list[str] = [header.rstrip(), ""] for table, rows in tables: for row in rows: lines += row_lines(table=table, row=row) - path.write_text("\n".join(lines)) + return "\n".join(lines) diff --git a/scripts/grammar_obligations/_helpers/elisions.py b/scripts/grammar_obligations/_helpers/elisions.py index e63c50ac7..4b4155a91 100644 --- a/scripts/grammar_obligations/_helpers/elisions.py +++ b/scripts/grammar_obligations/_helpers/elisions.py @@ -6,7 +6,7 @@ from collections.abc import Mapping from pathlib import Path -from scripts.grammar_catalogue._helpers.toml_out import write_tables +from scripts.grammar_catalogue._helpers.toml_out import tables_text from scripts.grammar_obligations.constants import ( ELISION_START, FAMILY_ELISIONS, @@ -66,10 +66,12 @@ def _is_repetition_branch(branch: tuple[Node, ...]) -> bool: return TOKEN_ELISION in kinds and TOKEN_KEYWORD not in kinds and len(branch) > 1 -def write_elisions(*, root: Path, family: str, trees: dict[str, list[tuple[Node, ...]]]) -> None: - """Write elisions/.toml: each elision, its fill and the forms it defers to.""" +def elision_outputs( + *, root: Path, family: str, trees: dict[str, list[tuple[Node, ...]]] +) -> list[tuple[Path, str | None]]: + """elisions/.toml and its text (None: no file); nothing for a family without rules.""" if family not in FAMILY_ELISIONS: - return + return [] rules: dict[str, tuple[str, tuple[str, ...], str]] = FAMILY_ELISIONS[family] catalogued: set[str] = _catalogued(root=root) rows: list[Row] = [] @@ -94,16 +96,14 @@ def write_elisions(*, root: Path, family: str, trees: dict[str, list[tuple[Node, ) path: Path = root / "elisions" / f"{family}.toml" if not rows: - path.unlink(missing_ok=True) - return - path.parent.mkdir(parents=True, exist_ok=True) - write_tables( - path=path, + return [(path, None)] + text: str = tables_text( header=f"# Elisions (`...`) in the grammar of family {family}: what each renders as and " "the catalogued\n# forms that carry the elided syntax; generated by " "scripts/build_syntax_obligations.py.", tables=[("elision", rows)], ) + return [(path, text)] def _catalogued(*, root: Path) -> set[str]: diff --git a/scripts/syntax_obligations_snowflake/_helpers/__init__.py b/scripts/syntax_obligations_snowflake/_helpers/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/scripts/syntax_obligations_snowflake/_helpers/changed.py b/scripts/syntax_obligations_snowflake/_helpers/changed.py new file mode 100644 index 000000000..d6853ba2b --- /dev/null +++ b/scripts/syntax_obligations_snowflake/_helpers/changed.py @@ -0,0 +1,84 @@ +"""The families whose generated obligations can differ since a base revision.""" + +from __future__ import annotations + +import subprocess +import tomllib +from pathlib import Path + +from scripts.grammar_obligations.constants import FAMILY_ELISIONS +from scripts.grammar_obligations.types import Row +from scripts.replay_scope._helpers.git import changed +from scripts.syntax_obligations_snowflake.constants import ( + BUILDER_PATHS, + CATALOGUE_DIR, + FAMILY_DIRS, + FAMILY_FILE_DEPTH, + FAMILY_FILE_SUFFIX, + LIMITS_FILE, + LIMITS_TABLES, +) + + +def changed_families(*, root: Path, base: str, families: list[str]) -> list[str]: + """The named families whose catalogue, limits rows, generated files or builder changed.""" + top: Path = Path(_git(cwd=root, arguments=("rev-parse", "--show-toplevel")).strip()) + prefix: str = root.resolve().relative_to(top.resolve()).as_posix() + "/" + paths: list[str] = changed(root=top, base=base) + if any(path.startswith(BUILDER_PATHS) for path in paths): + return list(families) + selected: set[str] = set() + for path in paths: + selected |= ( + _file_families(relative=path.removeprefix(prefix)) if path.startswith(prefix) else set() + ) + if prefix + LIMITS_FILE in paths: + selected |= _limits_families(root=root, top=top, base=base, relative=prefix + LIMITS_FILE) + return [family for family in families if family in selected] + + +def _file_families(*, relative: str) -> set[str]: + """The families a changed file under the root feeds: its own, plus the elisions' readers.""" + parts: list[str] = relative.split("/") + if ( + len(parts) != FAMILY_FILE_DEPTH + or parts[0] not in FAMILY_DIRS + or not parts[1].endswith(FAMILY_FILE_SUFFIX) + ): + return set() + family: set[str] = {parts[1].removesuffix(FAMILY_FILE_SUFFIX)} + return family | FAMILY_ELISIONS.keys() if parts[0] == CATALOGUE_DIR else family + + +def _limits_families(*, root: Path, top: Path, base: str, relative: str) -> set[str]: + """The families whose rows differ between the merge base's limits.toml and the root's.""" + merge_base: str = _git(cwd=top, arguments=("merge-base", base, "HEAD")).strip() + before: dict[str, list[tuple[str, Row]]] = _limits_by_family( + _git(cwd=top, arguments=("show", f"{merge_base}:{relative}"), missing_ok=True) + ) + path: Path = root / LIMITS_FILE + after: dict[str, list[tuple[str, Row]]] = _limits_by_family( + path.read_text() if path.exists() else "" + ) + return { + family for family in before.keys() | after.keys() if before.get(family) != after.get(family) + } + + +def _limits_by_family(text: str) -> dict[str, list[tuple[str, Row]]]: + """limits.toml's rows grouped by the family their form id names, in file order per table.""" + grouped: dict[str, list[tuple[str, Row]]] = {} + document: dict[str, list[Row]] = tomllib.loads(text) + for table in LIMITS_TABLES: + for row in document.get(table, []): + family: str = str(row["form"]).split(".")[1] + grouped.setdefault(family, []).append((table, row)) + return grouped + + +def _git(*, cwd: Path, arguments: tuple[str, ...], missing_ok: bool = False) -> str: + """A Git command's output; with missing_ok, empty when it fails, as for a file not there.""" + result: subprocess.CompletedProcess[str] = subprocess.run( + ("git", *arguments), cwd=cwd, check=not missing_ok, capture_output=True, text=True + ) + return result.stdout if result.returncode == 0 else "" diff --git a/scripts/syntax_obligations_snowflake/constants.py b/scripts/syntax_obligations_snowflake/constants.py index 1a30a931f..3d343ab7a 100644 --- a/scripts/syntax_obligations_snowflake/constants.py +++ b/scripts/syntax_obligations_snowflake/constants.py @@ -2,6 +2,8 @@ from __future__ import annotations +from pathlib import Path + WITHDRAWN_HEADER: str = ( "# Derived rejections of family {family} withdrawn because the documented grammar also\n" "# renders their SQL as valid; generated by scripts/build_syntax_obligations.py." @@ -11,3 +13,17 @@ "\n# Each [[withdrawn_class]] withdraws one class of derived negatives by decision, once," " with its count." ) + +DEFAULT_ROOT: Path = Path("coverage/syntax") +BUILDER_PATHS: tuple[str, ...] = ( + "scripts/build_syntax_obligations_snowflake.py", + "scripts/syntax_obligations_snowflake/", + "scripts/grammar_obligations/", + "scripts/grammar_catalogue/", +) +CATALOGUE_DIR: str = "catalogue" +FAMILY_DIRS: tuple[str, ...] = (CATALOGUE_DIR, "obligations", "withdrawn", "elisions") +FAMILY_FILE_SUFFIX: str = ".toml" +FAMILY_FILE_DEPTH: int = 2 +LIMITS_FILE: str = "limits.toml" +LIMITS_TABLES: tuple[str, ...] = ("limit", "suppress", "reading") diff --git a/scripts/syntax_obligations_snowflake/main/build.py b/scripts/syntax_obligations_snowflake/main/build.py index f02bddec3..fd9b024e5 100644 --- a/scripts/syntax_obligations_snowflake/main/build.py +++ b/scripts/syntax_obligations_snowflake/main/build.py @@ -1,49 +1,82 @@ -"""Write coverage/syntax/obligations/.toml from the catalogue grammar and limits.toml.""" +"""Write, or with --check compare, the Snowflake obligations, withdrawn and elisions files.""" from __future__ import annotations import argparse +import sys from pathlib import Path -from scripts.grammar_catalogue._helpers.toml_out import write_tables -from scripts.grammar_obligations._helpers.elisions import write_elisions +from scripts.grammar_catalogue._helpers.toml_out import tables_text +from scripts.grammar_obligations._helpers.cases import sync_outputs +from scripts.grammar_obligations._helpers.elisions import elision_outputs from scripts.grammar_obligations._helpers.obligations import family_obligations, obligation_rows -from scripts.grammar_obligations.types import Row +from scripts.syntax_obligations_snowflake._helpers.changed import changed_families from scripts.syntax_obligations_snowflake.constants import ( + DEFAULT_ROOT, WITHDRAWN_CLASS_HEADER, WITHDRAWN_HEADER, ) def run_build(argv: list[str] | None = None) -> int: - """Regenerate the obligations of the named families.""" + """Regenerate the obligations of the named families, or with --check compare them.""" parser: argparse.ArgumentParser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--root", type=Path, default=Path("coverage/syntax")) + parser.add_argument("--root", type=Path, default=DEFAULT_ROOT) + parser.add_argument( + "--check", action="store_true", help="write nothing; fail when a committed file differs" + ) + parser.add_argument( + "--since", + metavar="REVISION", + help="build only the families whose inputs changed since the merge base with REVISION", + ) parser.add_argument("families", nargs="+") arguments: argparse.Namespace = parser.parse_args(argv) root: Path = arguments.root - for family in arguments.families: - path: Path = root / "obligations" / f"{family}.toml" - numbered, trees, withdrawn, classes, _ = family_obligations(root=root, family=family) - write_elisions(root=root, family=family, trees=trees) - rows: list[Row] = obligation_rows(numbered) - path.parent.mkdir(parents=True, exist_ok=True) - write_tables( - path=path, - header=f"# Obligations of family {family}; generated by " - "scripts/build_syntax_obligations.py from the catalogue grammar and limits.toml.\n" - "# See docs/syntax-coverage.md.", - tables=[("obligation", rows)], + families: list[str] = arguments.families + if arguments.since: + families = changed_families(root=root, base=arguments.since, families=families) + skipped: list[str] = [f for f in arguments.families if f not in families] + print( + f"syntax obligations: {len(families)} families changed since {arguments.since}" + f"{': ' + ' '.join(families) if families else ''}; skipped {len(skipped)}" ) - withdrawn_path: Path = root / "withdrawn" / f"{family}.toml" - if withdrawn or classes: - withdrawn_path.parent.mkdir(parents=True, exist_ok=True) - write_tables( - path=withdrawn_path, - header=WITHDRAWN_HEADER.format(family=family) - + (WITHDRAWN_CLASS_HEADER if classes else ""), - tables=[("withdrawn_class", classes), ("withdrawn", withdrawn)], - ) - elif withdrawn_path.exists(): - withdrawn_path.unlink() - return 0 + differing: list[Path] = [] + for family in families: + differing += sync_outputs( + outputs=_family_outputs(root=root, family=family), check=arguments.check + ) + if not arguments.check: + return 0 + for path in differing: + print(f"stale: {path} differs from what the build writes", file=sys.stderr) + print( + f"syntax obligations: {'FAILED' if differing else 'passed'}, {len(differing)} stale files;" + " run without --check to regenerate" + ) + return 1 if differing else 0 + + +def _family_outputs(*, root: Path, family: str) -> list[tuple[Path, str | None]]: + """The text of each file the build writes for one family (None: no file).""" + numbered, trees, withdrawn, classes, _ = family_obligations(root=root, family=family) + obligations: str = tables_text( + header=f"# Obligations of family {family}; generated by " + "scripts/build_syntax_obligations.py from the catalogue grammar and limits.toml.\n" + "# See docs/syntax-coverage.md.", + tables=[("obligation", obligation_rows(numbered))], + ) + withdrawals: str | None = ( + tables_text( + header=WITHDRAWN_HEADER.format(family=family) + + (WITHDRAWN_CLASS_HEADER if classes else ""), + tables=[("withdrawn_class", classes), ("withdrawn", withdrawn)], + ) + if withdrawn or classes + else None + ) + return [ + *elision_outputs(root=root, family=family, trees=trees), + (root / "obligations" / f"{family}.toml", obligations), + (root / "withdrawn" / f"{family}.toml", withdrawals), + ] diff --git a/tests/unit/scripts/syntax_obligations_snowflake/__init__.py b/tests/unit/scripts/syntax_obligations_snowflake/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/unit/scripts/syntax_obligations_snowflake/_test_types.py b/tests/unit/scripts/syntax_obligations_snowflake/_test_types.py new file mode 100644 index 000000000..75bfbb133 --- /dev/null +++ b/tests/unit/scripts/syntax_obligations_snowflake/_test_types.py @@ -0,0 +1,64 @@ +"""Test types for the Snowflake obligation build's --check and its family selection.""" + +from __future__ import annotations + +from dataclasses import dataclass + + +@dataclass(frozen=True) +class CheckTestCase: + description: str + family: str + relative: str + old: str + new: str + expected_code: int + expected_stale: tuple[str, ...] + + +@dataclass(frozen=True) +class MissingFileTestCase: + description: str + family: str + relative: str + expected_code: int + + +@dataclass(frozen=True) +class SelectionTestCase: + description: str + relative: str + old: str + new: str + expected_families: tuple[str, ...] + + +@dataclass(frozen=True) +class SkipTestCase: + description: str + families: tuple[str, ...] + expected_lines: tuple[str, ...] + + +@dataclass(frozen=True) +class ClosureTestCase: + description: str + module: str + selection_only: tuple[str, ...] + expected_outside: tuple[str, ...] + + +@dataclass(frozen=True) +class BranchTestCase: + description: str + branch_edit: tuple[str, str, str] + main_edit: tuple[str, str, str] + expected_families: tuple[str, ...] + + +@dataclass(frozen=True) +class StrayElisionTestCase: + description: str + arguments: tuple[str, ...] + expected_code: int + expected_text: str diff --git a/tests/unit/scripts/syntax_obligations_snowflake/helpers.py b/tests/unit/scripts/syntax_obligations_snowflake/helpers.py new file mode 100644 index 000000000..9851b21d1 --- /dev/null +++ b/tests/unit/scripts/syntax_obligations_snowflake/helpers.py @@ -0,0 +1,128 @@ +"""Small roots and repositories for the Snowflake obligation build's tests.""" + +from __future__ import annotations + +import subprocess +import sys +from itertools import filterfalse +from pathlib import Path + +from scripts.syntax_obligations_snowflake.main.build import run_build + +REPOSITORY: Path = Path(__file__).parents[4] +FAMILY: str = "transaction_session" +CATALOGUE: str = """ +[[form]] +id = "sf.transaction_session.use-database" +family = "transaction_session" +kind = "statement" +grammar = ["USE [ DATABASE ] "] +uses = [] +status = "catalogued" + +[[form]] +id = "sf.transaction_session.use-role" +family = "transaction_session" +kind = "statement" +grammar = ["USE ROLE "] +uses = [] +status = "catalogued" +""" +WITHDRAWN_ROW: str = 'form = "sf.transaction_session.use-role"' +ELISION_FAMILY: str = "expressions" +ELISION_CATALOGUE: str = """ +[[form]] +id = "sf.expressions.commands-model-function.model-default-version" +family = "expressions" +kind = "statement" +grammar = ["SELECT MODEL( )!( ... ) FROM "] +uses = [] +status = "catalogued" +""" +CATALOGUES: dict[str, str] = {FAMILY: CATALOGUE, ELISION_FAMILY: ELISION_CATALOGUE} +SELECTION_ROOT: str = "coverage/syntax" +SELECTION_FAMILIES: tuple[str, ...] = ("ddl", "expressions", "query", "utility") +SELECTION_LIMITS: str = """[[limit]] +form = "sf.ddl.create-table" +kind = "negative" +note = "a ddl limit" + +[[suppress]] +form = "sf.query.select" +kind = "positive" +reason = "a query suppression" +""" +BUILDER_MODULE: str = "scripts/grammar_obligations/x.py" +LOADED_MODULES: str = ( + "import importlib, sys\n" + "importlib.import_module(sys.argv[1])\n" + "print('\\n'.join(m.__file__ for n, m in sys.modules.items() if n.split('.')[0] == 'scripts'))" +) + + +def built_root(*, directory: Path, family: str) -> Path: + """A root holding one family's small catalogue, built once, so its files are as built.""" + root: Path = directory / "root" + (root / "catalogue").mkdir(parents=True) + (root / "catalogue" / f"{family}.toml").write_text(CATALOGUES[family]) + (root / "limits.toml").write_text("") + assert run_build(["--root", str(root), family]) == 0 + return root + + +def committed_repository(directory: Path) -> Path: + """A committed repository with each selection family's files, another dialect's and a module.""" + files: dict[str, str] = { + f"{SELECTION_ROOT}/limits.toml": SELECTION_LIMITS, + BUILDER_MODULE: "X = 0\n", + } + for part in ("catalogue", "obligations", "withdrawn"): + files |= _family_files(prefix=f"{SELECTION_ROOT}/{part}", part=part) + files |= _family_files(prefix=f"coverage/syntax-databricks/{part}", part=part) + for relative, text in files.items(): + (directory / relative).parent.mkdir(parents=True, exist_ok=True) + (directory / relative).write_text(text) + _git(directory, "init", "-q", "-b", "main") + commit(repository=directory, message="base") + return directory + + +def edit(*, repository: Path, relative: str, old: str, new: str) -> None: + """Replace the first old with new in one file of the repository.""" + path: Path = repository / relative + path.write_text(path.read_text().replace(old, new, 1)) + + +def commit(*, repository: Path, message: str) -> None: + """Commit every change in the repository.""" + _git(repository, "add", "-A") + _git( + repository, "-c", "user.name=t", "-c", "user.email=t@example.com", "commit", "-qm", message + ) + + +def checkout(*, repository: Path, arguments: tuple[str, ...]) -> None: + """Run git checkout with arguments, as to create or switch a branch.""" + _git(repository, "checkout", "-q", *arguments) + + +def scripts_outside(*, module: str, allowed: tuple[str, ...]) -> tuple[str, ...]: + """The files of scripts modules a fresh import of module loads that no allowed prefix covers.""" + output: str = subprocess.run( + (sys.executable, "-c", LOADED_MODULES, module), + cwd=REPOSITORY, + check=True, + capture_output=True, + text=True, + ).stdout + loaded: list[str] = [Path(line).relative_to(REPOSITORY).as_posix() for line in output.split()] + assert loaded + return tuple(filterfalse(lambda path: path.startswith(allowed), loaded)) + + +def _family_files(*, prefix: str, part: str) -> dict[str, str]: + return {f"{prefix}/{family}.toml": f"# {part} {family}\n" for family in SELECTION_FAMILIES} + + +def _git(repository: Path, *arguments: str) -> None: + subprocess.run(("git", *arguments), cwd=repository, check=True, capture_output=True) diff --git a/tests/unit/scripts/syntax_obligations_snowflake/test_changed.py b/tests/unit/scripts/syntax_obligations_snowflake/test_changed.py new file mode 100644 index 000000000..da652930b --- /dev/null +++ b/tests/unit/scripts/syntax_obligations_snowflake/test_changed.py @@ -0,0 +1,190 @@ +"""--since selects only the families whose inputs or generated files changed.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from scripts.syntax_obligations_snowflake._helpers.changed import changed_families +from scripts.syntax_obligations_snowflake.constants import BUILDER_PATHS +from scripts.syntax_obligations_snowflake.main.build import run_build +from tests.unit.scripts.syntax_obligations_snowflake._test_types import ( + BranchTestCase, + ClosureTestCase, + SelectionTestCase, + SkipTestCase, +) +from tests.unit.scripts.syntax_obligations_snowflake.helpers import ( + BUILDER_MODULE, + SELECTION_FAMILIES, + SELECTION_ROOT, + checkout, + commit, + committed_repository, + edit, + scripts_outside, +) + + +@pytest.mark.parametrize( + "test_case", + ( + SelectionTestCase( + description="nothing changed", + relative=f"{SELECTION_ROOT}/limits.toml", + old="a ddl limit", + new="a ddl limit", + expected_families=(), + ), + SelectionTestCase( + description="a ddl limit edited in the shared limits.toml", + relative=f"{SELECTION_ROOT}/limits.toml", + old='note = "a ddl limit"', + new='note = "an edited ddl limit"', + expected_families=("ddl",), + ), + SelectionTestCase( + description="a limits.toml comment that changes no row", + relative=f"{SELECTION_ROOT}/limits.toml", + old="[[limit]]", + new="# a comment\n[[limit]]", + expected_families=(), + ), + SelectionTestCase( + description="one family's withdrawn file edited", + relative=f"{SELECTION_ROOT}/withdrawn/utility.toml", + old="# withdrawn utility", + new="# withdrawn utility, edited", + expected_families=("utility",), + ), + SelectionTestCase( + description="a catalogue edited, which the elisions' families also read", + relative=f"{SELECTION_ROOT}/catalogue/ddl.toml", + old="# catalogue ddl", + new="# catalogue ddl, edited", + expected_families=("ddl", "expressions", "query"), + ), + SelectionTestCase( + description="another dialect's file of the same family", + relative="coverage/syntax-databricks/withdrawn/utility.toml", + old="# withdrawn utility", + new="# withdrawn utility, edited", + expected_families=(), + ), + SelectionTestCase( + description="a module of the builder edited", + relative=BUILDER_MODULE, + old="X = 0", + new="X = 1", + expected_families=SELECTION_FAMILIES, + ), + ), + ids=lambda case: case.description, +) +def test_given_a_change_since_the_base_when_selecting_then_only_affected_families_are_built( + test_case: SelectionTestCase, tmp_path: Path +) -> None: + repository: Path = committed_repository(tmp_path) + path: Path = repository / test_case.relative + path.write_text(path.read_text().replace(test_case.old, test_case.new, 1)) + + families: list[str] = changed_families( + root=repository / SELECTION_ROOT, base="main", families=list(SELECTION_FAMILIES) + ) + + assert families == list(test_case.expected_families) + + +@pytest.mark.parametrize( + "test_case", + ( + BranchTestCase( + description="a committed ddl edit while main moved on with a query edit", + branch_edit=(f"{SELECTION_ROOT}/limits.toml", "a ddl limit", "an edited ddl limit"), + main_edit=(f"{SELECTION_ROOT}/limits.toml", "a query suppression", "an edited one"), + expected_families=("ddl",), + ), + BranchTestCase( + description="a committed withdrawn edit while main moved on with a catalogue edit", + branch_edit=(f"{SELECTION_ROOT}/withdrawn/utility.toml", "utility", "utility, edited"), + main_edit=(f"{SELECTION_ROOT}/catalogue/ddl.toml", "ddl", "ddl, edited"), + expected_families=("utility",), + ), + ), + ids=lambda case: case.description, +) +def test_given_a_committed_branch_behind_its_base_when_selecting_then_only_its_own_changes_count( + test_case: BranchTestCase, tmp_path: Path +) -> None: + repository: Path = committed_repository(tmp_path) + checkout(repository=repository, arguments=("-b", "feature")) + edit( + repository=repository, + relative=test_case.branch_edit[0], + old=test_case.branch_edit[1], + new=test_case.branch_edit[2], + ) + commit(repository=repository, message="branch") + checkout(repository=repository, arguments=("main",)) + edit( + repository=repository, + relative=test_case.main_edit[0], + old=test_case.main_edit[1], + new=test_case.main_edit[2], + ) + commit(repository=repository, message="main") + checkout(repository=repository, arguments=("feature",)) + + families: list[str] = changed_families( + root=repository / SELECTION_ROOT, base="main", families=list(SELECTION_FAMILIES) + ) + + assert families == list(test_case.expected_families) + + +@pytest.mark.parametrize( + "test_case", + ( + SkipTestCase( + description="no family changed since the base", + families=SELECTION_FAMILIES, + expected_lines=( + "syntax obligations: 0 families changed since main; skipped 4", + "syntax obligations: passed, 0 stale files; run without --check to regenerate", + ), + ), + ), + ids=lambda case: case.description, +) +def test_given_no_family_changed_when_checking_since_the_base_then_the_check_skips_and_passes( + test_case: SkipTestCase, tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + root: Path = committed_repository(tmp_path) / SELECTION_ROOT + + code: int = run_build(["--root", str(root), "--check", "--since", "main", *test_case.families]) + + assert code == 0 + assert capsys.readouterr().out.splitlines() == list(test_case.expected_lines) + + +@pytest.mark.parametrize( + "test_case", + ( + ClosureTestCase( + description="the Snowflake obligation builder", + module="scripts.build_syntax_obligations_snowflake", + selection_only=("scripts/__init__.py", "scripts/replay_scope/"), + expected_outside=(), + ), + ), + ids=lambda case: case.description, +) +def test_given_the_builder_when_importing_it_then_every_module_it_loads_is_a_builder_path( + test_case: ClosureTestCase, +) -> None: + outside: tuple[str, ...] = scripts_outside( + module=test_case.module, allowed=BUILDER_PATHS + test_case.selection_only + ) + + assert outside == test_case.expected_outside diff --git a/tests/unit/scripts/syntax_obligations_snowflake/test_check.py b/tests/unit/scripts/syntax_obligations_snowflake/test_check.py new file mode 100644 index 000000000..31653df8b --- /dev/null +++ b/tests/unit/scripts/syntax_obligations_snowflake/test_check.py @@ -0,0 +1,177 @@ +"""The build's --check fails when a committed obligations, withdrawn or elisions file is stale.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from scripts.syntax_obligations_snowflake.main.build import run_build +from tests.unit.scripts.syntax_obligations_snowflake._test_types import ( + CheckTestCase, + MissingFileTestCase, + StrayElisionTestCase, +) +from tests.unit.scripts.syntax_obligations_snowflake.helpers import ( + ELISION_FAMILY, + FAMILY, + WITHDRAWN_ROW, + built_root, +) + +STALE_ROW: str = ( + '[[withdrawn]]\nform = "sf.transaction_session.use-database"\nkind = "negative_neighbour"\n' + 'target = ""\nnote = "a rejection the build no longer withdraws"\nsql = "USE ;"\n' +) + + +@pytest.mark.parametrize( + "test_case", + ( + CheckTestCase( + description="committed files as the build writes them", + family=FAMILY, + relative=f"withdrawn/{FAMILY}.toml", + old=WITHDRAWN_ROW, + new=WITHDRAWN_ROW, + expected_code=0, + expected_stale=(), + ), + CheckTestCase( + description="a withdrawn file holding a withdrawal the build no longer makes", + family=FAMILY, + relative=f"withdrawn/{FAMILY}.toml", + old="[[withdrawn]]\n", + new=STALE_ROW + "\n[[withdrawn]]\n", + expected_code=1, + expected_stale=(f"withdrawn/{FAMILY}.toml",), + ), + CheckTestCase( + description="a withdrawn file missing a withdrawal the build makes", + family=FAMILY, + relative=f"withdrawn/{FAMILY}.toml", + old=WITHDRAWN_ROW, + new='form = "sf.transaction_session.use-other"', + expected_code=1, + expected_stale=(f"withdrawn/{FAMILY}.toml",), + ), + CheckTestCase( + description="an obligation made optional", + family=FAMILY, + relative=f"obligations/{FAMILY}.toml", + old="required = true", + new="required = false", + expected_code=1, + expected_stale=(f"obligations/{FAMILY}.toml",), + ), + CheckTestCase( + description="an elision family's files as the build writes them", + family=ELISION_FAMILY, + relative=f"elisions/{ELISION_FAMILY}.toml", + old='fill = "1"', + new='fill = "1"', + expected_code=0, + expected_stale=(), + ), + CheckTestCase( + description="an elisions file with its fill edited", + family=ELISION_FAMILY, + relative=f"elisions/{ELISION_FAMILY}.toml", + old='fill = "1"', + new='fill = "2"', + expected_code=1, + expected_stale=(f"elisions/{ELISION_FAMILY}.toml",), + ), + ), + ids=lambda case: case.description, +) +def test_given_committed_files_when_checking_then_only_a_stale_file_fails_and_none_is_written( + test_case: CheckTestCase, tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + root: Path = built_root(directory=tmp_path, family=test_case.family) + path: Path = root / test_case.relative + edited: str = path.read_text().replace(test_case.old, test_case.new, 1) + path.write_text(edited) + capsys.readouterr() + + code: int = run_build(["--root", str(root), "--check", test_case.family]) + + assert code == test_case.expected_code + assert capsys.readouterr().err.splitlines() == [ + f"stale: {root / relative} differs from what the build writes" + for relative in test_case.expected_stale + ] + assert path.read_text() == edited + + +@pytest.mark.parametrize( + "test_case", + ( + MissingFileTestCase( + description="the withdrawn file deleted", + family=FAMILY, + relative=f"withdrawn/{FAMILY}.toml", + expected_code=1, + ), + MissingFileTestCase( + description="the obligations file deleted", + family=FAMILY, + relative=f"obligations/{FAMILY}.toml", + expected_code=1, + ), + MissingFileTestCase( + description="an elision family's elisions file deleted", + family=ELISION_FAMILY, + relative=f"elisions/{ELISION_FAMILY}.toml", + expected_code=1, + ), + ), + ids=lambda case: case.description, +) +def test_given_a_file_the_build_writes_is_missing_when_checking_then_it_fails_and_stays_missing( + test_case: MissingFileTestCase, tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + root: Path = built_root(directory=tmp_path, family=test_case.family) + path: Path = root / test_case.relative + path.unlink() + capsys.readouterr() + + code: int = run_build(["--root", str(root), "--check", test_case.family]) + + assert code == test_case.expected_code + assert capsys.readouterr().err.splitlines() == [ + f"stale: {path} differs from what the build writes" + ] + assert not path.exists() + + +@pytest.mark.parametrize( + "test_case", + ( + StrayElisionTestCase( + description="regenerating", + arguments=(FAMILY,), + expected_code=0, + expected_text="# kept\n", + ), + StrayElisionTestCase( + description="checking", + arguments=("--check", FAMILY), + expected_code=0, + expected_text="# kept\n", + ), + ), + ids=lambda case: case.description, +) +def test_given_an_elisions_file_of_a_family_without_rules_when_building_then_it_is_left_alone( + test_case: StrayElisionTestCase, tmp_path: Path +) -> None: + root: Path = built_root(directory=tmp_path, family=FAMILY) + stray: Path = root / "elisions" / f"{FAMILY}.toml" + stray.parent.mkdir() + stray.write_text("# kept\n") + + code: int = run_build(["--root", str(root), *test_case.arguments]) + + assert code == test_case.expected_code + assert stray.read_text() == test_case.expected_text