Skip to content

Commit 804864d

Browse files
committed
Merge build/evidence-make-targets: make targets for every suite and dialect, and one set of verify flags for CI and the gate
2 parents febd122 + 330feef commit 804864d

5 files changed

Lines changed: 148 additions & 29 deletions

File tree

‎.github/workflows/ci.yml‎

Lines changed: 4 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -51,16 +51,11 @@ jobs:
5151
run: make check-ci
5252
- name: Run tests
5353
run: make test
54-
- name: Replay recorded cases on DuckDB
54+
# `make verify-recorded`, with the same verify flags as `make gate`, then the measure and SQL
55+
# size checks.
56+
- name: Replay recorded translations on DuckDB and check their baselines
5557
run: >-
56-
cargo run --locked -p sqlquack -- verify --require-verified --python .venv/bin/python
57-
--translation-statements target/translation-statements.json --sql-sizes target/sql-sizes.json
58-
- name: Check the result-matching measure against its baselines
59-
run: >-
60-
make translation-results RESULTS_BASE="$EVIDENCE_BASE"
61-
- name: Check the generated SQL's size against its baselines
62-
run: >-
63-
make sql-sizes RESULTS_BASE="$EVIDENCE_BASE"
58+
make check-convert DIALECT=snowflake VERIFY_PROFILE=dev RESULTS_BASE="$EVIDENCE_BASE"
6459
6560
polyglot-check:
6661
name: Exact parser check on DuckShop

‎.gitignore‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ __pycache__/
1111
# The Python module `make python-module` builds into its package.
1212
/python/src/sqlquack/*.so
1313
.ruff_cache/
14+
.pytest_cache/
1415

1516
# Fensu
1617
.fensu/cache/
@@ -27,6 +28,7 @@ __pycache__/
2728
.vscode/
2829

2930
# Local agent configuration and generated skills
31+
/.amp/
3032
.claude/*
3133
/.agents/skills/
3234
/.claude/skills/

‎Makefile‎

Lines changed: 93 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@ PYTHON_TYPED := scripts tests/unit/scripts python/src $(PYTHON_API_TESTS)
3030
PYTHON_TESTS := tests/unit/scripts $(PYTHON_API_TESTS)
3131
endif
3232

33-
.PHONY: distribution-notices format rust-lint python-check fensu syntax-coverage depth-probe depth-probe-debug gate-build evidence-guard polyglot-guard engine-free generate-cases generated-cases generate-syntax-cases syntax-cases dialect-cases corpus-rerecord generate-referee-cases referee-cases referee-pending translation-results sql-sizes sql-size-baseline sql-benchmark check-ci check-merge test test-rust test-python python-module python-api python-wheel verify-recorded polyglot-duckshop census-duckshop fuzz-smoke gate gate-parallel census duckshop-compiled duckshop-measure
33+
.PHONY: distribution-notices format rust-lint python-check fensu syntax-coverage depth-probe depth-probe-debug gate-build evidence-guard polyglot-guard engine-free generate-cases generated-cases generate-syntax-cases syntax-cases dialect-cases corpus-rerecord generate-referee-cases referee-cases referee-pending translation-results help pending check-syntax record-syntax check-convert record-convert sql-sizes sql-size-baseline sql-benchmark check-ci check-merge test test-rust test-python python-module python-api python-wheel verify-recorded polyglot-duckshop census-duckshop fuzz-smoke gate gate-parallel census duckshop-compiled duckshop-measure
3434

3535
format:
3636
cargo fmt --all
@@ -59,10 +59,14 @@ fensu:
5959

6060
# Fails on any unresolved coverage reference, a form or obligation the baseline claims and the
6161
# current parser no longer supports, or a catalogue change the baseline was not re-recorded for.
62+
# The dialects whose syntax claims check-ci holds to their baseline, through `make check-syntax`.
63+
# REQUIRE_BASELINE=1 fails instead of falling back to `summary`, which does not compare with one.
64+
CHECK_SYNTAX_DIALECTS ?= snowflake sqlite duckdb
65+
6266
syntax-coverage: gate-build
63-
$(GATE_DIR)/syntax-coverage diff coverage/syntax/baseline.json
64-
$(GATE_DIR)/syntax-coverage diff --root coverage/syntax-sqlite coverage/syntax-sqlite/baseline.json
65-
$(GATE_DIR)/syntax-coverage diff --root coverage/syntax-duckdb coverage/syntax-duckdb/baseline.json
67+
@set -e; for dialect in $(CHECK_SYNTAX_DIALECTS); do \
68+
$(MAKE) --no-print-directory check-syntax DIALECT=$$dialect REQUIRE_BASELINE=1; \
69+
done
6670

6771
# Fails when a native parser crashes, panics or hangs on deeply nested SQL on a 2 MiB thread and
6872
# coverage/deep-nesting.toml does not list it, or when a listed failure no longer happens.
@@ -136,7 +140,7 @@ referee-pending:
136140
uv run python -m scripts.write_referee_pending $(REFEREE_ARGS)
137141

138142
# The static gate, the first step of `gate`, and the Python API's tests.
139-
check-ci: rust-lint python-check fensu syntax-coverage depth-probe evidence-guard polyglot-guard engine-free generated-cases syntax-cases dialect-cases referee-cases python-api distribution-notices
143+
check-ci: rust-lint python-check fensu syntax-coverage depth-probe evidence-guard polyglot-guard engine-free generated-cases syntax-cases dialect-cases referee-cases python-api distribution-notices ## The static gate CI runs.
140144

141145
# Each distribution ships the licence and third-party notices; its copies must match the root files.
142146
distribution-notices:
@@ -167,8 +171,9 @@ test-rust: gate-build
167171
RUST_TEST_JOBS=$(RUST_TEST_JOBS) RUST_TEST_FEATURES=$(GATE_FEATURES) \
168172
scripts/run_rust_tests.sh $(RUST_TEST_PROFILE)
169173

170-
# What every merge to main must pass, on the exact head: the static gate and the Rust tests.
171-
check-merge: check-ci test-rust
174+
# What every merge to main must pass, on the exact head: the static gate and the Rust tests. It does
175+
# not replay the recorded cases; `make verify-recorded` does, and CI runs it on every pull request.
176+
check-merge: check-ci test-rust ## The merge gate: check-ci and the Rust tests; it replays no recordings.
172177

173178
test-python: python-api
174179
uv run pytest $(filter-out $(PYTHON_API_TESTS),$(PYTHON_TESTS)) --ignore=$(PYTHON_API_TESTS) \
@@ -191,9 +196,13 @@ python-api: python-module
191196
python-wheel: | .venv/bin/python
192197
cd python && uvx --from "maturin[patchelf]>=1.9,<2" maturin build --release --locked
193198

194-
# Replays every recorded case on DuckDB; every rule and equivalence needs a passed case.
199+
# Replays every recorded case on DuckDB; every rule and equivalence needs a passed case. CI and
200+
# `make gate` both run verify-recorded, so these are the one set of verify flags.
195201
# Backup pushes may drop --require-verified; failed cases and error findings still block.
196202
VERIFY_FLAGS ?= --require-verified --strict-errors
203+
# The cargo profile verify is built with: release locally, dev in CI, where the tests' debug build
204+
# is already cached.
205+
VERIFY_PROFILE ?= release
197206
# Verify reports evidence coverage (docs/coverage.md) and writes it to COVERAGE_REPORT as JSON.
198207
# REQUIRE_COVERAGE=1 fails on any gap; it stays 0 until the generated hazard cases are recorded.
199208
COVERAGE_REPORT ?= target/coverage-report.json
@@ -219,13 +228,83 @@ SQL_SIZE_BASELINE := coverage/sql-size/baseline.json
219228
SQL_SIZE_BASE_FILE := target/sql-size-base.json
220229
SQL_SIZE_ALLOWANCES := coverage/sql-size/allowances.toml
221230

222-
verify-recorded:
231+
verify-recorded: ## Replay every recorded translation on DuckDB; check-convert DIALECT=snowflake runs it.
223232
mkdir -p $(dir $(COVERAGE_REPORT))
224-
cargo run --release --locked -p sqlquack -- verify $(VERIFY_FLAGS) $(COVERAGE_FLAGS) --python $(VERIFY_PYTHON) \
233+
cargo run --profile $(VERIFY_PROFILE) --locked -p sqlquack -- verify $(VERIFY_FLAGS) $(COVERAGE_FLAGS) --python $(VERIFY_PYTHON) \
234+
--cases $(call suite_dir,convert,snowflake) \
225235
--translation-statements $(TRANSLATION_STATEMENTS) --sql-sizes $(SQL_SIZES)
226236
$(MAKE) translation-results
227237
$(MAKE) sql-sizes
228238

239+
# The suites, one pattern for every dialect; README.md, "Running the suites", walks through them.
240+
# DIALECT is always named, so no dialect is the unnamed default. The check- targets replay committed
241+
# evidence offline and need no account; the record- targets need the dialect's engine or account,
242+
# and refuse when RECORDER=0, as in CI.
243+
DIALECT ?=
244+
SUITE_DIALECTS := snowflake duckdb postgres sqlite bigquery databricks clickhouse sqlserver
245+
# Where a suite's committed inputs for a dialect live: $(call suite_dir,<suite>,<dialect>). Every
246+
# suite target reads its paths from here, so moving the inputs changes this one definition.
247+
suite_dir = $(if $(filter syntax,$(1)),$(if $(filter snowflake,$(2)),coverage/syntax,coverage/syntax-$(2)),cases)
248+
SYNTAX_ROOT = $(call suite_dir,syntax,$(DIALECT))
249+
CONVERT_CASES = $(call suite_dir,convert,$(DIALECT))
250+
# The referee that records each dialect's syntax receipts on its engine, with the arguments its
251+
# coverage README gives passed in RECORD_ARGS. Snowflake's capture tool and the Postgres referee are
252+
# not in the repository yet.
253+
RECORD_SYNTAX_duckdb = uv run python -m scripts.run_duckdb_receipts run
254+
RECORD_SYNTAX_sqlite = uv run python -m scripts.run_sqlite_receipts run
255+
RECORD_SYNTAX_bigquery = uv run -q --no-project --with google-auth --with requests python -m scripts.run_bigquery_receipts run
256+
RECORD_SYNTAX_databricks = uv run python -m scripts.run_databricks_receipts run
257+
258+
define suite_dialect
259+
@case " $(SUITE_DIALECTS) " in *" $(DIALECT) "*) ;; *) \
260+
echo "$@: name the dialect, as in make $@ DIALECT=snowflake; one of $(SUITE_DIALECTS)" >&2; exit 2;; esac
261+
endef
262+
263+
define convert_dialect
264+
@test "$(DIALECT)" = snowflake || { \
265+
echo "$@: name DIALECT=snowflake; Snowflake is the only source dialect today, and every conversion targets DuckDB" >&2; exit 2; }
266+
endef
267+
268+
define recording
269+
@test "$(RECORDER)" = 1 || { echo "$@: recording is off when RECORDER=$(RECORDER)" >&2; exit 2; }
270+
endef
271+
272+
help: ## List the suite and gate targets.
273+
@awk -F':.*## ' '/^[a-z][a-z-]*:.*## /{printf " %-15s %s\n", $$1, $$2}' $(firstword $(MAKEFILE_LIST))
274+
275+
pending: ## What DIALECT has not recorded yet, in both suites.
276+
$(suite_dialect)
277+
$(if $(wildcard $(SYNTAX_ROOT)),$(MAKE) --no-print-directory gate-build)
278+
$(if $(wildcard $(SYNTAX_ROOT)),$(GATE_DIR)/syntax-coverage missing --root $(SYNTAX_ROOT),@echo "pending: $(DIALECT) has no syntax suite yet")
279+
$(if $(filter snowflake,$(DIALECT)),cargo run --profile $(VERIFY_PROFILE) --locked -p sqlquack -- pending --cases $(CONVERT_CASES),@echo "pending: $(DIALECT) has no translation suite; Snowflake is the only source dialect today")
280+
281+
check-syntax: ## Rerun DIALECT's parser against its recorded syntax evidence, offline.
282+
$(suite_dialect)
283+
@test -d $(SYNTAX_ROOT) || { echo "check-syntax: $(DIALECT) has no recorded syntax evidence yet" >&2; exit 2; }
284+
@test "$(REQUIRE_BASELINE)" != 1 || test -f $(SYNTAX_ROOT)/baseline.json || { \
285+
echo "check-syntax: $(SYNTAX_ROOT)/baseline.json is missing, so nothing would catch a lost form or obligation" >&2; exit 2; }
286+
$(MAKE) --no-print-directory gate-build
287+
$(GATE_DIR)/syntax-coverage $(if $(wildcard $(SYNTAX_ROOT)/baseline.json),diff --root $(SYNTAX_ROOT) $(SYNTAX_ROOT)/baseline.json,summary --root $(SYNTAX_ROOT))
288+
289+
record-syntax: ## Record DIALECT's syntax receipts on its engine; options in RECORD_ARGS.
290+
$(suite_dialect)
291+
$(recording)
292+
@test $(DIALECT) != snowflake || { \
293+
echo "record-syntax: Snowflake's syntax capture tool is not public yet; it comes into the repository with the evidence move" >&2; exit 2; }
294+
@test -n "$(RECORD_SYNTAX_$(DIALECT))" || { echo "record-syntax: $(DIALECT)'s syntax referee is not in the repository yet" >&2; exit 2; }
295+
$(RECORD_SYNTAX_$(DIALECT)) $(RECORD_ARGS)
296+
297+
check-convert: ## Replay DIALECT's recorded translations on DuckDB, offline; CASES narrows it.
298+
$(convert_dialect)
299+
$(if $(CASES),cargo run --profile $(VERIFY_PROFILE) --locked -p sqlquack -- verify --strict-errors --cases $(CASES) --python $(VERIFY_PYTHON))
300+
$(if $(CASES),,$(MAKE) --no-print-directory verify-recorded)
301+
302+
record-convert: ## Record DIALECT's pending translation cases on Snowflake; options in RECORD_ARGS.
303+
$(convert_dialect)
304+
$(recording)
305+
cargo run --profile $(VERIFY_PROFILE) --locked -p sqlquack -- pending --cases $(CONVERT_CASES)
306+
uv run sqlquack-record $(RECORD_ARGS)
307+
229308
# The generated-SQL size ratchet over the sizes verify wrote; see verify-recorded.
230309
sql-sizes:
231310
@if git show $(RESULTS_BASE):$(SQL_SIZE_BASELINE) > $(SQL_SIZE_BASE_FILE) 2>/dev/null; \
@@ -334,11 +413,11 @@ fuzz-smoke:
334413
done
335414

336415
# Everything CI runs: the static gate, the tests, the recorded-case replay, the exact check and the
337-
# timed conversion on DuckShop, and the fuzz smoke. The pre-push verify hook runs this.
416+
# timed conversion on DuckShop, and the fuzz smoke.
338417
# After the static gate, the independent targets run in parallel, each target's output kept
339-
# together; any failure fails the gate. The verify hook builds what they need beforehand. The fuzz
340-
# smoke and then the SQL timing benchmark run alone afterwards, so each gets the machine to itself.
341-
gate: check-ci $(if $(filter command line,$(origin DUCKSHOP_COMPILED)),,duckshop-compiled)
418+
# together; any failure fails the gate. The fuzz smoke and then the SQL timing benchmark run alone
419+
# afterwards, so each gets the machine to itself.
420+
gate: check-ci $(if $(filter command line,$(origin DUCKSHOP_COMPILED)),,duckshop-compiled) ## Everything: check-ci, the tests, the replay, DuckShop, fuzz and the SQL benchmark.
342421
$(MAKE) -j --output-sync=target gate-parallel
343422
$(MAKE) fuzz-smoke
344423
$(MAKE) sql-benchmark

‎README.md‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -59,6 +59,27 @@ result = sqlquack.convert(
5959

6060
[Python API](https://sqlquack.io/docs/python/)
6161

62+
## Running the suites
63+
64+
SQLQuack's parsers and conversion rules are checked against evidence recorded on the real engines. Rerunning it needs no account:
65+
66+
```bash
67+
git clone https://github.com/sqlstack-io/sqlquack && cd sqlquack
68+
uv sync
69+
make check-syntax DIALECT=snowflake # rerun the parser against recorded syntax evidence
70+
make check-convert DIALECT=snowflake # replay recorded translations on DuckDB
71+
```
72+
73+
| Command | What it does |
74+
|---|---|
75+
| `make pending DIALECT=snowflake` | What isn't recorded yet, in both suites |
76+
| `make check-syntax DIALECT=snowflake` | Rerun the parser against recorded evidence (offline) |
77+
| `make record-syntax DIALECT=duckdb` | Record evidence for new syntax cases |
78+
| `make check-convert DIALECT=snowflake [CASES=cases/temporal]` | Replay recorded translations on DuckDB (offline) |
79+
| `make record-convert DIALECT=snowflake` | Record new translation cases on Snowflake |
80+
81+
Recording needs the dialect's engine: DuckDB and SQLite record locally, while Snowflake, BigQuery and Databricks need a connection. Recording Snowflake syntax evidence isn't public yet. `make help` lists every target.
82+
6283
---
6384

6485
Built in Rust. Free and open source. [Apache 2.0](https://github.com/sqlstack-io/sqlquack/blob/main/LICENSE).

‎docs/architecture.md‎

Lines changed: 28 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -442,6 +442,22 @@ Recorded warehouse results are the only arbiter of correctness. Recording runs o
442442
the maintainer; results are append-only and named by the hash of their case, and everything that
443443
could be weakened to make a failing case pass lives in one reviewed policy file.
444444

445+
### Running the suites
446+
447+
Each suite has one pattern for every dialect, and `make help` lists the targets. The check- targets
448+
replay committed evidence offline; the record- targets need the dialect's engine or account and
449+
refuse when `RECORDER=0`. Every target needs `DIALECT`; no dialect is the default.
450+
451+
| Target | What it does |
452+
|---|---|
453+
| `make pending DIALECT=<d>` | Lists what is not recorded yet: the syntax suite's unmet obligations (`syntax-coverage missing`) and, for Snowflake, writes the translation suite's pending cases. |
454+
| `make check-syntax DIALECT=<d>` | Reparses every recorded syntax case of `coverage/syntax-<d>` (`coverage/syntax` for Snowflake) and compares with its receipts: `syntax-coverage diff` against the baseline where there is one, `summary` otherwise. `make check-ci` runs it for Snowflake, SQLite and DuckDB. |
455+
| `make record-syntax DIALECT=<d>` | Runs the dialect's syntax referee, `scripts/run_<d>_receipts.py run`, with its arguments in `RECORD_ARGS`. Snowflake's capture tool and the Postgres referee are not in the repository yet, so it refuses those. |
456+
| `make check-convert DIALECT=snowflake [CASES=<folder>]` | Replays recorded translations on DuckDB (`make verify-recorded`, or only `CASES`). |
457+
| `make record-convert DIALECT=snowflake` | Writes the pending cases and records them on Snowflake (Flow, below). |
458+
459+
Every conversion targets DuckDB, so only the source dialect is named; Snowflake is the only one today.
460+
445461
### Cases
446462

447463
Case files live under `cases/<category>/<construct>.toml`. Each file names an edge-case category
@@ -607,8 +623,11 @@ setting reaches the repository. A pinned parameter whose account default differs
607623

608624
### Flow
609625

610-
1. `sqlquack pending` writes `evidence/pending/<hash>.json` for every case without a result.
611-
2. The maintainer runs `sqlquack-record`. Before running anything it compiles every pending
626+
1. `make pending DIALECT=snowflake` (`sqlquack pending`) writes `evidence/pending/<hash>.json` for
627+
every case without a result.
628+
2. The maintainer runs `make record-convert DIALECT=snowflake` (`sqlquack pending`, then
629+
`sqlquack-record` with any options in `RECORD_ARGS`) from a machine with a Snowflake connection;
630+
CI never records, and the target refuses when `RECORDER=0`. Before running anything it compiles every pending
612631
case's SQL with `EXPLAIN`, which runs nothing and creates nothing, and when any fails with a
613632
SQLSTATE class 0A or 42 error it stops and lists them, so a mistake such as a reserved word used
614633
as an alias costs no recording. A case that records Snowflake refusing to compile its SQL
@@ -637,8 +656,12 @@ setting reaches the repository. A pinned parameter whose account default differs
637656
(the outermost `input` CTE and an `ORDER BY ordinal` that only keeps rows in input order) is
638657
converted with provenance muted, so its casts, literals and set operation count neither toward
639658
claims nor toward the error check's declared failures; a query's own CTEs follow the fixture's
640-
in the same `WITH` and do count. The pre-push gate
641-
(`make gate`) runs `verify --require-verified`, which fails while any claim is unverified.
659+
in the same `WITH` and do count. `make check-convert DIALECT=snowflake` runs `make
660+
verify-recorded`, which runs `verify --require-verified --strict-errors` and fails while any
661+
claim is unverified; CI and `make gate` both run it, so both use those flags. `CASES=<folder>`
662+
replays only that folder, with `--strict-errors` alone. `make check-merge` does not replay
663+
recorded cases. A new case's results are committed with the case once `make check-convert
664+
DIALECT=snowflake` passes.
642665
Where Snowflake and DuckDB both raised, verify also checks the errors (see Error checks).
643666
4. `sqlquack review` writes every case as a sqllogictest-style record (`source snowflake`,
644667
`converted duckdb` or `refused`, then `query snowflake nosort|rowsort` with the column names and
@@ -711,5 +734,4 @@ For a passed case where both raised, verify reports `agrees` (same kind, and a r
711734
conversion used declares the recorded error), `divergence_raise` (DuckDB raised a known
712735
divergence's guard), or a finding: `errno_mismatch` (a block tagged another errno),
713736
`kind_mismatch`, `undeclared` or `unknown_snowflake_error`. Findings are reported and counted;
714-
`verify --strict-errors` fails on them. The gate does not pass that flag until the existing
715-
error cases are re-recorded with their messages.
737+
`verify --strict-errors` fails on them; `make verify-recorded`, and so CI and `make gate`, pass it.

0 commit comments

Comments
 (0)