From 46bd991084505d8cbd060be7cb4e27b38204a49e Mon Sep 17 00:00:00 2001 From: Josh Taillon Date: Tue, 1 Sep 2026 21:15:37 -0600 Subject: [PATCH 1/4] test: one conformance corpus and one definitions contract for both packages The 14 YAML fixtures both packages need move from pkg-r's test fixtures to tests/shared/, so there is one copy rather than a second hand-maintained one in pkg-py. The R suite reads the synced copy, as it does for every shared fixture. tests/shared/definitions.json pins what both packages agree to consume: the export-record contract, the grain metadata call_metrics needs for its mixed-grain guard, and the data-dict problem code each invalid fixture must produce. export_records is generated from the data-dict binary at the pinned commit and the generator refuses to run against any other build, since a fixture from a different revision would bless whatever that build does. mixed_grain comes from the typed IR rather than the export, so it is hand-maintained and the generator preserves it. The fixture does not replace the conformance harness. That harness compares against a real binary; this pins the contract. --- pkg-py/tests/test_definitions_fixture.py | 106 +++ .../invalid/between-temporal.yaml | 0 .../invalid/columns-non-filter.yaml | 0 .../invalid/columns-transitive.yaml | 0 .../definition-export/invalid/cycle.yaml | 0 .../definition-export/invalid/duplicate.yaml | 0 .../invalid/nested-aggregate.yaml | 0 .../definition-export/invalid/parse.yaml | 0 .../invalid/regex-engine.yaml | 0 .../definition-export/invalid/shadow.yaml | 0 .../definition-export/invalid/type.yaml | 0 .../definition-export/invalid/unknown.yaml | 0 .../definition-export/valid/core.yaml | 0 .../definition-export/valid/functions.yaml | 0 .../definition-export/valid/language.yaml | 0 .../testthat/fixtures/shared/definitions.json | 743 ++++++++++++++++++ pkg-r/tests/testthat/helper-data-dict.R | 24 +- .../tests/testthat/test-definition-compile.R | 2 +- pkg-r/tests/testthat/test-definition-export.R | 10 +- scripts/generate-definitions-fixture.sh | 87 ++ tests/shared/README.md | 2 +- .../invalid/between-temporal.yaml | 14 + .../invalid/columns-non-filter.yaml | 11 + .../invalid/columns-transitive.yaml | 16 + .../definition-export/invalid/cycle.yaml | 13 + .../definition-export/invalid/duplicate.yaml | 13 + .../invalid/nested-aggregate.yaml | 11 + .../definition-export/invalid/parse.yaml | 11 + .../invalid/regex-engine.yaml | 11 + .../definition-export/invalid/shadow.yaml | 11 + .../definition-export/invalid/type.yaml | 11 + .../definition-export/invalid/unknown.yaml | 11 + .../shared/definition-export/valid/core.yaml | 25 + .../definition-export/valid/functions.yaml | 28 + .../definition-export/valid/language.yaml | 109 +++ tests/shared/definitions.json | 743 ++++++++++++++++++ 36 files changed, 1991 insertions(+), 21 deletions(-) create mode 100644 pkg-py/tests/test_definitions_fixture.py rename pkg-r/tests/testthat/fixtures/{ => shared}/definition-export/invalid/between-temporal.yaml (100%) rename pkg-r/tests/testthat/fixtures/{ => shared}/definition-export/invalid/columns-non-filter.yaml (100%) rename pkg-r/tests/testthat/fixtures/{ => shared}/definition-export/invalid/columns-transitive.yaml (100%) rename pkg-r/tests/testthat/fixtures/{ => shared}/definition-export/invalid/cycle.yaml (100%) rename pkg-r/tests/testthat/fixtures/{ => shared}/definition-export/invalid/duplicate.yaml (100%) rename pkg-r/tests/testthat/fixtures/{ => shared}/definition-export/invalid/nested-aggregate.yaml (100%) rename pkg-r/tests/testthat/fixtures/{ => shared}/definition-export/invalid/parse.yaml (100%) rename pkg-r/tests/testthat/fixtures/{ => shared}/definition-export/invalid/regex-engine.yaml (100%) rename pkg-r/tests/testthat/fixtures/{ => shared}/definition-export/invalid/shadow.yaml (100%) rename pkg-r/tests/testthat/fixtures/{ => shared}/definition-export/invalid/type.yaml (100%) rename pkg-r/tests/testthat/fixtures/{ => shared}/definition-export/invalid/unknown.yaml (100%) rename pkg-r/tests/testthat/fixtures/{ => shared}/definition-export/valid/core.yaml (100%) rename pkg-r/tests/testthat/fixtures/{ => shared}/definition-export/valid/functions.yaml (100%) rename pkg-r/tests/testthat/fixtures/{ => shared}/definition-export/valid/language.yaml (100%) create mode 100644 pkg-r/tests/testthat/fixtures/shared/definitions.json create mode 100755 scripts/generate-definitions-fixture.sh create mode 100644 tests/shared/definition-export/invalid/between-temporal.yaml create mode 100644 tests/shared/definition-export/invalid/columns-non-filter.yaml create mode 100644 tests/shared/definition-export/invalid/columns-transitive.yaml create mode 100644 tests/shared/definition-export/invalid/cycle.yaml create mode 100644 tests/shared/definition-export/invalid/duplicate.yaml create mode 100644 tests/shared/definition-export/invalid/nested-aggregate.yaml create mode 100644 tests/shared/definition-export/invalid/parse.yaml create mode 100644 tests/shared/definition-export/invalid/regex-engine.yaml create mode 100644 tests/shared/definition-export/invalid/shadow.yaml create mode 100644 tests/shared/definition-export/invalid/type.yaml create mode 100644 tests/shared/definition-export/invalid/unknown.yaml create mode 100644 tests/shared/definition-export/valid/core.yaml create mode 100644 tests/shared/definition-export/valid/functions.yaml create mode 100644 tests/shared/definition-export/valid/language.yaml create mode 100644 tests/shared/definitions.json diff --git a/pkg-py/tests/test_definitions_fixture.py b/pkg-py/tests/test_definitions_fixture.py new file mode 100644 index 00000000..5da0b1ee --- /dev/null +++ b/pkg-py/tests/test_definitions_fixture.py @@ -0,0 +1,106 @@ +"""The definitions contract both packages consume. + +This checks the fixture's own integrity: that it covers the corpus, that its +sections agree with each other, and that it is not empty. Running the cases +against an implementation comes with the registry and the compiler. +""" + +from typing import Any + +import yaml + +from ._shared import SHARED_DIR, load_shared_fixture + +SPEC = load_shared_fixture("definitions") +CORPUS = SHARED_DIR / SPEC["corpus_dir"] +EXPORTS: dict[str, dict[str, Any]] = SPEC["export_records"] + + +def test_the_fixture_is_not_empty() -> None: + # An empty fixture would collect zero cases and every runner would pass. + assert EXPORTS + assert sum(len(cases) for cases in EXPORTS.values()) > 0 + assert SPEC["invalid"] + + +def test_every_valid_corpus_file_has_records() -> None: + on_disk = {path.name for path in (CORPUS / "valid").glob("*.yaml")} + + assert on_disk == set(EXPORTS) + + +def test_every_invalid_corpus_file_has_an_error_code() -> None: + on_disk = {path.name for path in (CORPUS / "invalid").glob("*.yaml")} + + assert on_disk == set(SPEC["invalid"]) + + +def test_grain_metadata_covers_every_exported_definition() -> None: + for name, cases in EXPORTS.items(): + assert set(SPEC["mixed_grain"][name]) == set(cases), name + + +def test_grain_metadata_pins_at_least_one_mixed_case() -> None: + # All-false grain would pin nothing: the guard it feeds would look correct + # while never having been exercised. + values = [ + value for cases in SPEC["mixed_grain"].values() for value in cases.values() + ] + assert any(values) + assert not all(values) + + +def test_every_record_carries_a_duckdb_translation() -> None: + for cases in EXPORTS.values(): + for key, record in cases.items(): + assert record["translation"]["target"] == "SQL(duckdb)", key + assert record["translation"]["code"], key + assert record["kind"] in {"metric", "filter", "derived"}, key + + +def test_every_record_names_the_expression_it_came_from() -> None: + for cases in EXPORTS.values(): + for key, record in cases.items(): + assert record["expression"], key + + +def test_a_definition_with_no_single_inferred_type_carries_none() -> None: + # data-dict omits `type` when an expression infers no single one, such as + # a CASE over both a date and a datetime column. Pinned because the + # obvious implementation invents a type instead of leaving it out. + typeless = { + key + for cases in EXPORTS.values() + for key, record in cases.items() + if record["type"] is None + } + + assert "survey::mixed temporal case" in typeless + assert all( + record["type"] is None or isinstance(record["type"], str) + for cases in EXPORTS.values() + for record in cases.values() + ) + + +def test_records_match_the_definitions_authored_in_the_corpus() -> None: + # The fixture is generated, so this catches a stale copy rather than a + # wrong one: a definition added to the corpus without regenerating. + for name, cases in EXPORTS.items(): + document = yaml.safe_load( + (CORPUS / "valid" / name).read_text(encoding="utf-8") + ) + authored = { + f"{table['name']}::{definition['name']}" + for table in document.get("tables", []) + for definition in table.get("definitions") or [] + } + assert authored == set(cases), name + + +def test_referenced_definitions_resolve_within_their_fixture() -> None: + for name, cases in EXPORTS.items(): + names = {key.split("::", 1)[1] for key in cases} + for key, record in cases.items(): + for referenced in record["definitions"]: + assert referenced in names, f"{name} {key} -> {referenced}" diff --git a/pkg-r/tests/testthat/fixtures/definition-export/invalid/between-temporal.yaml b/pkg-r/tests/testthat/fixtures/shared/definition-export/invalid/between-temporal.yaml similarity index 100% rename from pkg-r/tests/testthat/fixtures/definition-export/invalid/between-temporal.yaml rename to pkg-r/tests/testthat/fixtures/shared/definition-export/invalid/between-temporal.yaml diff --git a/pkg-r/tests/testthat/fixtures/definition-export/invalid/columns-non-filter.yaml b/pkg-r/tests/testthat/fixtures/shared/definition-export/invalid/columns-non-filter.yaml similarity index 100% rename from pkg-r/tests/testthat/fixtures/definition-export/invalid/columns-non-filter.yaml rename to pkg-r/tests/testthat/fixtures/shared/definition-export/invalid/columns-non-filter.yaml diff --git a/pkg-r/tests/testthat/fixtures/definition-export/invalid/columns-transitive.yaml b/pkg-r/tests/testthat/fixtures/shared/definition-export/invalid/columns-transitive.yaml similarity index 100% rename from pkg-r/tests/testthat/fixtures/definition-export/invalid/columns-transitive.yaml rename to pkg-r/tests/testthat/fixtures/shared/definition-export/invalid/columns-transitive.yaml diff --git a/pkg-r/tests/testthat/fixtures/definition-export/invalid/cycle.yaml b/pkg-r/tests/testthat/fixtures/shared/definition-export/invalid/cycle.yaml similarity index 100% rename from pkg-r/tests/testthat/fixtures/definition-export/invalid/cycle.yaml rename to pkg-r/tests/testthat/fixtures/shared/definition-export/invalid/cycle.yaml diff --git a/pkg-r/tests/testthat/fixtures/definition-export/invalid/duplicate.yaml b/pkg-r/tests/testthat/fixtures/shared/definition-export/invalid/duplicate.yaml similarity index 100% rename from pkg-r/tests/testthat/fixtures/definition-export/invalid/duplicate.yaml rename to pkg-r/tests/testthat/fixtures/shared/definition-export/invalid/duplicate.yaml diff --git a/pkg-r/tests/testthat/fixtures/definition-export/invalid/nested-aggregate.yaml b/pkg-r/tests/testthat/fixtures/shared/definition-export/invalid/nested-aggregate.yaml similarity index 100% rename from pkg-r/tests/testthat/fixtures/definition-export/invalid/nested-aggregate.yaml rename to pkg-r/tests/testthat/fixtures/shared/definition-export/invalid/nested-aggregate.yaml diff --git a/pkg-r/tests/testthat/fixtures/definition-export/invalid/parse.yaml b/pkg-r/tests/testthat/fixtures/shared/definition-export/invalid/parse.yaml similarity index 100% rename from pkg-r/tests/testthat/fixtures/definition-export/invalid/parse.yaml rename to pkg-r/tests/testthat/fixtures/shared/definition-export/invalid/parse.yaml diff --git a/pkg-r/tests/testthat/fixtures/definition-export/invalid/regex-engine.yaml b/pkg-r/tests/testthat/fixtures/shared/definition-export/invalid/regex-engine.yaml similarity index 100% rename from pkg-r/tests/testthat/fixtures/definition-export/invalid/regex-engine.yaml rename to pkg-r/tests/testthat/fixtures/shared/definition-export/invalid/regex-engine.yaml diff --git a/pkg-r/tests/testthat/fixtures/definition-export/invalid/shadow.yaml b/pkg-r/tests/testthat/fixtures/shared/definition-export/invalid/shadow.yaml similarity index 100% rename from pkg-r/tests/testthat/fixtures/definition-export/invalid/shadow.yaml rename to pkg-r/tests/testthat/fixtures/shared/definition-export/invalid/shadow.yaml diff --git a/pkg-r/tests/testthat/fixtures/definition-export/invalid/type.yaml b/pkg-r/tests/testthat/fixtures/shared/definition-export/invalid/type.yaml similarity index 100% rename from pkg-r/tests/testthat/fixtures/definition-export/invalid/type.yaml rename to pkg-r/tests/testthat/fixtures/shared/definition-export/invalid/type.yaml diff --git a/pkg-r/tests/testthat/fixtures/definition-export/invalid/unknown.yaml b/pkg-r/tests/testthat/fixtures/shared/definition-export/invalid/unknown.yaml similarity index 100% rename from pkg-r/tests/testthat/fixtures/definition-export/invalid/unknown.yaml rename to pkg-r/tests/testthat/fixtures/shared/definition-export/invalid/unknown.yaml diff --git a/pkg-r/tests/testthat/fixtures/definition-export/valid/core.yaml b/pkg-r/tests/testthat/fixtures/shared/definition-export/valid/core.yaml similarity index 100% rename from pkg-r/tests/testthat/fixtures/definition-export/valid/core.yaml rename to pkg-r/tests/testthat/fixtures/shared/definition-export/valid/core.yaml diff --git a/pkg-r/tests/testthat/fixtures/definition-export/valid/functions.yaml b/pkg-r/tests/testthat/fixtures/shared/definition-export/valid/functions.yaml similarity index 100% rename from pkg-r/tests/testthat/fixtures/definition-export/valid/functions.yaml rename to pkg-r/tests/testthat/fixtures/shared/definition-export/valid/functions.yaml diff --git a/pkg-r/tests/testthat/fixtures/definition-export/valid/language.yaml b/pkg-r/tests/testthat/fixtures/shared/definition-export/valid/language.yaml similarity index 100% rename from pkg-r/tests/testthat/fixtures/definition-export/valid/language.yaml rename to pkg-r/tests/testthat/fixtures/shared/definition-export/valid/language.yaml diff --git a/pkg-r/tests/testthat/fixtures/shared/definitions.json b/pkg-r/tests/testthat/fixtures/shared/definitions.json new file mode 100644 index 00000000..a7e19ed3 --- /dev/null +++ b/pkg-r/tests/testthat/fixtures/shared/definitions.json @@ -0,0 +1,743 @@ +{ + "corpus_dir": "definition-export", + "data_dict_commit": "d950c5ac90d0ab939d330600f3a5ee1bfde0f604", + "export_records": { + "core.yaml": { + "orders::enterprise_revenue": { + "columns": [ + "order_total" + ], + "definitions": [ + "is_enterprise" + ], + "expression": "SUM(CASE WHEN is_enterprise THEN order_total ELSE 0 END)", + "kind": "metric", + "translation": { + "code": "sum(CASE WHEN \"is_enterprise\" THEN \"order_total\" ELSE 0 END)", + "error": null, + "notes": [ + "DuckDB sums integers at 128 bits, so a total data-dict reports as an overflow (D09) may succeed." + ], + "target": "SQL(duckdb)" + }, + "type": "number" + }, + "orders::is_enterprise": { + "columns": [ + "tile_size" + ], + "definitions": [], + "expression": "tile_size IN ('Mid-Market-3', 'Enterprise-1')", + "kind": "filter", + "translation": { + "code": "\"tile_size\" IN ('Mid-Market-3', 'Enterprise-1')", + "error": null, + "notes": [], + "target": "SQL(duckdb)" + }, + "type": "boolean" + }, + "orders::list_price": { + "columns": [ + "order_total" + ], + "definitions": [], + "expression": "order_total * 1.2", + "kind": "derived", + "translation": { + "code": "\"order_total\" * 1.2", + "error": null, + "notes": [], + "target": "SQL(duckdb)" + }, + "type": "number" + }, + "orders::net_revenue": { + "columns": [ + "status_cd", + "order_total" + ], + "definitions": [], + "expression": "SUM(CASE WHEN status_cd = 90 THEN 0 ELSE order_total END)", + "kind": "metric", + "translation": { + "code": "sum(CASE WHEN \"status_cd\" = 90 THEN 0 ELSE \"order_total\" END)", + "error": null, + "notes": [ + "DuckDB compares a NaN as equal to itself and greater than every number, where data-dict answers false; a row holding one passes here and is reported there.", + "DuckDB sums integers at 128 bits, so a total data-dict reports as an overflow (D09) may succeed." + ], + "target": "SQL(duckdb)" + }, + "type": "number" + } + }, + "functions.yaml": { + "values::boolean folds": { + "columns": [ + "flag" + ], + "definitions": [], + "expression": "ANY(flag) OR ALL(flag)", + "kind": "metric", + "translation": { + "code": "bool_or(\"flag\") OR bool_and(\"flag\")", + "error": null, + "notes": [], + "target": "SQL(duckdb)" + }, + "type": "boolean" + }, + "values::finite": { + "columns": [ + "number" + ], + "definitions": [], + "expression": "IS_FINITE(number) AND NOT IS_INFINITE(number) AND NOT IS_NAN(number)", + "kind": "filter", + "translation": { + "code": "isfinite(\"number\") AND NOT isinf(\"number\") AND NOT isnan(\"number\")", + "error": null, + "notes": [], + "target": "SQL(duckdb)" + }, + "type": "boolean" + }, + "values::folds": { + "columns": [ + "number" + ], + "definitions": [], + "expression": "MIN(number) + MAX(number) + SUM(number) + AVG(number) + COUNT(number) + COUNT_DISTINCT(number) + ROW_COUNT()", + "kind": "metric", + "translation": { + "code": "min(\"number\") + max(\"number\") + sum(\"number\") + avg(\"number\") + count(\"number\") + count(DISTINCT \"number\") + count(*)", + "error": null, + "notes": [ + "DuckDB sums integers at 128 bits, so a total data-dict reports as an overflow (D09) may succeed." + ], + "target": "SQL(duckdb)" + }, + "type": "number" + }, + "values::numeric": { + "columns": [ + "number" + ], + "definitions": [], + "expression": "ABS(number) + FLOOR(number) + CEIL(number) + ROUND(number, 2)", + "kind": "derived", + "translation": { + "code": "abs(\"number\") + floor(\"number\") + ceil(\"number\") + round(\"number\", 2)", + "error": null, + "notes": [], + "target": "SQL(duckdb)" + }, + "type": "number" + }, + "values::patterns": { + "columns": [ + "text" + ], + "definitions": [], + "expression": "text LIKE 'A_%' AND text SIMILAR TO 'A.*'", + "kind": "filter", + "translation": { + "code": "regexp_full_match(\"text\", '^A..*$') AND regexp_full_match(\"text\", 'A.*')", + "error": null, + "notes": [], + "target": "SQL(duckdb)" + }, + "type": "boolean" + }, + "values::remainder": { + "columns": [ + "number" + ], + "definitions": [], + "expression": "MOD(number, 3)", + "kind": "derived", + "translation": { + "code": "mod(mod(\"number\", 3) + 3, 3)", + "error": null, + "notes": [ + "DuckDB yields null for an integer modulus by zero, where data-dict yields a NaN." + ], + "target": "SQL(duckdb)" + }, + "type": "number" + }, + "values::strings": { + "columns": [ + "text" + ], + "definitions": [], + "expression": "STARTS_WITH(LOWER(TRIM(text)), 'a') OR ENDS_WITH(UPPER(text), 'Z')", + "kind": "filter", + "translation": { + "code": "starts_with(lower(trim(\"text\")), 'a') OR ends_with(upper(\"text\"), 'Z')", + "error": null, + "notes": [], + "target": "SQL(duckdb)" + }, + "type": "boolean" + } + }, + "language.yaml": { + "survey::amount band": { + "columns": [ + "amount" + ], + "definitions": [], + "expression": "amount <= 2 * MIN(amount)", + "kind": "filter", + "translation": { + "code": "\"amount\" <= 2 * min(\"amount\")", + "error": null, + "notes": [ + "DuckDB compares a NaN as equal to itself and greater than every number, where data-dict answers false; a row holding one passes here and is reported there." + ], + "target": "SQL(duckdb)" + }, + "type": "boolean" + }, + "survey::anything missing": { + "columns": [ + "q1", + "q2", + "created", + "observed", + "amount", + "ratio", + "pattern", + "category", + "profile", + "tags" + ], + "definitions": [], + "expression": "COLUMNS(*) IS NOT NULL", + "kind": "filter", + "translation": { + "code": "\"q1\" IS NOT NULL AND \"q2\" IS NOT NULL AND \"created\" IS NOT NULL AND \"observed\" IS NOT NULL AND \"amount\" IS NOT NULL AND \"ratio\" IS NOT NULL AND \"pattern\" IS NOT NULL AND \"category\" IS NOT NULL AND \"profile\" IS NOT NULL AND \"tags\" IS NOT NULL AND \"untyped\" IS NOT NULL", + "error": null, + "notes": [], + "target": "SQL(duckdb)" + }, + "type": "boolean" + }, + "survey::case without else": { + "columns": [ + "q1", + "amount" + ], + "definitions": [], + "expression": "CASE WHEN q1 THEN amount END", + "kind": "derived", + "translation": { + "code": "CASE WHEN \"q1\" THEN \"amount\" END", + "error": null, + "notes": [], + "target": "SQL(duckdb)" + }, + "type": "number" + }, + "survey::category A": { + "columns": [ + "category" + ], + "definitions": [], + "expression": "category = 'A'", + "kind": "filter", + "translation": { + "code": "\"category\" = 'A'", + "error": null, + "notes": [], + "target": "SQL(duckdb)" + }, + "type": "boolean" + }, + "survey::complete": { + "columns": [ + "q1", + "q2" + ], + "definitions": [], + "expression": "COLUMNS([q1, q2]) IS NOT NULL", + "kind": "filter", + "translation": { + "code": "\"q1\" IS NOT NULL AND \"q2\" IS NOT NULL", + "error": null, + "notes": [], + "target": "SQL(duckdb)" + }, + "type": "boolean" + }, + "survey::computed interval": { + "columns": [ + "observed", + "amount" + ], + "definitions": [], + "expression": "observed + interval(amount + 1, days)", + "kind": "derived", + "translation": { + "code": "\"observed\" + (\"amount\" + 1 * INTERVAL '1 days')", + "error": null, + "notes": [], + "target": "SQL(duckdb)" + }, + "type": "datetime" + }, + "survey::constant": { + "columns": [], + "definitions": [], + "expression": "1", + "kind": "metric", + "translation": { + "code": "1", + "error": null, + "notes": [], + "target": "SQL(duckdb)" + }, + "type": "number" + }, + "survey::deep score": { + "columns": [ + "profile.geo.latitude", + "profile.nick names" + ], + "definitions": [], + "expression": "profile.geo.latitude + LENGTH(profile.`nick names`)", + "kind": "derived", + "translation": { + "code": "\"profile\".\"geo\".\"latitude\" + length(\"profile\".\"nick names\")", + "error": null, + "notes": [], + "target": "SQL(duckdb)" + }, + "type": "number" + }, + "survey::dynamic match": { + "columns": [ + "pattern", + "profile.zip" + ], + "definitions": [], + "expression": "pattern LIKE profile.zip", + "kind": "filter", + "translation": { + "code": "\"pattern\" LIKE \"profile\".\"zip\"", + "error": null, + "notes": [], + "target": "SQL(duckdb)" + }, + "type": "boolean" + }, + "survey::exact match": { + "columns": [ + "pattern" + ], + "definitions": [], + "expression": "pattern LIKE 'a'", + "kind": "filter", + "translation": { + "code": "\"pattern\" = 'a'", + "error": null, + "notes": [], + "target": "SQL(duckdb)" + }, + "type": "boolean" + }, + "survey::fractional interval": { + "columns": [ + "observed" + ], + "definitions": [], + "expression": "observed - interval(1.5, hours)", + "kind": "derived", + "translation": { + "code": "\"observed\" - (1.5 * INTERVAL '1 hours')", + "error": null, + "notes": [], + "target": "SQL(duckdb)" + }, + "type": "datetime" + }, + "survey::fractional time": { + "columns": [ + "observed" + ], + "definitions": [], + "expression": "observed <= '2024-01-01T01:30:00.123Z'", + "kind": "filter", + "translation": { + "code": "\"observed\" <= TIMESTAMP '2024-01-01 01:30:00.123'", + "error": null, + "notes": [], + "target": "SQL(duckdb)" + }, + "type": "boolean" + }, + "survey::fresh": { + "columns": [ + "observed" + ], + "definitions": [], + "expression": "observed >= NOW() - interval(2, weeks)", + "kind": "filter", + "translation": { + "code": "\"observed\" >= current_timestamp - INTERVAL '2 weeks'", + "error": null, + "notes": [], + "target": "SQL(duckdb)" + }, + "type": "boolean" + }, + "survey::long postal": { + "columns": [], + "definitions": [ + "postal length" + ], + "expression": "`postal length` > 5", + "kind": "filter", + "translation": { + "code": "\"postal length\" > 5", + "error": null, + "notes": [ + "DuckDB compares a NaN as equal to itself and greater than every number, where data-dict answers false; a row holding one passes here and is reported there." + ], + "target": "SQL(duckdb)" + }, + "type": "boolean" + }, + "survey::missing tags": { + "columns": [ + "tags" + ], + "definitions": [], + "expression": "tags IS NULL", + "kind": "filter", + "translation": { + "code": "\"tags\" IS NULL", + "error": null, + "notes": [], + "target": "SQL(duckdb)" + }, + "type": "boolean" + }, + "survey::missing unknown": { + "columns": [ + "untyped" + ], + "definitions": [], + "expression": "untyped IS NULL", + "kind": "filter", + "translation": { + "code": "\"untyped\" IS NULL", + "error": null, + "notes": [], + "target": "SQL(duckdb)" + }, + "type": "boolean" + }, + "survey::mixed temporal case": { + "columns": [ + "q1", + "created", + "observed" + ], + "definitions": [], + "expression": "CASE WHEN q1 THEN created ELSE observed END", + "kind": "derived", + "translation": { + "code": "CASE WHEN \"q1\" THEN \"created\" ELSE \"observed\" END", + "error": null, + "notes": [], + "target": "SQL(duckdb)" + }, + "type": null + }, + "survey::negative infinity": { + "columns": [ + "amount" + ], + "definitions": [], + "expression": "amount > -INF", + "kind": "filter", + "translation": { + "code": "\"amount\" > -CAST('Infinity' AS DOUBLE)", + "error": null, + "notes": [ + "DuckDB compares a NaN as equal to itself and greater than every number, where data-dict answers false; a row holding one passes here and is reported there." + ], + "target": "SQL(duckdb)" + }, + "type": "boolean" + }, + "survey::negative quotient": { + "columns": [ + "amount", + "ratio" + ], + "definitions": [], + "expression": "-(amount - 2) / (ratio + 1.0)", + "kind": "derived", + "translation": { + "code": "-(\"amount\" - 2) / (\"ratio\" + 1.0)", + "error": null, + "notes": [], + "target": "SQL(duckdb)" + }, + "type": "number" + }, + "survey::null logic": { + "columns": [ + "q1" + ], + "definitions": [], + "expression": "NOT q1 OR NULL IS NULL", + "kind": "filter", + "translation": { + "code": "NOT \"q1\" OR NULL IS NULL", + "error": null, + "notes": [], + "target": "SQL(duckdb)" + }, + "type": "boolean" + }, + "survey::nullable date shift": { + "columns": [ + "created" + ], + "definitions": [], + "expression": "created + NULL", + "kind": "derived", + "translation": { + "code": "\"created\" + NULL", + "error": null, + "notes": [], + "target": "SQL(duckdb)" + }, + "type": "datetime" + }, + "survey::offset time": { + "columns": [ + "observed" + ], + "definitions": [], + "expression": "observed >= '2024-01-01T01:30:00-06:00'", + "kind": "filter", + "translation": { + "code": "\"observed\" >= TIMESTAMP '2024-01-01 07:30:00'", + "error": null, + "notes": [], + "target": "SQL(duckdb)" + }, + "type": "boolean" + }, + "survey::postal length": { + "columns": [ + "profile.zip" + ], + "definitions": [], + "expression": "LENGTH(profile.zip)", + "kind": "derived", + "translation": { + "code": "length(\"profile\".\"zip\")", + "error": null, + "notes": [], + "target": "SQL(duckdb)" + }, + "type": "number" + }, + "survey::precise threshold": { + "columns": [ + "amount" + ], + "definitions": [], + "expression": "amount > 0.12345678901234567", + "kind": "filter", + "translation": { + "code": "\"amount\" > 0.12345678901234566", + "error": null, + "notes": [ + "DuckDB compares a NaN as equal to itself and greater than every number, where data-dict answers false; a row holding one passes here and is reported there." + ], + "target": "SQL(duckdb)" + }, + "type": "boolean" + }, + "survey::prefix match": { + "columns": [ + "pattern" + ], + "definitions": [], + "expression": "pattern LIKE 'a%'", + "kind": "filter", + "translation": { + "code": "starts_with(\"pattern\", 'a')", + "error": null, + "notes": [], + "target": "SQL(duckdb)" + }, + "type": "boolean" + }, + "survey::range predicate": { + "columns": [ + "amount" + ], + "definitions": [], + "expression": "amount BETWEEN -10 AND 10 AND amount NOT IN (0, 1)", + "kind": "filter", + "translation": { + "code": "\"amount\" BETWEEN -10 AND 10 AND \"amount\" NOT IN (0, 1)", + "error": null, + "notes": [ + "DuckDB compares a NaN as equal to itself and greater than every number, where data-dict answers false; a row holding one passes here and is reported there." + ], + "target": "SQL(duckdb)" + }, + "type": "boolean" + }, + "survey::recent": { + "columns": [ + "created" + ], + "definitions": [], + "expression": "created >= '2024-01-01'", + "kind": "filter", + "translation": { + "code": "\"created\" >= DATE '2024-01-01'", + "error": null, + "notes": [], + "target": "SQL(duckdb)" + }, + "type": "boolean" + }, + "survey::right associative arithmetic": { + "columns": [ + "amount", + "ratio" + ], + "definitions": [], + "expression": "amount - (ratio - 1)", + "kind": "derived", + "translation": { + "code": "\"amount\" - (\"ratio\" - 1)", + "error": null, + "notes": [], + "target": "SQL(duckdb)" + }, + "type": "number" + }, + "survey::selected dates": { + "columns": [ + "created" + ], + "definitions": [], + "expression": "COLUMNS('^created$') >= '2020-01-01'", + "kind": "filter", + "translation": { + "code": "\"created\" >= '2020-01-01'", + "error": null, + "notes": [], + "target": "SQL(duckdb)" + }, + "type": "boolean" + }, + "survey::suffix mismatch": { + "columns": [ + "pattern" + ], + "definitions": [], + "expression": "pattern NOT LIKE '%z'", + "kind": "filter", + "translation": { + "code": "NOT ends_with(\"pattern\", 'z')", + "error": null, + "notes": [], + "target": "SQL(duckdb)" + }, + "type": "boolean" + }, + "survey::untyped count": { + "columns": [ + "untyped" + ], + "definitions": [], + "expression": "COUNT(untyped)", + "kind": "metric", + "translation": { + "code": "count(\"untyped\")", + "error": null, + "notes": [], + "target": "SQL(duckdb)" + }, + "type": "number" + } + } + }, + "invalid": { + "between-temporal.yaml": "S21", + "columns-non-filter.yaml": "S21", + "columns-transitive.yaml": "S21", + "cycle.yaml": "S34", + "duplicate.yaml": "S10", + "nested-aggregate.yaml": "S30", + "parse.yaml": "S19", + "regex-engine.yaml": "S21", + "shadow.yaml": "S33", + "type.yaml": "S21", + "unknown.yaml": "S20" + }, + "mixed_grain": { + "core.yaml": { + "orders::enterprise_revenue": false, + "orders::is_enterprise": false, + "orders::list_price": false, + "orders::net_revenue": false + }, + "functions.yaml": { + "values::boolean folds": false, + "values::finite": false, + "values::folds": false, + "values::numeric": false, + "values::patterns": false, + "values::remainder": false, + "values::strings": false + }, + "language.yaml": { + "survey::amount band": true, + "survey::anything missing": false, + "survey::case without else": false, + "survey::category A": false, + "survey::complete": false, + "survey::computed interval": false, + "survey::constant": false, + "survey::deep score": false, + "survey::dynamic match": false, + "survey::exact match": false, + "survey::fractional interval": false, + "survey::fractional time": false, + "survey::fresh": false, + "survey::long postal": false, + "survey::missing tags": false, + "survey::missing unknown": false, + "survey::mixed temporal case": false, + "survey::negative infinity": false, + "survey::negative quotient": false, + "survey::null logic": false, + "survey::nullable date shift": false, + "survey::offset time": false, + "survey::postal length": false, + "survey::precise threshold": false, + "survey::prefix match": false, + "survey::range predicate": false, + "survey::recent": false, + "survey::right associative arithmetic": false, + "survey::selected dates": false, + "survey::suffix mismatch": false, + "survey::untyped count": false + } + } +} diff --git a/pkg-r/tests/testthat/helper-data-dict.R b/pkg-r/tests/testthat/helper-data-dict.R index cd7e67de..c112c91f 100644 --- a/pkg-r/tests/testthat/helper-data-dict.R +++ b/pkg-r/tests/testthat/helper-data-dict.R @@ -31,30 +31,26 @@ data_dict_cli_context <- function(result) { ) } +# The conformance corpus is a cross-language fixture: both suites read the +# same YAML files. These read the synced copy, as every shared fixture does. +definition_fixture_path <- function(name, kind = "valid") { + test_path("fixtures", "shared", "definition-export", kind, name) +} + definition_fixture_paths <- function(kind) { sort(Sys.glob(test_path( "fixtures", + "shared", "definition-export", kind, "*.yaml" ))) } +# Which data-dict problem code each invalid fixture must produce. Shared, so +# both suites hold one copy of the mapping. definition_fixture_error_code <- function(path) { - codes <- c( - "between-temporal.yaml" = "S21", - "columns-non-filter.yaml" = "S21", - "columns-transitive.yaml" = "S21", - "cycle.yaml" = "S34", - "duplicate.yaml" = "S10", - "nested-aggregate.yaml" = "S30", - "parse.yaml" = "S19", - "regex-engine.yaml" = "S21", - "shadow.yaml" = "S33", - "type.yaml" = "S21", - "unknown.yaml" = "S20" - ) - unname(codes[[basename(path)]]) + shared_fixture("definitions")$invalid[[basename(path)]] } definition_export_contract <- function(export) { diff --git a/pkg-r/tests/testthat/test-definition-compile.R b/pkg-r/tests/testthat/test-definition-compile.R index 09f74760..4c7f462e 100644 --- a/pkg-r/tests/testthat/test-definition-compile.R +++ b/pkg-r/tests/testthat/test-definition-compile.R @@ -1,5 +1,5 @@ definition_compile_fixture <- function(name, source) { - path <- test_path("fixtures", "definition-export", "valid", name) + path <- definition_fixture_path(name) definition_compile_source(yaml::read_yaml(path), source) } diff --git a/pkg-r/tests/testthat/test-definition-export.R b/pkg-r/tests/testthat/test-definition-export.R index 5506d69b..cb23b7ad 100644 --- a/pkg-r/tests/testthat/test-definition-export.R +++ b/pkg-r/tests/testthat/test-definition-export.R @@ -1,6 +1,6 @@ test_that("landed definition envelopes export inferred records", { skip_if_not_installed("yaml") - path <- test_path("fixtures", "definition-export", "valid", "core.yaml") + path <- definition_fixture_path("core.yaml") export <- definition_export_spec(yaml::read_yaml(path)) definitions <- export$tables$orders$definitions @@ -36,7 +36,7 @@ test_that("the expression parser preserves data-dict precedence", { test_that("quoted definition names and struct fields resolve separately", { skip_if_not_installed("yaml") - path <- test_path("fixtures", "definition-export", "valid", "language.yaml") + path <- definition_fixture_path("language.yaml") definitions <- definition_export_spec(yaml::read_yaml( path ))$tables$survey$definitions @@ -53,7 +53,7 @@ test_that("quoted definition names and struct fields resolve separately", { test_that("COLUMNS selections expand in DuckDB translations", { skip_if_not_installed("yaml") - path <- test_path("fixtures", "definition-export", "valid", "language.yaml") + path <- definition_fixture_path("language.yaml") definitions <- definition_export_spec(yaml::read_yaml( path ))$tables$survey$definitions @@ -72,7 +72,7 @@ test_that("COLUMNS selections expand in DuckDB translations", { test_that("DuckDB mappings carry data-dict fidelity notes", { skip_if_not_installed("yaml") - path <- test_path("fixtures", "definition-export", "valid", "functions.yaml") + path <- definition_fixture_path("functions.yaml") definitions <- definition_export_spec(yaml::read_yaml( path ))$tables$values$definitions @@ -98,7 +98,7 @@ test_that("DuckDB mappings carry data-dict fidelity notes", { test_that("DuckDB literals use data-dict's canonical forms", { skip_if_not_installed("yaml") - path <- test_path("fixtures", "definition-export", "valid", "language.yaml") + path <- definition_fixture_path("language.yaml") definitions <- definition_export_spec(yaml::read_yaml( path ))$tables$survey$definitions diff --git a/scripts/generate-definitions-fixture.sh b/scripts/generate-definitions-fixture.sh new file mode 100755 index 00000000..7bede432 --- /dev/null +++ b/scripts/generate-definitions-fixture.sh @@ -0,0 +1,87 @@ +#!/usr/bin/env bash +# Regenerate the export-record half of tests/shared/definitions.json from the +# pinned data-dict binary. +# +# `export_records` is data-dict's own output, projected to the fields both +# packages consume. `mixed_grain` and `invalid` are not in that output: +# grain is derived from the typed IR, and the problem codes come from +# validate-spec. Both are hand-maintained and this script preserves them. +# +# The binary is the authority. Regenerating against a build from any other +# revision would quietly bless whatever that build does. +set -euo pipefail + +commit="d950c5ac90d0ab939d330600f3a5ee1bfde0f604" +root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +corpus="$root/tests/shared/definition-export/valid" +out="$root/tests/shared/definitions.json" + +if ! command -v data-dict >/dev/null; then + echo "data-dict is not on PATH. Install it at the pinned commit:" >&2 + echo " cargo install --git https://github.com/tidyverse/data-dict --rev $commit data-dict-cli" >&2 + exit 1 +fi + +installed="$(cargo install --list 2>/dev/null | grep -c "data-dict?rev=$commit" || true)" +if [ "$installed" -eq 0 ]; then + echo "The data-dict on PATH was not built from $commit." >&2 + echo "A fixture generated from another revision is not authoritative." >&2 + exit 1 +fi + +python3 - "$commit" "$corpus" "$out" <<'PY' +import json +import pathlib +import subprocess +import sys + +commit, corpus, out = sys.argv[1], pathlib.Path(sys.argv[2]), pathlib.Path(sys.argv[3]) +existing = json.loads(out.read_text()) if out.exists() else {} + +records = {} +for path in sorted(corpus.glob("*.yaml")): + export = json.loads( + subprocess.run( + ["data-dict", "export-spec", str(path)], + check=True, + capture_output=True, + text=True, + ).stdout + ) + cases = {} + for table in export.get("tables", []): + for definition in table.get("definitions") or []: + duckdb = next( + ( + item + for item in definition.get("translations") or [] + if item.get("target") == "SQL(duckdb)" + ), + {}, + ) + cases[f"{table['name']}::{definition['name']}"] = { + "expression": definition.get("expression"), + "kind": definition.get("kind"), + "type": definition.get("type"), + "columns": definition.get("columns") or [], + "definitions": definition.get("definitions") or [], + "translation": { + "target": duckdb.get("target"), + "code": duckdb.get("code"), + "error": duckdb.get("error"), + "notes": duckdb.get("notes") or [], + }, + } + records[path.name] = cases + +spec = { + "data_dict_commit": commit, + "corpus_dir": "definition-export", + "export_records": records, + "mixed_grain": existing.get("mixed_grain", {}), + "invalid": existing.get("invalid", {}), +} +out.write_text(json.dumps(spec, indent=2, sort_keys=True) + "\n") +total = sum(len(cases) for cases in records.values()) +print(f"Wrote {out.name} with {total} definitions from {len(records)} fixtures") +PY diff --git a/tests/shared/README.md b/tests/shared/README.md index c4fc16bd..f54deb26 100644 --- a/tests/shared/README.md +++ b/tests/shared/README.md @@ -25,7 +25,7 @@ The Python suite reads this directory directly. The R suite cannot. `testthat` n - **Span names and attributes.** `commons_conversation_turn`, `commons_agent_create`, `commons_data_source_create`, and friends. Also `gen_ai.conversation.id`, `commons.provenance.tag`, and the exact JSON shape of `commons.citation.candidates`. This contract lets the R trajectory reviewer read Python traces. Write it so that it survives the conversation-id ownership moving upstream to shinychat. - **The provenance and citation behavior.** The `derive_provenance_tag()` truth table, `normalize_citation()` input/output pairs, `match_citation()` verdicts including both guards (10-character minimum, only-the-quote-verifies), `parse_commons_citation()` well-formed and malformed bodies. Also the chunk-invariance cases of the streaming scanner. The scanner is a pure chunks-in/string-out function, so it is ideal fixture material. - **The citation dialect and display copy.** The `` grammar and the `PROVENANCE_DISPLAY` strings, so that both UIs say the same words. -- **The definitions interface.** The data-dict CLI JSON contract that both packages consume, plus the grain metadata that `call_metrics` needs for its mixed-grain guard. +- **The definitions interface.** `definitions.json` pins the export-record contract both packages consume, the grain metadata `call_metrics` needs for its mixed-grain guard, and the data-dict problem code each invalid fixture must produce. `definition-export/` is the conformance corpus itself, read by both suites rather than copied into either. `export_records` is generated from the data-dict binary at the pinned commit by `scripts/generate-definitions-fixture.sh`, which refuses to run against a binary built from anything else; `mixed_grain` and `invalid` are hand-maintained and the generator preserves them. This fixture does not replace the conformance harness: the harness compares against a real binary, while this pins what both packages agree to consume. - **Trace file naming.** `trace(-[0-9]+)?\.jsonl`, one OTLP envelope per line. ## Conventions diff --git a/tests/shared/definition-export/invalid/between-temporal.yaml b/tests/shared/definition-export/invalid/between-temporal.yaml new file mode 100644 index 00000000..6df53ce9 --- /dev/null +++ b/tests/shared/definition-export/invalid/between-temporal.yaml @@ -0,0 +1,14 @@ +$version: "0.1.0" +$learn_more: https://data-dict.tidyverse.org/ +tables: + - name: events + columns: + - name: created + type: date + range: [2020-01-01, 2030-01-01] + - name: observed + type: datetime + range: [2020-01-01T00:00:00Z, 2030-01-01T00:00:00Z] + definitions: + - name: broken + expr: "'2024-01-01' BETWEEN created AND observed" diff --git a/tests/shared/definition-export/invalid/columns-non-filter.yaml b/tests/shared/definition-export/invalid/columns-non-filter.yaml new file mode 100644 index 00000000..c5a9c079 --- /dev/null +++ b/tests/shared/definition-export/invalid/columns-non-filter.yaml @@ -0,0 +1,11 @@ +$version: "0.1.0" +$learn_more: https://data-dict.tidyverse.org/ +tables: + - name: orders + columns: + - name: amount + type: number(quantity) + range: [0, 1000] + definitions: + - name: broken + expr: COLUMNS(*) + 1 diff --git a/tests/shared/definition-export/invalid/columns-transitive.yaml b/tests/shared/definition-export/invalid/columns-transitive.yaml new file mode 100644 index 00000000..3e0ff808 --- /dev/null +++ b/tests/shared/definition-export/invalid/columns-transitive.yaml @@ -0,0 +1,16 @@ +$version: "0.1.0" +$learn_more: https://data-dict.tidyverse.org/ +tables: + - name: orders + columns: + - name: amount + type: number(quantity) + range: [0, 1000] + - name: quantity + type: number(quantity) + range: [0, 100] + definitions: + - name: positive amount + expr: COLUMNS([amount]) > 0 + - name: broken + expr: '`positive amount` AND COLUMNS([quantity]) > 0' diff --git a/tests/shared/definition-export/invalid/cycle.yaml b/tests/shared/definition-export/invalid/cycle.yaml new file mode 100644 index 00000000..387ef904 --- /dev/null +++ b/tests/shared/definition-export/invalid/cycle.yaml @@ -0,0 +1,13 @@ +$version: "0.1.0" +$learn_more: https://data-dict.tidyverse.org/ +tables: + - name: orders + columns: + - name: amount + type: number(quantity) + range: [0, 1000] + definitions: + - name: chicken + expr: egg AND amount > 0 + - name: egg + expr: chicken AND amount < 10 diff --git a/tests/shared/definition-export/invalid/duplicate.yaml b/tests/shared/definition-export/invalid/duplicate.yaml new file mode 100644 index 00000000..ffc8b042 --- /dev/null +++ b/tests/shared/definition-export/invalid/duplicate.yaml @@ -0,0 +1,13 @@ +$version: "0.1.0" +$learn_more: https://data-dict.tidyverse.org/ +tables: + - name: orders + columns: + - name: amount + type: number(quantity) + range: [0, 1000] + definitions: + - name: total + expr: SUM(amount) + - name: total + expr: AVG(amount) diff --git a/tests/shared/definition-export/invalid/nested-aggregate.yaml b/tests/shared/definition-export/invalid/nested-aggregate.yaml new file mode 100644 index 00000000..b1ae5d74 --- /dev/null +++ b/tests/shared/definition-export/invalid/nested-aggregate.yaml @@ -0,0 +1,11 @@ +$version: "0.1.0" +$learn_more: https://data-dict.tidyverse.org/ +tables: + - name: orders + columns: + - name: amount + type: number(quantity) + range: [0, 1000] + definitions: + - name: broken + expr: AVG(MIN(amount)) diff --git a/tests/shared/definition-export/invalid/parse.yaml b/tests/shared/definition-export/invalid/parse.yaml new file mode 100644 index 00000000..b878826f --- /dev/null +++ b/tests/shared/definition-export/invalid/parse.yaml @@ -0,0 +1,11 @@ +$version: "0.1.0" +$learn_more: https://data-dict.tidyverse.org/ +tables: + - name: orders + columns: + - name: amount + type: number(quantity) + range: [0, 1000] + definitions: + - name: broken + expr: amount + diff --git a/tests/shared/definition-export/invalid/regex-engine.yaml b/tests/shared/definition-export/invalid/regex-engine.yaml new file mode 100644 index 00000000..0a32ccd7 --- /dev/null +++ b/tests/shared/definition-export/invalid/regex-engine.yaml @@ -0,0 +1,11 @@ +$version: "0.1.0" +$learn_more: https://data-dict.tidyverse.org/ +tables: + - name: messages + columns: + - name: text + type: string + examples: [a, b] + definitions: + - name: broken + expr: "text SIMILAR TO '(?>a)'" diff --git a/tests/shared/definition-export/invalid/shadow.yaml b/tests/shared/definition-export/invalid/shadow.yaml new file mode 100644 index 00000000..dd46edb7 --- /dev/null +++ b/tests/shared/definition-export/invalid/shadow.yaml @@ -0,0 +1,11 @@ +$version: "0.1.0" +$learn_more: https://data-dict.tidyverse.org/ +tables: + - name: orders + columns: + - name: amount + type: number(quantity) + range: [0, 1000] + definitions: + - name: amount + expr: SUM(amount) diff --git a/tests/shared/definition-export/invalid/type.yaml b/tests/shared/definition-export/invalid/type.yaml new file mode 100644 index 00000000..5bd508ef --- /dev/null +++ b/tests/shared/definition-export/invalid/type.yaml @@ -0,0 +1,11 @@ +$version: "0.1.0" +$learn_more: https://data-dict.tidyverse.org/ +tables: + - name: orders + columns: + - name: amount + type: number(quantity) + range: [0, 1000] + definitions: + - name: broken + expr: LOWER(amount) diff --git a/tests/shared/definition-export/invalid/unknown.yaml b/tests/shared/definition-export/invalid/unknown.yaml new file mode 100644 index 00000000..0920950f --- /dev/null +++ b/tests/shared/definition-export/invalid/unknown.yaml @@ -0,0 +1,11 @@ +$version: "0.1.0" +$learn_more: https://data-dict.tidyverse.org/ +tables: + - name: orders + columns: + - name: amount + type: number(quantity) + range: [0, 1000] + definitions: + - name: broken + expr: missing + amount diff --git a/tests/shared/definition-export/valid/core.yaml b/tests/shared/definition-export/valid/core.yaml new file mode 100644 index 00000000..e301d3d6 --- /dev/null +++ b/tests/shared/definition-export/valid/core.yaml @@ -0,0 +1,25 @@ +$version: "0.1.0" +$learn_more: https://data-dict.tidyverse.org/ +tables: + - name: orders + columns: + - name: status_cd + type: number(id) + examples: [10, 90] + - name: order_total + type: number(quantity) + range: [0, 1000] + - name: tile_size + type: string + examples: [Consumer, Mid-Market-3, Enterprise-1] + definitions: + - name: net_revenue + description: Realized revenue excluding returned orders. + expr: SUM(CASE WHEN status_cd = 90 THEN 0 ELSE order_total END) + - name: is_enterprise + label: Enterprise segment + expr: tile_size IN ('Mid-Market-3', 'Enterprise-1') + - name: enterprise_revenue + expr: SUM(CASE WHEN is_enterprise THEN order_total ELSE 0 END) + - name: list_price + expr: order_total * 1.2 diff --git a/tests/shared/definition-export/valid/functions.yaml b/tests/shared/definition-export/valid/functions.yaml new file mode 100644 index 00000000..f75a94fe --- /dev/null +++ b/tests/shared/definition-export/valid/functions.yaml @@ -0,0 +1,28 @@ +$version: "0.1.0" +$learn_more: https://data-dict.tidyverse.org/ +tables: + - name: values + columns: + - name: number + type: number(quantity) + range: [-100, 100] + - name: text + type: string + examples: [Alpha, omega] + - name: flag + type: boolean + definitions: + - name: strings + expr: STARTS_WITH(LOWER(TRIM(text)), 'a') OR ENDS_WITH(UPPER(text), 'Z') + - name: patterns + expr: text LIKE 'A_%' AND text SIMILAR TO 'A.*' + - name: numeric + expr: ABS(number) + FLOOR(number) + CEIL(number) + ROUND(number, 2) + - name: remainder + expr: MOD(number, 3) + - name: finite + expr: IS_FINITE(number) AND NOT IS_INFINITE(number) AND NOT IS_NAN(number) + - name: folds + expr: MIN(number) + MAX(number) + SUM(number) + AVG(number) + COUNT(number) + COUNT_DISTINCT(number) + ROW_COUNT() + - name: boolean folds + expr: ANY(flag) OR ALL(flag) diff --git a/tests/shared/definition-export/valid/language.yaml b/tests/shared/definition-export/valid/language.yaml new file mode 100644 index 00000000..e1f44753 --- /dev/null +++ b/tests/shared/definition-export/valid/language.yaml @@ -0,0 +1,109 @@ +$version: "0.1.0" +$learn_more: https://data-dict.tidyverse.org/ +tables: + - name: survey + columns: + - name: q1 + type: boolean + - name: q2 + type: boolean + - name: created + type: date + range: [2020-01-01, 2030-01-01] + - name: observed + type: datetime + range: [2020-01-01T00:00:00Z, 2030-01-01T00:00:00Z] + - name: amount + type: number(quantity) + range: [0, 1000] + - name: ratio + type: number(quantity) + range: [-10, 10] + - name: pattern + type: string + examples: ["%a", "a%"] + - name: category + type: enum + values: [A, B] + - name: profile + type: struct + fields: + - name: zip + type: string + examples: ["02139", "60601"] + - name: geo + type: struct + fields: + - name: latitude + type: number(quantity) + range: [-90, 90] + - name: nick names + type: string + examples: [Al, Bee] + - name: tags + type: list(string) + examples: [A, B, C] + - name: untyped + definitions: + - name: complete + expr: COLUMNS([q1, q2]) IS NOT NULL + - name: recent + expr: created >= '2024-01-01' + - name: fresh + expr: observed >= NOW() - interval(2, weeks) + - name: postal length + expr: LENGTH(profile.zip) + - name: long postal + expr: '`postal length` > 5' + - name: amount band + expr: amount <= 2 * MIN(amount) + - name: constant + expr: "1" + - name: anything missing + expr: COLUMNS(*) IS NOT NULL + - name: untyped count + expr: COUNT(untyped) + - name: negative quotient + expr: -(amount - 2) / (ratio + 1.0) + - name: range predicate + expr: amount BETWEEN -10 AND 10 AND amount NOT IN (0, 1) + - name: null logic + expr: NOT q1 OR NULL IS NULL + - name: exact match + expr: pattern LIKE 'a' + - name: prefix match + expr: pattern LIKE 'a%' + - name: suffix mismatch + expr: pattern NOT LIKE '%z' + - name: dynamic match + expr: pattern LIKE profile.zip + - name: case without else + expr: CASE WHEN q1 THEN amount END + - name: mixed temporal case + expr: CASE WHEN q1 THEN created ELSE observed END + - name: offset time + expr: observed >= '2024-01-01T01:30:00-06:00' + - name: fractional time + expr: observed <= '2024-01-01T01:30:00.123Z' + - name: computed interval + expr: observed + interval(amount + 1, days) + - name: fractional interval + expr: observed - interval(1.5, hours) + - name: category A + expr: category = 'A' + - name: deep score + expr: profile.geo.latitude + LENGTH(profile.`nick names`) + - name: missing tags + expr: tags IS NULL + - name: missing unknown + expr: untyped IS NULL + - name: selected dates + expr: COLUMNS('^created$') >= '2020-01-01' + - name: negative infinity + expr: amount > -INF + - name: precise threshold + expr: amount > 0.12345678901234567 + - name: right associative arithmetic + expr: amount - (ratio - 1) + - name: nullable date shift + expr: created + NULL diff --git a/tests/shared/definitions.json b/tests/shared/definitions.json new file mode 100644 index 00000000..a7e19ed3 --- /dev/null +++ b/tests/shared/definitions.json @@ -0,0 +1,743 @@ +{ + "corpus_dir": "definition-export", + "data_dict_commit": "d950c5ac90d0ab939d330600f3a5ee1bfde0f604", + "export_records": { + "core.yaml": { + "orders::enterprise_revenue": { + "columns": [ + "order_total" + ], + "definitions": [ + "is_enterprise" + ], + "expression": "SUM(CASE WHEN is_enterprise THEN order_total ELSE 0 END)", + "kind": "metric", + "translation": { + "code": "sum(CASE WHEN \"is_enterprise\" THEN \"order_total\" ELSE 0 END)", + "error": null, + "notes": [ + "DuckDB sums integers at 128 bits, so a total data-dict reports as an overflow (D09) may succeed." + ], + "target": "SQL(duckdb)" + }, + "type": "number" + }, + "orders::is_enterprise": { + "columns": [ + "tile_size" + ], + "definitions": [], + "expression": "tile_size IN ('Mid-Market-3', 'Enterprise-1')", + "kind": "filter", + "translation": { + "code": "\"tile_size\" IN ('Mid-Market-3', 'Enterprise-1')", + "error": null, + "notes": [], + "target": "SQL(duckdb)" + }, + "type": "boolean" + }, + "orders::list_price": { + "columns": [ + "order_total" + ], + "definitions": [], + "expression": "order_total * 1.2", + "kind": "derived", + "translation": { + "code": "\"order_total\" * 1.2", + "error": null, + "notes": [], + "target": "SQL(duckdb)" + }, + "type": "number" + }, + "orders::net_revenue": { + "columns": [ + "status_cd", + "order_total" + ], + "definitions": [], + "expression": "SUM(CASE WHEN status_cd = 90 THEN 0 ELSE order_total END)", + "kind": "metric", + "translation": { + "code": "sum(CASE WHEN \"status_cd\" = 90 THEN 0 ELSE \"order_total\" END)", + "error": null, + "notes": [ + "DuckDB compares a NaN as equal to itself and greater than every number, where data-dict answers false; a row holding one passes here and is reported there.", + "DuckDB sums integers at 128 bits, so a total data-dict reports as an overflow (D09) may succeed." + ], + "target": "SQL(duckdb)" + }, + "type": "number" + } + }, + "functions.yaml": { + "values::boolean folds": { + "columns": [ + "flag" + ], + "definitions": [], + "expression": "ANY(flag) OR ALL(flag)", + "kind": "metric", + "translation": { + "code": "bool_or(\"flag\") OR bool_and(\"flag\")", + "error": null, + "notes": [], + "target": "SQL(duckdb)" + }, + "type": "boolean" + }, + "values::finite": { + "columns": [ + "number" + ], + "definitions": [], + "expression": "IS_FINITE(number) AND NOT IS_INFINITE(number) AND NOT IS_NAN(number)", + "kind": "filter", + "translation": { + "code": "isfinite(\"number\") AND NOT isinf(\"number\") AND NOT isnan(\"number\")", + "error": null, + "notes": [], + "target": "SQL(duckdb)" + }, + "type": "boolean" + }, + "values::folds": { + "columns": [ + "number" + ], + "definitions": [], + "expression": "MIN(number) + MAX(number) + SUM(number) + AVG(number) + COUNT(number) + COUNT_DISTINCT(number) + ROW_COUNT()", + "kind": "metric", + "translation": { + "code": "min(\"number\") + max(\"number\") + sum(\"number\") + avg(\"number\") + count(\"number\") + count(DISTINCT \"number\") + count(*)", + "error": null, + "notes": [ + "DuckDB sums integers at 128 bits, so a total data-dict reports as an overflow (D09) may succeed." + ], + "target": "SQL(duckdb)" + }, + "type": "number" + }, + "values::numeric": { + "columns": [ + "number" + ], + "definitions": [], + "expression": "ABS(number) + FLOOR(number) + CEIL(number) + ROUND(number, 2)", + "kind": "derived", + "translation": { + "code": "abs(\"number\") + floor(\"number\") + ceil(\"number\") + round(\"number\", 2)", + "error": null, + "notes": [], + "target": "SQL(duckdb)" + }, + "type": "number" + }, + "values::patterns": { + "columns": [ + "text" + ], + "definitions": [], + "expression": "text LIKE 'A_%' AND text SIMILAR TO 'A.*'", + "kind": "filter", + "translation": { + "code": "regexp_full_match(\"text\", '^A..*$') AND regexp_full_match(\"text\", 'A.*')", + "error": null, + "notes": [], + "target": "SQL(duckdb)" + }, + "type": "boolean" + }, + "values::remainder": { + "columns": [ + "number" + ], + "definitions": [], + "expression": "MOD(number, 3)", + "kind": "derived", + "translation": { + "code": "mod(mod(\"number\", 3) + 3, 3)", + "error": null, + "notes": [ + "DuckDB yields null for an integer modulus by zero, where data-dict yields a NaN." + ], + "target": "SQL(duckdb)" + }, + "type": "number" + }, + "values::strings": { + "columns": [ + "text" + ], + "definitions": [], + "expression": "STARTS_WITH(LOWER(TRIM(text)), 'a') OR ENDS_WITH(UPPER(text), 'Z')", + "kind": "filter", + "translation": { + "code": "starts_with(lower(trim(\"text\")), 'a') OR ends_with(upper(\"text\"), 'Z')", + "error": null, + "notes": [], + "target": "SQL(duckdb)" + }, + "type": "boolean" + } + }, + "language.yaml": { + "survey::amount band": { + "columns": [ + "amount" + ], + "definitions": [], + "expression": "amount <= 2 * MIN(amount)", + "kind": "filter", + "translation": { + "code": "\"amount\" <= 2 * min(\"amount\")", + "error": null, + "notes": [ + "DuckDB compares a NaN as equal to itself and greater than every number, where data-dict answers false; a row holding one passes here and is reported there." + ], + "target": "SQL(duckdb)" + }, + "type": "boolean" + }, + "survey::anything missing": { + "columns": [ + "q1", + "q2", + "created", + "observed", + "amount", + "ratio", + "pattern", + "category", + "profile", + "tags" + ], + "definitions": [], + "expression": "COLUMNS(*) IS NOT NULL", + "kind": "filter", + "translation": { + "code": "\"q1\" IS NOT NULL AND \"q2\" IS NOT NULL AND \"created\" IS NOT NULL AND \"observed\" IS NOT NULL AND \"amount\" IS NOT NULL AND \"ratio\" IS NOT NULL AND \"pattern\" IS NOT NULL AND \"category\" IS NOT NULL AND \"profile\" IS NOT NULL AND \"tags\" IS NOT NULL AND \"untyped\" IS NOT NULL", + "error": null, + "notes": [], + "target": "SQL(duckdb)" + }, + "type": "boolean" + }, + "survey::case without else": { + "columns": [ + "q1", + "amount" + ], + "definitions": [], + "expression": "CASE WHEN q1 THEN amount END", + "kind": "derived", + "translation": { + "code": "CASE WHEN \"q1\" THEN \"amount\" END", + "error": null, + "notes": [], + "target": "SQL(duckdb)" + }, + "type": "number" + }, + "survey::category A": { + "columns": [ + "category" + ], + "definitions": [], + "expression": "category = 'A'", + "kind": "filter", + "translation": { + "code": "\"category\" = 'A'", + "error": null, + "notes": [], + "target": "SQL(duckdb)" + }, + "type": "boolean" + }, + "survey::complete": { + "columns": [ + "q1", + "q2" + ], + "definitions": [], + "expression": "COLUMNS([q1, q2]) IS NOT NULL", + "kind": "filter", + "translation": { + "code": "\"q1\" IS NOT NULL AND \"q2\" IS NOT NULL", + "error": null, + "notes": [], + "target": "SQL(duckdb)" + }, + "type": "boolean" + }, + "survey::computed interval": { + "columns": [ + "observed", + "amount" + ], + "definitions": [], + "expression": "observed + interval(amount + 1, days)", + "kind": "derived", + "translation": { + "code": "\"observed\" + (\"amount\" + 1 * INTERVAL '1 days')", + "error": null, + "notes": [], + "target": "SQL(duckdb)" + }, + "type": "datetime" + }, + "survey::constant": { + "columns": [], + "definitions": [], + "expression": "1", + "kind": "metric", + "translation": { + "code": "1", + "error": null, + "notes": [], + "target": "SQL(duckdb)" + }, + "type": "number" + }, + "survey::deep score": { + "columns": [ + "profile.geo.latitude", + "profile.nick names" + ], + "definitions": [], + "expression": "profile.geo.latitude + LENGTH(profile.`nick names`)", + "kind": "derived", + "translation": { + "code": "\"profile\".\"geo\".\"latitude\" + length(\"profile\".\"nick names\")", + "error": null, + "notes": [], + "target": "SQL(duckdb)" + }, + "type": "number" + }, + "survey::dynamic match": { + "columns": [ + "pattern", + "profile.zip" + ], + "definitions": [], + "expression": "pattern LIKE profile.zip", + "kind": "filter", + "translation": { + "code": "\"pattern\" LIKE \"profile\".\"zip\"", + "error": null, + "notes": [], + "target": "SQL(duckdb)" + }, + "type": "boolean" + }, + "survey::exact match": { + "columns": [ + "pattern" + ], + "definitions": [], + "expression": "pattern LIKE 'a'", + "kind": "filter", + "translation": { + "code": "\"pattern\" = 'a'", + "error": null, + "notes": [], + "target": "SQL(duckdb)" + }, + "type": "boolean" + }, + "survey::fractional interval": { + "columns": [ + "observed" + ], + "definitions": [], + "expression": "observed - interval(1.5, hours)", + "kind": "derived", + "translation": { + "code": "\"observed\" - (1.5 * INTERVAL '1 hours')", + "error": null, + "notes": [], + "target": "SQL(duckdb)" + }, + "type": "datetime" + }, + "survey::fractional time": { + "columns": [ + "observed" + ], + "definitions": [], + "expression": "observed <= '2024-01-01T01:30:00.123Z'", + "kind": "filter", + "translation": { + "code": "\"observed\" <= TIMESTAMP '2024-01-01 01:30:00.123'", + "error": null, + "notes": [], + "target": "SQL(duckdb)" + }, + "type": "boolean" + }, + "survey::fresh": { + "columns": [ + "observed" + ], + "definitions": [], + "expression": "observed >= NOW() - interval(2, weeks)", + "kind": "filter", + "translation": { + "code": "\"observed\" >= current_timestamp - INTERVAL '2 weeks'", + "error": null, + "notes": [], + "target": "SQL(duckdb)" + }, + "type": "boolean" + }, + "survey::long postal": { + "columns": [], + "definitions": [ + "postal length" + ], + "expression": "`postal length` > 5", + "kind": "filter", + "translation": { + "code": "\"postal length\" > 5", + "error": null, + "notes": [ + "DuckDB compares a NaN as equal to itself and greater than every number, where data-dict answers false; a row holding one passes here and is reported there." + ], + "target": "SQL(duckdb)" + }, + "type": "boolean" + }, + "survey::missing tags": { + "columns": [ + "tags" + ], + "definitions": [], + "expression": "tags IS NULL", + "kind": "filter", + "translation": { + "code": "\"tags\" IS NULL", + "error": null, + "notes": [], + "target": "SQL(duckdb)" + }, + "type": "boolean" + }, + "survey::missing unknown": { + "columns": [ + "untyped" + ], + "definitions": [], + "expression": "untyped IS NULL", + "kind": "filter", + "translation": { + "code": "\"untyped\" IS NULL", + "error": null, + "notes": [], + "target": "SQL(duckdb)" + }, + "type": "boolean" + }, + "survey::mixed temporal case": { + "columns": [ + "q1", + "created", + "observed" + ], + "definitions": [], + "expression": "CASE WHEN q1 THEN created ELSE observed END", + "kind": "derived", + "translation": { + "code": "CASE WHEN \"q1\" THEN \"created\" ELSE \"observed\" END", + "error": null, + "notes": [], + "target": "SQL(duckdb)" + }, + "type": null + }, + "survey::negative infinity": { + "columns": [ + "amount" + ], + "definitions": [], + "expression": "amount > -INF", + "kind": "filter", + "translation": { + "code": "\"amount\" > -CAST('Infinity' AS DOUBLE)", + "error": null, + "notes": [ + "DuckDB compares a NaN as equal to itself and greater than every number, where data-dict answers false; a row holding one passes here and is reported there." + ], + "target": "SQL(duckdb)" + }, + "type": "boolean" + }, + "survey::negative quotient": { + "columns": [ + "amount", + "ratio" + ], + "definitions": [], + "expression": "-(amount - 2) / (ratio + 1.0)", + "kind": "derived", + "translation": { + "code": "-(\"amount\" - 2) / (\"ratio\" + 1.0)", + "error": null, + "notes": [], + "target": "SQL(duckdb)" + }, + "type": "number" + }, + "survey::null logic": { + "columns": [ + "q1" + ], + "definitions": [], + "expression": "NOT q1 OR NULL IS NULL", + "kind": "filter", + "translation": { + "code": "NOT \"q1\" OR NULL IS NULL", + "error": null, + "notes": [], + "target": "SQL(duckdb)" + }, + "type": "boolean" + }, + "survey::nullable date shift": { + "columns": [ + "created" + ], + "definitions": [], + "expression": "created + NULL", + "kind": "derived", + "translation": { + "code": "\"created\" + NULL", + "error": null, + "notes": [], + "target": "SQL(duckdb)" + }, + "type": "datetime" + }, + "survey::offset time": { + "columns": [ + "observed" + ], + "definitions": [], + "expression": "observed >= '2024-01-01T01:30:00-06:00'", + "kind": "filter", + "translation": { + "code": "\"observed\" >= TIMESTAMP '2024-01-01 07:30:00'", + "error": null, + "notes": [], + "target": "SQL(duckdb)" + }, + "type": "boolean" + }, + "survey::postal length": { + "columns": [ + "profile.zip" + ], + "definitions": [], + "expression": "LENGTH(profile.zip)", + "kind": "derived", + "translation": { + "code": "length(\"profile\".\"zip\")", + "error": null, + "notes": [], + "target": "SQL(duckdb)" + }, + "type": "number" + }, + "survey::precise threshold": { + "columns": [ + "amount" + ], + "definitions": [], + "expression": "amount > 0.12345678901234567", + "kind": "filter", + "translation": { + "code": "\"amount\" > 0.12345678901234566", + "error": null, + "notes": [ + "DuckDB compares a NaN as equal to itself and greater than every number, where data-dict answers false; a row holding one passes here and is reported there." + ], + "target": "SQL(duckdb)" + }, + "type": "boolean" + }, + "survey::prefix match": { + "columns": [ + "pattern" + ], + "definitions": [], + "expression": "pattern LIKE 'a%'", + "kind": "filter", + "translation": { + "code": "starts_with(\"pattern\", 'a')", + "error": null, + "notes": [], + "target": "SQL(duckdb)" + }, + "type": "boolean" + }, + "survey::range predicate": { + "columns": [ + "amount" + ], + "definitions": [], + "expression": "amount BETWEEN -10 AND 10 AND amount NOT IN (0, 1)", + "kind": "filter", + "translation": { + "code": "\"amount\" BETWEEN -10 AND 10 AND \"amount\" NOT IN (0, 1)", + "error": null, + "notes": [ + "DuckDB compares a NaN as equal to itself and greater than every number, where data-dict answers false; a row holding one passes here and is reported there." + ], + "target": "SQL(duckdb)" + }, + "type": "boolean" + }, + "survey::recent": { + "columns": [ + "created" + ], + "definitions": [], + "expression": "created >= '2024-01-01'", + "kind": "filter", + "translation": { + "code": "\"created\" >= DATE '2024-01-01'", + "error": null, + "notes": [], + "target": "SQL(duckdb)" + }, + "type": "boolean" + }, + "survey::right associative arithmetic": { + "columns": [ + "amount", + "ratio" + ], + "definitions": [], + "expression": "amount - (ratio - 1)", + "kind": "derived", + "translation": { + "code": "\"amount\" - (\"ratio\" - 1)", + "error": null, + "notes": [], + "target": "SQL(duckdb)" + }, + "type": "number" + }, + "survey::selected dates": { + "columns": [ + "created" + ], + "definitions": [], + "expression": "COLUMNS('^created$') >= '2020-01-01'", + "kind": "filter", + "translation": { + "code": "\"created\" >= '2020-01-01'", + "error": null, + "notes": [], + "target": "SQL(duckdb)" + }, + "type": "boolean" + }, + "survey::suffix mismatch": { + "columns": [ + "pattern" + ], + "definitions": [], + "expression": "pattern NOT LIKE '%z'", + "kind": "filter", + "translation": { + "code": "NOT ends_with(\"pattern\", 'z')", + "error": null, + "notes": [], + "target": "SQL(duckdb)" + }, + "type": "boolean" + }, + "survey::untyped count": { + "columns": [ + "untyped" + ], + "definitions": [], + "expression": "COUNT(untyped)", + "kind": "metric", + "translation": { + "code": "count(\"untyped\")", + "error": null, + "notes": [], + "target": "SQL(duckdb)" + }, + "type": "number" + } + } + }, + "invalid": { + "between-temporal.yaml": "S21", + "columns-non-filter.yaml": "S21", + "columns-transitive.yaml": "S21", + "cycle.yaml": "S34", + "duplicate.yaml": "S10", + "nested-aggregate.yaml": "S30", + "parse.yaml": "S19", + "regex-engine.yaml": "S21", + "shadow.yaml": "S33", + "type.yaml": "S21", + "unknown.yaml": "S20" + }, + "mixed_grain": { + "core.yaml": { + "orders::enterprise_revenue": false, + "orders::is_enterprise": false, + "orders::list_price": false, + "orders::net_revenue": false + }, + "functions.yaml": { + "values::boolean folds": false, + "values::finite": false, + "values::folds": false, + "values::numeric": false, + "values::patterns": false, + "values::remainder": false, + "values::strings": false + }, + "language.yaml": { + "survey::amount band": true, + "survey::anything missing": false, + "survey::case without else": false, + "survey::category A": false, + "survey::complete": false, + "survey::computed interval": false, + "survey::constant": false, + "survey::deep score": false, + "survey::dynamic match": false, + "survey::exact match": false, + "survey::fractional interval": false, + "survey::fractional time": false, + "survey::fresh": false, + "survey::long postal": false, + "survey::missing tags": false, + "survey::missing unknown": false, + "survey::mixed temporal case": false, + "survey::negative infinity": false, + "survey::negative quotient": false, + "survey::null logic": false, + "survey::nullable date shift": false, + "survey::offset time": false, + "survey::postal length": false, + "survey::precise threshold": false, + "survey::prefix match": false, + "survey::range predicate": false, + "survey::recent": false, + "survey::right associative arithmetic": false, + "survey::selected dates": false, + "survey::suffix mismatch": false, + "survey::untyped count": false + } + } +} From b46e24893f5f1fc5bc4006b8f0e75fcdd64f841e Mon Sep 17 00:00:00 2001 From: Josh Taillon Date: Tue, 1 Sep 2026 21:34:18 -0600 Subject: [PATCH 2/4] test: assert the definitions contract against R, and resolve the pinned binary The fixture landed with no runner exercising export_records or mixed_grain, so most of it could drift without either package noticing. tests/shared/README.md asks for runners to land with a fixture for exactly this reason. R can assert it now, and does: its export contract and its grain metadata are compared against the fixture for every valid case. Python joins when its compiler exists; until then its runner checks the fixture's own integrity. Comparison sorts both sides. The generated file sorts its keys so diffs stay readable, while the export keeps authored order, and this fixture is a keyed contract rather than a sequence. The generator checked the cargo installation and then ran whichever data-dict PATH resolved, so a different binary could have generated the fixture while the check passed. It now invokes the cargo-installed path directly. Regenerating produced a byte-identical fixture. --- pkg-r/tests/testthat/helper-data-dict.R | 42 +++++++++++++++++++ pkg-r/tests/testthat/test-definition-export.R | 36 ++++++++++++++++ scripts/generate-definitions-fixture.sh | 19 +++++---- 3 files changed, 90 insertions(+), 7 deletions(-) diff --git a/pkg-r/tests/testthat/helper-data-dict.R b/pkg-r/tests/testthat/helper-data-dict.R index c112c91f..5291c11b 100644 --- a/pkg-r/tests/testthat/helper-data-dict.R +++ b/pkg-r/tests/testthat/helper-data-dict.R @@ -86,3 +86,45 @@ definition_export_contract <- function(export) { } out } + +# The fixture arrives from JSON as nested lists, while +# definition_export_contract() produces character vectors. Normalize the +# fixture side so a comparison reports a real difference rather than a +# difference in how each format spells an empty sequence. +definition_fixture_contract <- function(cases) { + lapply(cases, function(case) { + translation <- case$translation %||% list() + list( + expression = case$expression, + kind = case$kind, + type = case$type, + columns = as.character(unlist(case$columns, use.names = FALSE)), + definitions = as.character(unlist(case$definitions, use.names = FALSE)), + translation = list( + target = translation$target, + code = translation$code, + error = translation$error, + notes = as.character(unlist(translation$notes, use.names = FALSE)) + ) + ) + }) +} + +# Grain is derived from the typed IR rather than exported, so it is compared +# separately from the export contract. +definition_export_grain <- function(export) { + out <- list() + for (table in export$tables) { + definitions <- table$definitions + if (length(definitions) == 0) { + next + } + names(definitions) <- vapply(definitions, `[[`, character(1), "name") + mixed <- definition_mixed_grain(definitions) + for (i in seq_along(definitions)) { + key <- paste(table$name, names(definitions)[[i]], sep = "::") + out[[key]] <- mixed[[i]] + } + } + out +} diff --git a/pkg-r/tests/testthat/test-definition-export.R b/pkg-r/tests/testthat/test-definition-export.R index cb23b7ad..cb96cb7d 100644 --- a/pkg-r/tests/testthat/test-definition-export.R +++ b/pkg-r/tests/testthat/test-definition-export.R @@ -179,3 +179,39 @@ test_that("invalid fixtures also fail an installed data-dict", { expect_contains(codes, definition_fixture_error_code(path)) } }) + +test_that("the export matches the shared definitions contract", { + skip_if_not_installed("yaml") + spec <- shared_fixture("definitions") + paths <- definition_fixture_paths("valid") + expect_gt(length(paths), 0) + + for (path in paths) { + export <- definition_export_spec(yaml::read_yaml(path)) + # Keyed contract, not a sequence: the generated file sorts its keys for + # stable diffs while the export keeps authored order. + local <- definition_export_contract(export) + fixture <- definition_fixture_contract(spec$export_records[[basename(path)]]) + expect_equal( + local[order(names(local))], + fixture[order(names(fixture))], + info = basename(path) + ) + } +}) + +test_that("grain metadata matches the shared definitions contract", { + skip_if_not_installed("yaml") + spec <- shared_fixture("definitions") + + for (path in definition_fixture_paths("valid")) { + export <- definition_export_spec(yaml::read_yaml(path)) + local <- definition_export_grain(export) + fixture <- spec$mixed_grain[[basename(path)]] + expect_equal( + local[order(names(local))], + fixture[order(names(fixture))], + info = basename(path) + ) + } +}) diff --git a/scripts/generate-definitions-fixture.sh b/scripts/generate-definitions-fixture.sh index 7bede432..34c5d137 100755 --- a/scripts/generate-definitions-fixture.sh +++ b/scripts/generate-definitions-fixture.sh @@ -16,33 +16,38 @@ root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" corpus="$root/tests/shared/definition-export/valid" out="$root/tests/shared/definitions.json" -if ! command -v data-dict >/dev/null; then - echo "data-dict is not on PATH. Install it at the pinned commit:" >&2 +# Invoke the cargo-installed binary by path rather than whatever PATH +# resolves. Checking the cargo installation and then running `data-dict` +# would let a different binary earlier on PATH generate the fixture while +# the check still passed. +data_dict="${CARGO_HOME:-$HOME/.cargo}/bin/data-dict" +if [ ! -x "$data_dict" ]; then + echo "No cargo-installed data-dict at $data_dict. Install it at the pinned commit:" >&2 echo " cargo install --git https://github.com/tidyverse/data-dict --rev $commit data-dict-cli" >&2 exit 1 fi -installed="$(cargo install --list 2>/dev/null | grep -c "data-dict?rev=$commit" || true)" -if [ "$installed" -eq 0 ]; then - echo "The data-dict on PATH was not built from $commit." >&2 +if ! cargo install --list 2>/dev/null | grep -q "data-dict?rev=$commit"; then + echo "$data_dict was not built from $commit." >&2 echo "A fixture generated from another revision is not authoritative." >&2 exit 1 fi -python3 - "$commit" "$corpus" "$out" <<'PY' +python3 - "$commit" "$corpus" "$out" "$data_dict" <<'PY' import json import pathlib import subprocess import sys commit, corpus, out = sys.argv[1], pathlib.Path(sys.argv[2]), pathlib.Path(sys.argv[3]) +data_dict = sys.argv[4] existing = json.loads(out.read_text()) if out.exists() else {} records = {} for path in sorted(corpus.glob("*.yaml")): export = json.loads( subprocess.run( - ["data-dict", "export-spec", str(path)], + [data_dict, "export-spec", str(path)], check=True, capture_output=True, text=True, From c643fe1111108f47d53ae1ae2bcf6692114be445 Mon Sep 17 00:00:00 2001 From: Josh Taillon Date: Tue, 1 Sep 2026 21:36:02 -0600 Subject: [PATCH 3/4] fix: read the whole cargo listing when checking the pinned binary grep -q closes the pipe on its first match, so with pipefail set cargo can die of SIGPIPE and fail the pipeline, rejecting a correctly pinned install. grep -c reads the full stream. --- scripts/generate-definitions-fixture.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/scripts/generate-definitions-fixture.sh b/scripts/generate-definitions-fixture.sh index 34c5d137..a6a6f481 100755 --- a/scripts/generate-definitions-fixture.sh +++ b/scripts/generate-definitions-fixture.sh @@ -27,7 +27,11 @@ if [ ! -x "$data_dict" ]; then exit 1 fi -if ! cargo install --list 2>/dev/null | grep -q "data-dict?rev=$commit"; then +# grep -c rather than -q: with pipefail, -q closes the pipe on its first +# match and cargo can die of SIGPIPE, failing the pipeline and rejecting a +# correctly pinned install. +pinned="$(cargo install --list 2>/dev/null | grep -F -c "data-dict?rev=$commit" || true)" +if [ "$pinned" -eq 0 ]; then echo "$data_dict was not built from $commit." >&2 echo "A fixture generated from another revision is not authoritative." >&2 exit 1 From 7faab7ffb79437d9ce692ebaf26bb57b6d0d86de Mon Sep 17 00:00:00 2001 From: Josh Taillon Date: Tue, 1 Sep 2026 21:36:55 -0600 Subject: [PATCH 4/4] fix: do not let a failed cargo listing look like a pinned install `|| true` covered the whole pipeline, so cargo failing after emitting a matching line would have counted as verification. The listing is captured and its status checked first; the tolerated failure is now only grep's no-match. --- scripts/generate-definitions-fixture.sh | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/scripts/generate-definitions-fixture.sh b/scripts/generate-definitions-fixture.sh index a6a6f481..4a58c7cd 100755 --- a/scripts/generate-definitions-fixture.sh +++ b/scripts/generate-definitions-fixture.sh @@ -27,10 +27,16 @@ if [ ! -x "$data_dict" ]; then exit 1 fi -# grep -c rather than -q: with pipefail, -q closes the pipe on its first -# match and cargo can die of SIGPIPE, failing the pipeline and rejecting a -# correctly pinned install. -pinned="$(cargo install --list 2>/dev/null | grep -F -c "data-dict?rev=$commit" || true)" +# Cargo's listing is captured before it is searched, so a failure to produce +# it is not mistaken for an answer about it. grep runs over the captured +# text rather than a pipe, which also avoids -q closing the pipe on its +# first match and killing cargo with SIGPIPE under pipefail. +if ! listing="$(cargo install --list 2>/dev/null)"; then + echo "cargo install --list failed, so the binary's revision cannot be verified." >&2 + exit 1 +fi + +pinned="$(printf '%s\n' "$listing" | grep -F -c "data-dict?rev=$commit" || true)" if [ "$pinned" -eq 0 ]; then echo "$data_dict was not built from $commit." >&2 echo "A fixture generated from another revision is not authoritative." >&2