Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 5 additions & 5 deletions pkg-py/src/commons/_catalog/_core.py
Original file line number Diff line number Diff line change
Expand Up @@ -87,6 +87,10 @@ class TableRegistry:
class MergedDictionary:
dictionary: Any
relations: dict[str, Relation]
# What the merge matched, for the definition compiler: "tables" maps each
# authored table name to the relation label it matched, or None, and
# "columns" maps each authored column name to the spelling the warehouse
# reported. None when there was no catalog to match against.
definition_bindings: dict[str, Any] | None


Expand Down Expand Up @@ -402,11 +406,7 @@ def merge_dictionary(
return MergedDictionary(
dictionary=dictionary,
relations=relations,
definition_bindings={
"tables": matches,
"columns": column_matches,
"strict": True,
},
definition_bindings={"tables": matches, "columns": column_matches},
)


Expand Down
71 changes: 0 additions & 71 deletions pkg-py/src/commons/_catalog/_import.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,11 +17,9 @@
from . import _databricks, _snowflake
from ._core import (
Manifest,
MergedDictionary,
Relation,
Selector,
check_exclude,
has_suffix,
id_type,
merge_dictionary,
table_registry,
Expand Down Expand Up @@ -105,7 +103,6 @@ def access_check(table_id: TableId, label: str) -> None:
access_check=access_check,
)
check_session(backend, session)
_check_definitions_bound(merged, registry.dropped, exclude, identifier_case)

# The manifest starts with every relation unknown, including the ones
# just probed. Carrying the construction-time answer forward would save a
Expand All @@ -125,74 +122,6 @@ def access_check(table_id: TableId, label: str) -> None:
)


def _check_definitions_bound(
merged: MergedDictionary,
dropped: list[Relation] | None = None,
exclude: list[str] | None = None,
identifier_case: str | None = None,
) -> None:
"""Refuse definitions the merge renamed out from under.

The merge re-keys an authored dictionary to the warehouse's own labels
and column spellings, but a definition's expression still names what the
author wrote. Lowering it as written would emit SQL against identifiers
the warehouse does not have, so it is refused until the compiler can bind
the two together.
"""
bindings = merged.definition_bindings
exports = getattr(merged.dictionary, "definition_exports", None) or {}
if not bindings or not exports:
return
for authored_table, definitions in exports.items():
if not definitions:
continue
# An authored table that matched nothing is dropped by the merge, so
# its definitions would go with it and the agent would never be told.
if bindings["tables"].get(authored_table) is None:
if _was_excluded(authored_table, dropped, identifier_case):
raise ValueError(
f"Authored table {authored_table!r} declares definitions, "
f"and exclude dropped it from the catalog listing. Narrow "
f"{exclude!r}, or drop the table from the data dictionary."
)
raise ValueError(
f"Authored table {authored_table!r} declares definitions, and "
f"does not match an exposed relation. Name it as the data "
f"source selects it, or drop it from the data dictionary."
)
# Every authored column is checked, not only the ones a definition
# reads: which columns an expression touches is in the compiler's
# parse tree, and the refusal is temporary either way. A column the
# warehouse never reported is caught here too, since a definition
# over it would lower to SQL naming nothing at all.
columns = bindings["columns"].get(authored_table) or {}
unbound = [name for name, discovered in columns.items() if discovered != name]
if unbound:
raise NotImplementedError(
f"Table {authored_table!r} declares definitions, and the "
f"warehouse does not have column {unbound[0]!r} under that "
f"name. Binding a definition to the discovered spelling is "
f"not available yet."
)


def _was_excluded(
authored_table: str, dropped: list[Relation] | None, identifier_case: str | None
) -> bool:
"""Whether exclude is what removed the relation an authored name meant.

Compared against the relations exclude actually dropped rather than
against the patterns: a pattern is written in the warehouse's spelling
and an authored name need not be, so only the folded names line up. The
authored name may be qualified, so it is matched as a suffix, by the
same rule the merge uses to find the relation in the first place.
"""
suffix = authored_table.split(".")
return any(
has_suffix(item, suffix, identifier_case) for item in dropped or []
)


def _selectors(backend: Any, reader: Any, tables: Any) -> list[Selector]:
"""Read a `tables` selection, defaulting to the connection's namespace.

Expand Down
14 changes: 9 additions & 5 deletions pkg-py/src/commons/_data_source.py
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,10 @@ def _with_compiled_definitions(source: DataSource) -> DataSource:
from ._definitions import attach_compiled_definitions

attach_compiled_definitions(
source.dictionary, source.dialect(), set(source.tables)
source.dictionary,
source.dialect(),
set(source.tables),
source.definition_bindings,
)
return source

Expand Down Expand Up @@ -387,10 +390,11 @@ def data_source(
dialect during construction, so construction raises if the dialect has
no emitter (DuckDB, Snowflake, and Databricks have one), if a definition
sits on a table the source does not expose, or if a metric mixes row and
aggregate grain. On a warehouse it also raises if a table declaring
definitions matched no exposed relation, and, until the compiler can
bind an authored name to the discovered one, if the warehouse spells one
of that table's columns differently.
aggregate grain. On a warehouse the authored column spellings are bound
to the names the catalog reported before anything is lowered, so it
raises there only if a table declaring definitions matched no exposed
relation, or if a definition names an authored column the selected
relation does not have.
"""
from ._data_dictionary import as_data_dictionary

Expand Down
Loading
Loading