Skip to content

Commit ed7c6d3

Browse files
committed
fix(snowflake): tell missing schemas from missing databases with SHOW SCHEMAS
- SHOW TABLES/VIEWS report the same generic 2043 error for a missing schema and a missing database, so any missing-object error from a SHOW ... IN SCHEMA read is disambiguated once per plan with SHOW SCHEMAS LIKE ... IN DATABASE: a missing schema lists empty (a fresh target plans as a first run), a missing or unusable database fails naming it, and an existing schema re-raises the original error. - Bounded inspection reads started inside a bounded worker run serially, so capped-schema fallback chunks share the outer concurrency bound. - Semantic source candidates are deduplicated by relation identity, so a relation seen by both SHOW and the name lookup is matched once. Refs CHI-455
1 parent 91bd1cc commit ed7c6d3

15 files changed

Lines changed: 497 additions & 79 deletions

File tree

‎src/sqlbuild/adapter/relations/_helpers/inspection_context.py‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818
_ACTIVE_CATALOG: ContextVar[InspectionCatalog | None] = ContextVar(
1919
"sqlbuild_inspection_catalog", default=None
2020
)
21+
_INSIDE_WORKER: ContextVar[bool] = ContextVar("sqlbuild_inside_inspection_worker", default=False)
2122
_QUERY_SINK: ContextVar[Callable[[InspectionQueryRecord], None] | None] = ContextVar(
2223
"sqlbuild_inspection_query_sink", default=None
2324
)
@@ -72,3 +73,15 @@ def deactivate_query_sink(token: Token[Callable[[InspectionQueryRecord], None] |
7273
"""Restore the sink that was active before ``activate_query_sink``."""
7374

7475
_QUERY_SINK.reset(token)
76+
77+
78+
def inside_inspection_worker() -> bool:
79+
"""Return whether this thread is running one bounded inspection read."""
80+
81+
return _INSIDE_WORKER.get()
82+
83+
84+
def mark_inspection_worker() -> None:
85+
"""Mark this worker context so nested bounded reads run serially."""
86+
87+
_ = _INSIDE_WORKER.set(True)

‎src/sqlbuild/adapter/relations/classes/inspection_catalog.py‎

Lines changed: 15 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,9 +2,10 @@
22

33
from __future__ import annotations
44

5+
import threading
56
import time
67
from collections.abc import Callable
7-
from typing import Any
8+
from typing import Any, cast
89

910
from sqlbuild.adapter.contract.models import ColumnInfo, RelationInfo
1011
from sqlbuild.adapter.relations._helpers.inspection_context import emit_inspection_record
@@ -38,9 +39,22 @@ def __init__(self, *, adapter: Any, connection: Any) -> None:
3839
self._relation_listings: dict[_SchemaKey, SchemaRelationListing] = {}
3940
self._column_listings: dict[_SchemaKey, SchemaColumnListing] = {}
4041
self._capped_column_schemas: set[_SchemaKey] = set()
42+
self._remembered: dict[tuple[str, ...], object] = {}
43+
self._remember_lock: threading.Lock = threading.Lock()
4144
self._relation_requests: dict[_RelationRequestKey, tuple[RelationInfo, ...]] = {}
4245
self._relation_columns: dict[_RelationIdentity, tuple[ColumnInfo, ...] | None] = {}
4346

47+
def remember[ValueT](self, *, key: tuple[str, ...], compute: Callable[[], ValueT]) -> ValueT:
48+
"""Compute one adapter-owned metadata fact at most once per planning invocation."""
49+
50+
with self._remember_lock:
51+
if key in self._remembered:
52+
return cast(ValueT, self._remembered[key])
53+
value: ValueT = compute()
54+
with self._remember_lock:
55+
self._remembered.setdefault(key, value)
56+
return value
57+
4458
@property
4559
def schema_scoped(self) -> bool:
4660
"""Return whether schema listings come from one case-aware read per schema."""

‎src/sqlbuild/adapter/relations/main/run_bounded_inspections.py‎

Lines changed: 12 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,10 @@
77
from contextvars import copy_context
88
from typing import cast
99

10+
from sqlbuild.adapter.relations._helpers.inspection_context import (
11+
inside_inspection_worker,
12+
mark_inspection_worker,
13+
)
1014
from sqlbuild.adapter.relations.types import InspectionCompletion
1115

1216

@@ -16,9 +20,9 @@ def run_bounded_inspections[ResultT](
1620
concurrency: int,
1721
on_complete: InspectionCompletion[ResultT] | None = None,
1822
) -> list[ResultT]:
19-
"""Return results in input order, report completions on this thread, raise lowest failure."""
23+
"""Return ordered results, raising the lowest failure; nested calls run serially."""
2024

21-
if concurrency <= 1 or len(tasks) <= 1:
25+
if concurrency <= 1 or len(tasks) <= 1 or inside_inspection_worker():
2226
results: list[ResultT] = []
2327
index: int
2428
task: Callable[[], ResultT]
@@ -47,7 +51,7 @@ def _collect[ResultT](
4751
"""Stop starting tasks after a failure, let running ones finish, raise the lowest failure."""
4852

4953
futures: dict[Future[ResultT], int] = {
50-
cast(Future[ResultT], pool.submit(copy_context().run, task)): index
54+
cast(Future[ResultT], pool.submit(copy_context().run, _run_as_worker, task)): index
5155
for index, task in enumerate(tasks)
5256
}
5357
completed: dict[int, ResultT] = {}
@@ -71,3 +75,8 @@ def _collect[ResultT](
7175
if failures:
7276
raise failures[min(failures)]
7377
return completed
78+
79+
80+
def _run_as_worker[ResultT](task: Callable[[], ResultT]) -> ResultT:
81+
_ = mark_inspection_worker()
82+
return task()

‎src/sqlbuild/adapters/snowflake/_helpers/show_metadata.py‎

Lines changed: 8 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -8,11 +8,12 @@
88
from sqlbuild.adapter.relations.main.relation_age_timestamp import relation_age_timestamp_utc
99
from sqlbuild.adapter.relations.models import ListedRelation
1010

11-
_MISSING_OBJECT_ERRNO: int = 2003
12-
_MISSING_OBJECT_MESSAGE: str = "does not exist or not authorized"
11+
_MISSING_OBJECT_ERRNOS: frozenset[int] = frozenset({2003, 2043})
12+
_MISSING_OBJECT_MESSAGES: tuple[str, ...] = (
13+
"does not exist or not authorized",
14+
"object does not exist, or operation cannot be performed",
15+
)
1316
_TRUE_FLAGS: frozenset[str] = frozenset({"Y", "YES", "TRUE"})
14-
_MISSING_SCHEMA_MARKER: str = "schema '"
15-
_MISSING_DATABASE_MARKER: str = "database '"
1617
_TRANSIENT_KIND: str = "TRANSIENT"
1718
_TEMPORARY_KIND: str = "TEMPORARY"
1819

@@ -54,22 +55,11 @@ def listed_relation_from_show_view(
5455

5556

5657
def is_missing_object_error(error: Exception) -> bool:
57-
"""Return whether SHOW failed because the schema is absent or invisible to the role."""
58-
59-
return (
60-
getattr(error, "errno", None) == _MISSING_OBJECT_ERRNO
61-
or _MISSING_OBJECT_MESSAGE in str(error).lower()
62-
)
63-
64-
65-
def is_missing_schema_error(error: Exception) -> bool:
66-
"""Return whether SHOW failed for a missing schema; a missing database is a real failure."""
58+
"""Return whether SHOW failed because an object is absent or invisible to the role."""
6759

6860
message: str = str(error).lower()
69-
return (
70-
is_missing_object_error(error)
71-
and _MISSING_SCHEMA_MARKER in message
72-
and _MISSING_DATABASE_MARKER not in message
61+
return getattr(error, "errno", None) in _MISSING_OBJECT_ERRNOS or any(
62+
marker in message for marker in _MISSING_OBJECT_MESSAGES
7363
)
7464

7565

‎src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py‎

Lines changed: 91 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -116,7 +116,6 @@
116116
)
117117
from sqlbuild.adapters.snowflake._helpers.show_metadata import (
118118
is_missing_object_error,
119-
is_missing_schema_error,
120119
listed_relation_from_show_table,
121120
listed_relation_from_show_view,
122121
)
@@ -1817,14 +1816,23 @@ def read_schema_relation_listing(
18171816
) -> SchemaRelationListing:
18181817
"""List one schema's tables and views with SHOW, which needs no running warehouse."""
18191818

1819+
show_database: str = self._show_database(
1820+
connection=connection, database=database, schema=schema
1821+
)
18201822
scope: str = self._show_schema_scope(
18211823
connection=connection, database=database, schema=schema
18221824
)
18231825
tables: list[dict[str, object]] = self._fetch_paged_show_rows(
1824-
connection=connection, query=f"SHOW TABLES IN SCHEMA {scope}"
1826+
connection=connection,
1827+
query=f"SHOW TABLES IN SCHEMA {scope}",
1828+
show_database=show_database,
1829+
schema=schema,
18251830
)
18261831
views: list[dict[str, object]] = self._fetch_paged_show_rows(
1827-
connection=connection, query=f"SHOW VIEWS IN SCHEMA {scope}"
1832+
connection=connection,
1833+
query=f"SHOW VIEWS IN SCHEMA {scope}",
1834+
show_database=show_database,
1835+
schema=schema,
18281836
)
18291837
entries: dict[str, ListedRelation] = {}
18301838
row: dict[str, object]
@@ -1853,8 +1861,13 @@ def read_schema_column_listing(
18531861
scope: str = self._show_schema_scope(
18541862
connection=connection, database=database, schema=schema
18551863
)
1856-
rows: list[dict[str, object]] = self._fetch_show_rows(
1857-
connection=connection, query=f"SHOW COLUMNS IN SCHEMA {scope}"
1864+
rows: list[dict[str, object]] = self._fetch_schema_show_rows(
1865+
connection=connection,
1866+
query=f"SHOW COLUMNS IN SCHEMA {scope}",
1867+
show_database=self._show_database(
1868+
connection=connection, database=database, schema=schema
1869+
),
1870+
schema=schema,
18581871
)
18591872
if len(rows) != _SHOW_RESULT_LIMIT:
18601873
columns: dict[str, list[ColumnInfo]] = {}
@@ -1955,14 +1968,20 @@ def _show_schema_scope(
19551968
) -> str:
19561969
"""Qualify SHOW with the database so a missing one is reported, never listed empty."""
19571970

1971+
resolved: str = self._show_database(connection=connection, database=database, schema=schema)
1972+
return f"{self.render_identifier(resolved)}.{self.render_identifier(schema)}"
1973+
1974+
def _show_database(
1975+
self, *, connection: _SnowflakeConnection, database: str | None, schema: str
1976+
) -> str:
19581977
resolved: str | None = (
19591978
database if database is not None else self._current_database(connection=connection)
19601979
)
19611980
if resolved is None:
19621981
raise AdapterUserError(
19631982
message=f"Snowflake schema {schema} needs a database; the session has none"
19641983
)
1965-
return f"{self.render_identifier(resolved)}.{self.render_identifier(schema)}"
1984+
return resolved
19661985

19671986
@staticmethod
19681987
def _current_database(*, connection: _SnowflakeConnection) -> str | None:
@@ -1980,15 +1999,15 @@ def _current_database(*, connection: _SnowflakeConnection) -> str | None:
19801999
return current
19812000

19822001
def _fetch_paged_show_rows(
1983-
self, *, connection: _SnowflakeConnection, query: str
2002+
self, *, connection: _SnowflakeConnection, query: str, show_database: str, schema: str
19842003
) -> list[dict[str, object]]:
19852004
"""Page SHOW output past its row cap; SHOW orders rows by name."""
19862005

19872006
rows: list[dict[str, object]] = []
19882007
page_query: str = f"{query} LIMIT {_SHOW_RESULT_LIMIT}"
19892008
while True:
1990-
page: list[dict[str, object]] = self._fetch_show_rows(
1991-
connection=connection, query=page_query
2009+
page: list[dict[str, object]] = self._fetch_schema_show_rows(
2010+
connection=connection, query=page_query, show_database=show_database, schema=schema
19922011
)
19932012
rows.extend(page)
19942013
if len(page) < _SHOW_RESULT_LIMIT:
@@ -2000,24 +2019,80 @@ def _fetch_paged_show_rows(
20002019
def _fetch_show_rows(
20012020
*, connection: _SnowflakeConnection, query: str
20022021
) -> list[dict[str, object]]:
2003-
"""Run one SHOW command; a missing or unauthorized schema lists nothing."""
2022+
"""Run one SHOW command and return its rows keyed by lowercase output column."""
20042023

20052024
def fetch() -> list[dict[str, object]]:
20062025
cursor: Any = connection.cursor()
20072026
try:
2008-
try:
2009-
cursor.execute(query)
2010-
except Exception as error:
2011-
if is_missing_schema_error(error):
2012-
return []
2013-
raise
2027+
cursor.execute(query)
20142028
names: list[str] = [str(column[0]).lower() for column in cursor.description or ()]
20152029
return [dict(zip(names, row, strict=False)) for row in cursor.fetchall()]
20162030
finally:
20172031
cursor.close()
20182032

20192033
return run_recorded_inspection_query(sql=query, run=fetch)
20202034

2035+
def _fetch_schema_show_rows(
2036+
self, *, connection: _SnowflakeConnection, query: str, show_database: str, schema: str
2037+
) -> list[dict[str, object]]:
2038+
"""Run SHOW ... IN SCHEMA; a schema that does not exist lists nothing, as before."""
2039+
2040+
try:
2041+
return self._fetch_show_rows(connection=connection, query=query)
2042+
except Exception as error:
2043+
if not is_missing_object_error(error) or self._schema_exists(
2044+
connection=connection, show_database=show_database, schema=schema
2045+
):
2046+
raise
2047+
return []
2048+
2049+
def _schema_exists(
2050+
self, *, connection: _SnowflakeConnection, show_database: str, schema: str
2051+
) -> bool:
2052+
"""Tell a missing schema from a missing database or another failure, once per plan."""
2053+
2054+
def read() -> bool:
2055+
return self._read_schema_exists(
2056+
connection=connection, show_database=show_database, schema=schema
2057+
)
2058+
2059+
catalog: InspectionCatalog | None = active_inspection_catalog(
2060+
adapter=self, connection=connection
2061+
)
2062+
if catalog is None:
2063+
return read()
2064+
return catalog.remember(
2065+
key=(
2066+
"snowflake_schema_exists",
2067+
self._information_schema_identifier(show_database),
2068+
self._information_schema_identifier(schema),
2069+
),
2070+
compute=read,
2071+
)
2072+
2073+
def _read_schema_exists(
2074+
self, *, connection: _SnowflakeConnection, show_database: str, schema: str
2075+
) -> bool:
2076+
stored_schema: str = self._information_schema_identifier(schema)
2077+
query: str = (
2078+
f"SHOW SCHEMAS LIKE '{stored_schema.replace(chr(39), chr(39) * 2)}' "
2079+
f"IN DATABASE {self.render_identifier(show_database)}"
2080+
)
2081+
try:
2082+
rows: list[dict[str, object]] = self._fetch_show_rows(
2083+
connection=connection, query=query
2084+
)
2085+
except Exception as error:
2086+
if not is_missing_object_error(error):
2087+
raise
2088+
raise AdapterUserError(
2089+
message=(
2090+
f"Snowflake database {show_database} does not exist or the current role "
2091+
f"cannot use it: {error}"
2092+
)
2093+
) from error
2094+
return any(str(row.get("name")) == stored_schema for row in rows)
2095+
20212096
def read_relation_columns(
20222097
self, *, connection: _SnowflakeConnection, relation: RelationInfo
20232098
) -> tuple[ColumnInfo, ...] | None:

‎src/sqlbuild/compiler/planner/_helpers/warehouse/semantic_sources.py‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -122,7 +122,11 @@ def _list_source_candidates(
122122
names=tuple(sorted({entry.table or entry.name for entry in pending})),
123123
)
124124
)
125-
return tuple(dict.fromkeys(relations))
125+
unique: dict[tuple[str | None, str | None, str], RelationInfo] = {}
126+
relation: RelationInfo
127+
for relation in relations:
128+
_ = unique.setdefault(relation.identity, relation)
129+
return tuple(unique.values())
126130

127131

128132
def _has_candidate(*, entry: SourceEntry, relations: list[RelationInfo]) -> bool:

‎tests/integration/src/sqlbuild/compiler/planner/main/_test_types.py‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -141,3 +141,22 @@ class SnowflakeManySchemasTestCase:
141141
models_per_schema: int
142142
expected_per_relation_column_reads: int
143143
expected_schema_column_reads: int
144+
145+
146+
@dataclass(frozen=True)
147+
class SnowflakeFreshTargetTestCase:
148+
"""Planning before the first build, when target schemas or the database do not exist."""
149+
150+
description: str
151+
warehouse_schemas: frozenset[str]
152+
expected_reason: str
153+
expected_schema_checks: int
154+
155+
156+
@dataclass(frozen=True)
157+
class SnowflakeMissingDatabaseTestCase:
158+
"""Planning against a database that does not exist or the role cannot use."""
159+
160+
description: str
161+
warehouse_database: str
162+
expected_error_fragment: str

‎tests/integration/src/sqlbuild/compiler/planner/main/helpers.py‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -62,6 +62,7 @@
6262
SourceCursorInputPlanErrorTestCase,
6363
)
6464
from tests.unit.src.sqlbuild.adapters.snowflake.inspection.helpers import (
65+
FakeRelation,
6566
OfflineSnowflakeAdapter,
6667
RecordedQuery,
6768
RecordingSnowflakeWarehouse,
@@ -534,3 +535,19 @@ def offline_cursor_bound_relations(warehouse: RecordingSnowflakeWarehouse) -> tu
534535
_CURSOR_BOUND_RELATION.findall(query.sql)[0]
535536
for query in warehouse.queries_of_kind("cursor_bounds")
536537
)
538+
539+
540+
def relations_in_schemas(
541+
*, project: SyntheticSnowflakeProject, schemas: frozenset[str]
542+
) -> tuple[FakeRelation, ...]:
543+
"""Keep only the synthetic relations that already exist in the given schemas."""
544+
545+
return tuple(filter(lambda relation: relation.schema in schemas, project.relations))
546+
547+
548+
def relations_in_database(
549+
*, project: SyntheticSnowflakeProject, database: str
550+
) -> tuple[FakeRelation, ...]:
551+
"""Move every synthetic relation into another database."""
552+
553+
return tuple(replace(relation, database=database) for relation in project.relations)

0 commit comments

Comments
 (0)