Skip to content

Commit 91bd1cc

Browse files
committed
fix(planner): harden shared inspection reads from review findings
- Semantic source fallback lists candidates by name in every schema and matches schemas case-insensitively, as before the catalog; schema-scoped catalogs answer declared schemas from listings first. - Bounded inspections cancel queued reads and re-raise promptly on any exception in the coordinating thread (including Ctrl-C); after a read fails, running reads finish and the lowest-index failure is raised. - Snowflake SHOW is always qualified with a database (the session's when none is given); only a missing schema lists empty, a missing database fails as before. - Column reads choose per-relation SHOW COLUMNS by the request's total size, as before, not per schema; a relation dropped after listing has no columns instead of failing. - A cap-sized SHOW COLUMNS result rereads only the needed relations from INFORMATION_SCHEMA in concurrent IN chunks of at most 50 names. Refs CHI-455
1 parent e75e2b2 commit 91bd1cc

19 files changed

Lines changed: 904 additions & 122 deletions

File tree

‎src/sqlbuild/adapter/relations/classes/inspection_catalog.py‎

Lines changed: 76 additions & 41 deletions
Original file line numberDiff line numberDiff line change
@@ -37,9 +37,16 @@ def __init__(self, *, adapter: Any, connection: Any) -> None:
3737
self._concurrency: int = max(1, int(getattr(adapter, "metadata_inspection_concurrency", 1)))
3838
self._relation_listings: dict[_SchemaKey, SchemaRelationListing] = {}
3939
self._column_listings: dict[_SchemaKey, SchemaColumnListing] = {}
40+
self._capped_column_schemas: set[_SchemaKey] = set()
4041
self._relation_requests: dict[_RelationRequestKey, tuple[RelationInfo, ...]] = {}
4142
self._relation_columns: dict[_RelationIdentity, tuple[ColumnInfo, ...] | None] = {}
4243

44+
@property
45+
def schema_scoped(self) -> bool:
46+
"""Return whether schema listings come from one case-aware read per schema."""
47+
48+
return self._reader is not None
49+
4350
@property
4451
def concurrency(self) -> int:
4552
"""Return the bounded number of concurrent reads this adapter supports."""
@@ -237,75 +244,103 @@ def _read_columns_directly(self, *, relations: tuple[RelationInfo, ...]) -> None
237244
def _read_columns_by_schema(
238245
self, *, reader: SchemaScopedMetadataReader, relations: tuple[RelationInfo, ...]
239246
) -> None:
247+
"""Use exact per-relation reads for small requests and one read per schema otherwise."""
248+
240249
unscoped: list[RelationInfo] = []
241-
by_schema: dict[_SchemaKey, list[RelationInfo]] = {}
250+
remaining: list[RelationInfo] = []
242251
relation: RelationInfo
243252
for relation in relations:
244253
if relation.schema is None:
245254
unscoped.append(relation)
246255
continue
247-
by_schema.setdefault(
248-
self._schema_key(database=relation.database, schema=relation.schema), []
249-
).append(relation)
256+
listing: SchemaColumnListing | None = self._column_listings.get(
257+
self._schema_key(database=relation.database, schema=relation.schema)
258+
)
259+
if listing is None:
260+
remaining.append(relation)
261+
continue
262+
self._relation_columns[relation.identity] = listing.columns_by_stored_name.get(
263+
reader.metadata_name_key(relation.name)
264+
)
250265
if unscoped:
251266
self._read_columns_directly(relations=tuple(unscoped))
252-
listing_keys: list[_SchemaKey] = []
253-
listing_tasks: list[Callable[[], SchemaColumnListing]] = []
254-
exact_relations: list[RelationInfo] = []
255-
schema_key: _SchemaKey
256-
scoped: list[RelationInfo]
257-
for schema_key, scoped in by_schema.items():
258-
if schema_key in self._column_listings:
259-
continue
260-
if len(scoped) <= reader.exact_column_inspection_limit:
261-
exact_relations.extend(scoped)
262-
continue
263-
listing_keys.append(schema_key)
264-
listing_tasks.append(
267+
if not remaining:
268+
return
269+
if len(remaining) <= reader.exact_column_inspection_limit:
270+
exact_results: list[tuple[ColumnInfo, ...] | None] = run_bounded_inspections(
271+
tasks=tuple(
272+
self._bind_relation_columns(reader=reader, relation=exact)
273+
for exact in remaining
274+
),
275+
concurrency=self._concurrency,
276+
)
277+
exact_columns: tuple[ColumnInfo, ...] | None
278+
for relation, exact_columns in zip(remaining, exact_results, strict=True):
279+
self._relation_columns[relation.identity] = exact_columns
280+
return
281+
by_schema: dict[_SchemaKey, list[RelationInfo]] = {}
282+
for relation in remaining:
283+
by_schema.setdefault(
284+
self._schema_key(database=relation.database, schema=relation.schema or ""), []
285+
).append(relation)
286+
listings: list[SchemaColumnListing] = run_bounded_inspections(
287+
tasks=tuple(
265288
self._bind_column_listing(
266-
reader=reader, database=scoped[0].database, schema=scoped[0].schema or ""
289+
reader=reader,
290+
database=scoped[0].database,
291+
schema=scoped[0].schema or "",
292+
stored_names=self._stored_names(reader=reader, relations=scoped),
293+
known_capped=schema_key in self._capped_column_schemas,
267294
)
268-
)
269-
exact_tasks: list[Callable[[], tuple[ColumnInfo, ...]]] = [
270-
self._bind_relation_columns(reader=reader, relation=exact) for exact in exact_relations
271-
]
272-
results: list[SchemaColumnListing | tuple[ColumnInfo, ...]] = run_bounded_inspections(
273-
tasks=(*listing_tasks, *exact_tasks), concurrency=self._concurrency
295+
for schema_key, scoped in by_schema.items()
296+
),
297+
concurrency=self._concurrency,
274298
)
275-
listing: SchemaColumnListing | tuple[ColumnInfo, ...]
276-
for schema_key, listing in zip(listing_keys, results[: len(listing_keys)], strict=True):
277-
if isinstance(listing, SchemaColumnListing):
278-
self._column_listings[schema_key] = listing
279-
exact: RelationInfo
280-
exact_columns: SchemaColumnListing | tuple[ColumnInfo, ...]
281-
for exact, exact_columns in zip(exact_relations, results[len(listing_keys) :], strict=True):
282-
if not isinstance(exact_columns, SchemaColumnListing):
283-
self._relation_columns[exact.identity] = exact_columns
284-
for schema_key, scoped in by_schema.items():
285-
column_listing: SchemaColumnListing | None = self._column_listings.get(schema_key)
286-
if column_listing is None:
287-
continue
299+
schema_key: _SchemaKey
300+
scoped: list[RelationInfo]
301+
column_listing: SchemaColumnListing
302+
for (schema_key, scoped), column_listing in zip(by_schema.items(), listings, strict=True):
303+
if column_listing.complete:
304+
self._column_listings[schema_key] = column_listing
305+
else:
306+
self._capped_column_schemas.add(schema_key)
288307
for relation in scoped:
289308
self._relation_columns[relation.identity] = (
290309
column_listing.columns_by_stored_name.get(
291310
reader.metadata_name_key(relation.name)
292311
)
293312
)
294313

314+
@staticmethod
315+
def _stored_names(
316+
*, reader: SchemaScopedMetadataReader, relations: list[RelationInfo]
317+
) -> frozenset[str]:
318+
return frozenset(reader.metadata_name_key(relation.name) for relation in relations)
319+
295320
def _bind_column_listing(
296-
self, *, reader: SchemaScopedMetadataReader, database: str | None, schema: str
321+
self,
322+
*,
323+
reader: SchemaScopedMetadataReader,
324+
database: str | None,
325+
schema: str,
326+
stored_names: frozenset[str],
327+
known_capped: bool,
297328
) -> Callable[[], SchemaColumnListing]:
298329
def read_listing() -> SchemaColumnListing:
299330
return reader.read_schema_column_listing(
300-
connection=self.connection, database=database, schema=schema
331+
connection=self.connection,
332+
database=database,
333+
schema=schema,
334+
stored_names=stored_names,
335+
known_capped=known_capped,
301336
)
302337

303338
return read_listing
304339

305340
def _bind_relation_columns(
306341
self, *, reader: SchemaScopedMetadataReader, relation: RelationInfo
307-
) -> Callable[[], tuple[ColumnInfo, ...]]:
308-
def read_columns() -> tuple[ColumnInfo, ...]:
342+
) -> Callable[[], tuple[ColumnInfo, ...] | None]:
343+
def read_columns() -> tuple[ColumnInfo, ...] | None:
309344
return reader.read_relation_columns(connection=self.connection, relation=relation)
310345

311346
return read_columns

‎src/sqlbuild/adapter/relations/main/run_bounded_inspections.py‎

Lines changed: 36 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -28,28 +28,46 @@ def run_bounded_inspections[ResultT](
2828
on_complete(index=index, result=result)
2929
results.append(result)
3030
return results
31+
pool: ThreadPoolExecutor = ThreadPoolExecutor(max_workers=min(concurrency, len(tasks)))
32+
try:
33+
completed: dict[int, ResultT] = _collect(pool=pool, tasks=tasks, on_complete=on_complete)
34+
except BaseException:
35+
pool.shutdown(wait=False, cancel_futures=True)
36+
raise
37+
pool.shutdown(wait=True)
38+
return [completed[index] for index in range(len(tasks))]
39+
40+
41+
def _collect[ResultT](
42+
*,
43+
pool: ThreadPoolExecutor,
44+
tasks: Sequence[Callable[[], ResultT]],
45+
on_complete: InspectionCompletion[ResultT] | None,
46+
) -> dict[int, ResultT]:
47+
"""Stop starting tasks after a failure, let running ones finish, raise the lowest failure."""
48+
49+
futures: dict[Future[ResultT], int] = {
50+
cast(Future[ResultT], pool.submit(copy_context().run, task)): index
51+
for index, task in enumerate(tasks)
52+
}
3153
completed: dict[int, ResultT] = {}
3254
failures: dict[int, BaseException] = {}
33-
with ThreadPoolExecutor(max_workers=min(concurrency, len(tasks))) as pool:
34-
futures: dict[Future[ResultT], int] = {
35-
cast(Future[ResultT], pool.submit(copy_context().run, task)): index
36-
for index, task in enumerate(tasks)
37-
}
38-
future: Future[ResultT]
39-
for future in as_completed(futures):
40-
if future.cancelled():
41-
continue
42-
future_index: int = futures[future]
43-
error: BaseException | None = future.exception()
44-
if error is not None:
45-
failures[future_index] = error
55+
future: Future[ResultT]
56+
for future in as_completed(futures):
57+
if future.cancelled():
58+
continue
59+
future_index: int = futures[future]
60+
error: BaseException | None = future.exception()
61+
if error is not None:
62+
if not failures:
4663
pending: Future[ResultT]
4764
for pending in futures:
4865
_ = pending.cancel()
49-
continue
50-
completed[future_index] = future.result()
51-
if on_complete is not None and not failures:
52-
on_complete(index=future_index, result=completed[future_index])
66+
failures[future_index] = error
67+
continue
68+
completed[future_index] = future.result()
69+
if on_complete is not None and not failures:
70+
on_complete(index=future_index, result=completed[future_index])
5371
if failures:
5472
raise failures[min(failures)]
55-
return [completed[index] for index in range(len(tasks))]
73+
return completed

‎src/sqlbuild/adapter/relations/models.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@ class SchemaColumnListing:
3131
database: str | None
3232
schema: str
3333
columns_by_stored_name: dict[str, tuple[ColumnInfo, ...]]
34+
complete: bool = True
3435

3536

3637
@dataclass(frozen=True)

‎src/sqlbuild/adapter/relations/types.py‎

Lines changed: 10 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -31,15 +31,21 @@ def read_schema_relation_listing(
3131
...
3232

3333
def read_schema_column_listing(
34-
self, *, connection: Any, database: str | None, schema: str
34+
self,
35+
*,
36+
connection: Any,
37+
database: str | None,
38+
schema: str,
39+
stored_names: frozenset[str],
40+
known_capped: bool = False,
3541
) -> SchemaColumnListing:
36-
"""Read every column in one schema."""
42+
"""Read one schema's columns; an incomplete listing covers only ``stored_names``."""
3743
...
3844

3945
def read_relation_columns(
4046
self, *, connection: Any, relation: RelationInfo
41-
) -> tuple[ColumnInfo, ...]:
42-
"""Read one relation's columns."""
47+
) -> tuple[ColumnInfo, ...] | None:
48+
"""Read one relation's columns, or None when it no longer exists."""
4349
...
4450

4551
def metadata_name_key(self, name: str) -> str:

‎src/sqlbuild/adapters/snowflake/_helpers/show_metadata.py‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,8 @@
1111
_MISSING_OBJECT_ERRNO: int = 2003
1212
_MISSING_OBJECT_MESSAGE: str = "does not exist or not authorized"
1313
_TRUE_FLAGS: frozenset[str] = frozenset({"Y", "YES", "TRUE"})
14+
_MISSING_SCHEMA_MARKER: str = "schema '"
15+
_MISSING_DATABASE_MARKER: str = "database '"
1416
_TRANSIENT_KIND: str = "TRANSIENT"
1517
_TEMPORARY_KIND: str = "TEMPORARY"
1618

@@ -60,6 +62,17 @@ def is_missing_object_error(error: Exception) -> bool:
6062
)
6163

6264

65+
def is_missing_schema_error(error: Exception) -> bool:
66+
"""Return whether SHOW failed for a missing schema; a missing database is a real failure."""
67+
68+
message: str = str(error).lower()
69+
return (
70+
is_missing_object_error(error)
71+
and _MISSING_SCHEMA_MARKER in message
72+
and _MISSING_DATABASE_MARKER not in message
73+
)
74+
75+
6376
def _listed_relation(
6477
*,
6578
row: Mapping[str, object],

0 commit comments

Comments
 (0)