diff --git a/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_planning.rs b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_planning.rs index a6988a45e3..671e95bf4b 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_planning.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/janitor/janitor_planning.rs @@ -2,9 +2,11 @@ use std::collections::HashSet; +use pyo3::IntoPyObjectExt; use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::pyfunction; use pyo3::types::{PyBool, PyDict, PyDictMethods}; +use sqlbuild_analysis::assembly::target_checks::constants::PRESERVE_TARGET_VALUE; use sqlbuild_analysis::janitor_planning::constants::OLD_NAME_VIEW_REASON; use sqlbuild_analysis::janitor_planning::main::build_archive_name::build_archive_name; use sqlbuild_analysis::janitor_planning::main::query_diff_artifact_name::query_diff_artifact_name; @@ -123,6 +125,13 @@ fn candidate_records<'py>(candidates: &[Candidate<'py>]) -> PyResult, field: &str) -> PyResult> { + Ok(native_field(project, field)? + .extract::>()? + .filter(|value| value != PRESERVE_TARGET_VALUE)) +} + /// Review schemas whose markers the active target schema holds and that outlived any review run. fn review_schema_candidates<'py>( project: &Bound<'py, PyAny>, @@ -130,10 +139,9 @@ fn review_schema_candidates<'py>( now: &Bound<'py, PyAny>, ) -> PyResult>> { let py: Python<'py> = warehouse.py(); - let schema: Bound<'py, PyAny> = native_field(project, "effective_target_schema")?; - if schema.is_none() { + let Some(schema) = target_location(project, "effective_target_schema")? else { return Ok(Vec::new()); - } + }; let max_age: Bound<'py, PyAny> = days(py, LEFTOVER_REVIEW_SCHEMA_AGE_DAYS)?; let leftovers: Bound<'py, PyAny> = call_with( &host_entry(py, ExecutorHost::LeftoverReviewSchemas)?, @@ -142,9 +150,9 @@ fn review_schema_candidates<'py>( ("connection", warehouse.connection.clone()), ( "database", - native_field(project, "effective_target_database")?, + target_location(project, "effective_target_database")?.into_bound_py_any(py)?, ), - ("schema", schema), + ("schema", schema.into_bound_py_any(py)?), ("now", now.clone()), ("max_age", max_age), ], @@ -208,11 +216,9 @@ pub(crate) fn build_plan<'py>( push_artifact_schema(database.clone(), schema.clone()); } } - if let Some(schema) = - native_field(project, "effective_target_schema")?.extract::>()? - { + if let Some(schema) = target_location(project, "effective_target_schema")? { push_artifact_schema( - native_field(project, "effective_target_database")?.extract()?, + target_location(project, "effective_target_database")?, schema, ); } diff --git a/crates/sqlbuild-recall/src/writer/tests/helpers.rs b/crates/sqlbuild-recall/src/writer/tests/helpers.rs index ee089fca63..70e34fa417 100644 --- a/crates/sqlbuild-recall/src/writer/tests/helpers.rs +++ b/crates/sqlbuild-recall/src/writer/tests/helpers.rs @@ -12,6 +12,7 @@ use crate::store::main::open_sqlite_store::open_sqlite_store; use crate::store::main::query_store::query_store; use crate::store::main::read_output_transcript::read_output_transcript; use crate::store::models::OpenMode; +use crate::writer::_helpers::delivery::locked; use crate::writer::_helpers::queue::WriterState; use crate::writer::main::start_recall_session::start_recall_session; use crate::writer::models::{ @@ -175,8 +176,8 @@ pub(crate) fn unwritable_session() -> (bool, bool) { ) } -/// Seconds `close` took and whether delivery degraded, while another connection holds the write lock. -pub(crate) fn close_under_held_lock(shutdown: Duration) -> (f64, bool) { +/// While another connection holds the write lock: whether closing set its deadline exactly the budget after it began, the degraded reason, and the seconds `close` took. +pub(crate) fn close_under_held_lock(shutdown: Duration) -> (bool, Option, f64) { let directory = tempfile::tempdir().expect("a temporary directory can be created"); let path = directory.path().join("logs/recall.sqlite3"); let warm = start_recall_session(config(path.clone(), "warm")); @@ -191,10 +192,19 @@ pub(crate) fn close_under_held_lock(shutdown: Duration) -> (f64, bool) { }); session.append_output(OutputStream::Stdout, b"waiting\n"); let started = std::time::Instant::now(); + session.begin_close(); + let began = std::time::Instant::now(); + let deadline = locked(&session.shared.state) + .deadline + .expect("closing sets a deadline"); let summary = session.close(); let elapsed = started.elapsed().as_secs_f64(); drop(holder); - (elapsed, summary.degraded_reason.is_some()) + ( + started + shutdown <= deadline && deadline <= began + shutdown, + summary.degraded_reason, + elapsed, + ) } /// The notices and resulting store state after starting on a file that is not a database. diff --git a/crates/sqlbuild-recall/src/writer/tests/test_session.rs b/crates/sqlbuild-recall/src/writer/tests/test_session.rs index bce9001c10..e361a55722 100644 --- a/crates/sqlbuild-recall/src/writer/tests/test_session.rs +++ b/crates/sqlbuild-recall/src/writer/tests/test_session.rs @@ -24,6 +24,8 @@ use crate::writer::tests::test_types::{ }; const CLOSE_SLACK_SECONDS: f64 = 0.3; +/// Far beyond any budget a test sets; only a close that waits for the store itself reaches it. +const CLOSE_HANG_GUARD_SECONDS: f64 = 60.0; #[test] fn given_a_session_when_closed_then_every_record_and_the_delivery_rows_are_written() { @@ -87,25 +89,28 @@ fn given_an_unwritable_store_when_started_then_setup_reports_permission_and_clos #[test] fn given_a_held_write_lock_when_closed_then_the_shutdown_budget_caps_the_delay() { let test_cases = [HeldLockTestCase { - description: "a locked store costs at most the shutdown budget at exit", + description: "a locked store gives up at the deadline the budget sets when closing begins", shutdown_ms: 700, - expected_degraded: true, + expected_degraded_reason: Some( + "Recall delivery did not finish within the shutdown timeout", + ), }]; for test_case in test_cases { - let (elapsed, degraded) = + let (deadline_is_the_budget, degraded_reason, elapsed) = close_under_held_lock(Duration::from_millis(test_case.shutdown_ms)); - let budget: f64 = Duration::from_millis(test_case.shutdown_ms).as_secs_f64(); - assert!( - elapsed <= budget + CLOSE_SLACK_SECONDS, - "{}: {elapsed:.3}s", - test_case.description - ); + assert!(deadline_is_the_budget, "{}", test_case.description); assert_eq!( - degraded, test_case.expected_degraded, + degraded_reason.as_deref(), + test_case.expected_degraded_reason, "{}", test_case.description ); + assert!( + elapsed < CLOSE_HANG_GUARD_SECONDS, + "{}: close hung for {elapsed:.3}s", + test_case.description + ); } } diff --git a/crates/sqlbuild-recall/src/writer/tests/test_types.rs b/crates/sqlbuild-recall/src/writer/tests/test_types.rs index 6f42c0c016..80f944ace2 100644 --- a/crates/sqlbuild-recall/src/writer/tests/test_types.rs +++ b/crates/sqlbuild-recall/src/writer/tests/test_types.rs @@ -51,7 +51,7 @@ pub(crate) struct UnwritableTestCase { pub(crate) struct HeldLockTestCase { pub(crate) description: &'static str, pub(crate) shutdown_ms: u64, - pub(crate) expected_degraded: bool, + pub(crate) expected_degraded_reason: Option<&'static str>, } pub(crate) struct CorruptStoreTestCase { diff --git a/fensu.toml b/fensu.toml index dd9592d42d..e2cbdb90cf 100644 --- a/fensu.toml +++ b/fensu.toml @@ -1178,7 +1178,8 @@ max_positional_args = 1 [[targets.python.threshold_overrides]] paths = ["src/sqlbuild/__init__.py"] # Raised from 89,457 for the shared Python identity pass (identity_pass.py, object_source.py). -thresholds = { max_crate_code_lines = 89626 } +# Raised from 89,626 for helper-class anchors in Python identity (object_source.py, identity.py). +thresholds = { max_crate_code_lines = 89643 } reason = "The runtime package is above the code-line limit; this caps it at its current size until it is split into packages by area. Lower the cap as code moves out." [targets.python.rule_options.XSB074] @@ -1703,7 +1704,9 @@ select = ["FPRS", "XSB072", "XSB073", "XSB075", "XSB076", "XSB077", "XSB078", "X [[targets.rust.threshold_overrides]] paths = ["crates/sqlbuild-python/Cargo.toml"] -thresholds = { max_crate_code_lines = 206367 } +# Raised from 206,364: +3 for the build identity crate split (#1510), +4 for preserve target +# locations in janitor planning (#1512). +thresholds = { max_crate_code_lines = 206371 } reason = "The Python bindings crate is above the code-line limit; this caps it at its current size until it is split into crates by area. Lower the cap as code moves out." [[targets.rust.threshold_overrides]] diff --git a/src/sqlbuild/compiler/python_nodes/_helpers/identity.py b/src/sqlbuild/compiler/python_nodes/_helpers/identity.py index dab6c17c4d..3bd3b9ee12 100644 --- a/src/sqlbuild/compiler/python_nodes/_helpers/identity.py +++ b/src/sqlbuild/compiler/python_nodes/_helpers/identity.py @@ -13,7 +13,7 @@ from typing import Any, cast from sqlbuild._native import PythonIdentityDependency, PythonNodeIdentity -from sqlbuild.compiler.python_nodes._helpers.object_source import hash_text +from sqlbuild.compiler.python_nodes._helpers.object_source import class_anchor, hash_text from sqlbuild.compiler.python_nodes.classes.identity_pass import IdentityPass _IGNORED_PATH_PARTS: frozenset[str] = frozenset( @@ -182,8 +182,9 @@ def _visit_object( collected_dependencies: dict[tuple[str, str], PythonIdentityDependency] = dependencies collected_dependencies[key] = dependency + unwrapped: object = inspect.unwrap(cast(Any, obj)) globals_map: Mapping[str, object] | None = getattr( - inspect.unwrap(cast(Any, obj)), "__globals__", None + class_anchor(unwrapped) or unwrapped, "__globals__", None ) if globals_map is None: module: ModuleType | None = memo.module(obj) diff --git a/src/sqlbuild/compiler/python_nodes/_helpers/object_source.py b/src/sqlbuild/compiler/python_nodes/_helpers/object_source.py index b35cff7f39..3a7d7b7d60 100644 --- a/src/sqlbuild/compiler/python_nodes/_helpers/object_source.py +++ b/src/sqlbuild/compiler/python_nodes/_helpers/object_source.py @@ -6,23 +6,19 @@ import hashlib import inspect import linecache -import sys import textwrap from collections.abc import Callable from pathlib import Path -from types import ModuleType +from types import FunctionType, ModuleType from typing import Any, cast -_CLASS_FIRSTLINENO_VERSION: tuple[int, int] = (3, 13) -_FINDSOURCE_PARSES_FILE_PER_CLASS: bool = sys.version_info < _CLASS_FIRSTLINENO_VERSION - def normalized_source(*, obj: object, class_starts: Callable[..., dict[str, int]]) -> str: - """`inspect.getsource(obj)` dedented and stripped, finding classes through `class_starts`.""" + """`inspect.getsource(obj)` dedented and stripped; classes read from their own file.""" try: target: object = inspect.unwrap(cast(Any, obj)) - if not (_FINDSOURCE_PARSES_FILE_PER_CLASS and inspect.isclass(target)): + if not inspect.isclass(target): return textwrap.dedent(inspect.getsource(cast(Any, obj))).strip() lines, start = _class_location(cast(type, target), class_starts=class_starts) return textwrap.dedent("".join(inspect.getblock(lines[start:]))).strip() @@ -34,7 +30,7 @@ def source_file(obj: object) -> Path | None: """The file `inspect` reads `obj`'s source from, or `None` when it has none.""" try: - raw_path: str | None = inspect.getsourcefile(cast(Any, obj)) + raw_path: str | None = inspect.getsourcefile(cast(Any, class_anchor(obj) or obj)) except TypeError: return None if raw_path is None: @@ -66,6 +62,27 @@ def called_references(source: str) -> tuple[tuple[str, ...], ...]: return tuple(sorted(refs)) +def class_anchor(obj: object) -> FunctionType | None: + """A method of class `obj`; its file and globals outlive the module's sys.modules entry.""" + + if not inspect.isclass(obj): + return None + value: object + for value in vars(obj).values(): + method: object = value.__func__ if isinstance(value, staticmethod | classmethod) else value + if not isinstance(method, FunctionType): + continue + function: object = inspect.unwrap(method) + if ( + isinstance(function, FunctionType) + and function.__qualname__.startswith(f"{obj.__qualname__}.") + and function.__module__ == obj.__module__ + and not function.__code__.co_filename.startswith("<") + ): + return function + return None + + def _reference_parts(node: ast.AST) -> tuple[str, ...] | None: if isinstance(node, ast.Name): return (node.id,) @@ -86,21 +103,22 @@ def parse_class_starts(lines: list[str]) -> dict[str, int]: def _class_location( cls: type, *, class_starts: Callable[..., dict[str, int]] ) -> tuple[list[str], int]: - """The lines and start index `inspect.findsource` returns for `cls` on Python 3.12.""" - - file: str | None = inspect.getsourcefile(cls) - if file: - linecache.checkcache(file) - else: - file = inspect.getfile(cls) - if not (file.startswith("<") and file.endswith(">")): - raise OSError("source code not available") + """The lines and start index `inspect.findsource` returns for `cls`, read from its own file.""" + + path: Path | None = source_file(cls) + if path is None: + raise OSError("source code not available") + file: str = str(path) + linecache.checkcache(file) module: ModuleType | None = inspect.getmodule(cls, file) lines: list[str] = ( linecache.getlines(file, module.__dict__) if module else linecache.getlines(file) ) if not lines: raise OSError("could not get source code") + first_line: object = vars(cls).get("__firstlineno__") + if isinstance(first_line, int): + return lines, first_line - 1 start: int | None = class_starts(file=file, lines=lines).get(cls.__qualname__) if start is None: raise OSError("could not find class definition") diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/test_clickhouse_recall_views.py b/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/test_clickhouse_recall_views.py index 58c19acd4e..b5f6aa8be2 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/test_clickhouse_recall_views.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/clickhouse/test_clickhouse_recall_views.py @@ -375,6 +375,15 @@ def test_given_one_invocation_lookup_when_planned_then_each_table_is_read_once_b parent=tmp_path, config=clickhouse_e2e_config, database=recall_database ) assert run(project_dir, "build").returncode == 0 + # A command that ends before its first heartbeat writes none, and ClickHouse plans a table + # with no parts as a read of nothing, so give the heartbeats table a part before planning. + execute_clickhouse_sql( + config=clickhouse_e2e_config, + statements=( + f"INSERT INTO {recall_database}.recall_heartbeats (invocation_id) " + "VALUES ('planning-setup')", + ), + ) sql: str = ( f"SELECT * FROM {recall_database}.{test_case.view} WHERE invocation_id = 'any-invocation'" ) diff --git a/tests/integration/src/sqlbuild/cli/recall/_test_types.py b/tests/integration/src/sqlbuild/cli/recall/_test_types.py index 62838d5545..e161327af8 100644 --- a/tests/integration/src/sqlbuild/cli/recall/_test_types.py +++ b/tests/integration/src/sqlbuild/cli/recall/_test_types.py @@ -97,6 +97,9 @@ class HeldLockTestCase: shutdown_seconds: float builds_before_lock: int expected_exit_code: int + expected_notice: str + #: Far beyond the budget; only a build that waits for the lock itself reaches it. + hang_guard_seconds: float @dataclass(frozen=True) diff --git a/tests/integration/src/sqlbuild/cli/recall/helpers.py b/tests/integration/src/sqlbuild/cli/recall/helpers.py index 7b4fbf0819..569ed78183 100644 --- a/tests/integration/src/sqlbuild/cli/recall/helpers.py +++ b/tests/integration/src/sqlbuild/cli/recall/helpers.py @@ -349,20 +349,12 @@ def _still_running(process: subprocess.Popen[bytes]) -> bool: return process.poll() is None -def timed_build(project_dir: Path) -> tuple[subprocess.CompletedProcess[bytes], float, float]: - """A build with the seconds until its first stderr byte and until it exited.""" +def timed_build(project_dir: Path) -> tuple[subprocess.CompletedProcess[bytes], float]: + """A build with the seconds until it exited.""" started: float = time.monotonic() - process: subprocess.Popen[bytes] = start(project_dir, "build") - assert process.stderr is not None - first_byte: bytes = process.stderr.read(1) - first_output: float = time.monotonic() - started - stdout, stderr = process.communicate(timeout=600) - return ( - subprocess.CompletedProcess(process.args, process.returncode, stdout, first_byte + stderr), - first_output, - time.monotonic() - started, - ) + completed: subprocess.CompletedProcess[bytes] = run(project_dir, "build") + return completed, time.monotonic() - started @contextmanager @@ -395,7 +387,7 @@ def held_store_lock(project_dir: Path) -> Iterator[None]: def build_while_store_locked( project_dir: Path, -) -> tuple[subprocess.CompletedProcess[bytes], float, float]: +) -> tuple[subprocess.CompletedProcess[bytes], float]: """A timed build while another process holds the store's write lock.""" with held_store_lock(project_dir): diff --git a/tests/integration/src/sqlbuild/cli/recall/test_local_recall.py b/tests/integration/src/sqlbuild/cli/recall/test_local_recall.py index 2cb811acc8..7c8f5eec54 100644 --- a/tests/integration/src/sqlbuild/cli/recall/test_local_recall.py +++ b/tests/integration/src/sqlbuild/cli/recall/test_local_recall.py @@ -45,7 +45,6 @@ recall_rows, run, simultaneous_builds, - timed_build, write_corrupt_store, write_project, ) @@ -542,41 +541,39 @@ def test_given_a_successful_command_when_recorded_then_it_ends_completed( "test_case", [ HeldLockTestCase( - description="a store locked by another process costs at most the budget", + description="a store locked after setup gives up at the shutdown deadline", shutdown_seconds=1.0, builds_before_lock=1, expected_exit_code=0, + expected_notice="Recall delivery did not finish within the shutdown timeout", + hang_guard_seconds=60.0, ), HeldLockTestCase( - description="a store another process holds before setup never delays the start", + description="a store locked before setup gives up at the shutdown deadline", shutdown_seconds=1.0, builds_before_lock=0, expected_exit_code=0, + expected_notice="Recall delivery did not finish within the shutdown timeout", + hang_guard_seconds=60.0, ), ], ids=lambda case: case.description, ) -def test_given_a_locked_store_when_building_then_start_and_exit_each_add_at_most_the_budget( +def test_given_a_locked_store_when_building_then_it_gives_up_at_the_shutdown_deadline( test_case: HeldLockTestCase, tmp_path: Path ) -> None: budget: str = f'\n[recall]\nshutdown_timeout = "{int(test_case.shutdown_seconds * 1000)}ms"\n' - baseline_dir: Path = prepared_project(tmp_path / "baseline", budget, builds=1) project_dir: Path = prepared_project( tmp_path / "locked", budget, builds=test_case.builds_before_lock ) - _, baseline_first, baseline_total = timed_build(baseline_dir) - locked, locked_first, locked_total = build_while_store_locked(project_dir) + locked, locked_total = build_while_store_locked(project_dir) - slack: float = 0.75 + stderr: str = locked.stderr.decode(errors="replace") assert locked.returncode == test_case.expected_exit_code - assert locked_first <= baseline_first + slack - assert locked_total <= baseline_total + test_case.shutdown_seconds + slack - assert ( - locked.stderr.decode(errors="replace") - .rstrip() - .endswith("completed successfully; only its Recall history is affected.") - ) + assert test_case.expected_notice in stderr + assert stderr.rstrip().endswith("completed successfully; only its Recall history is affected.") + assert locked_total < test_case.hang_guard_seconds, stderr @pytest.mark.parametrize( diff --git a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_python_nodes.py b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_python_nodes.py index 463cbfeedd..0588ad2f31 100644 --- a/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_python_nodes.py +++ b/tests/unit/src/sqlbuild/compiler/discovery/_helpers/test_filesystem_python_nodes.py @@ -558,6 +558,71 @@ def test_given_python_root_helper_edit_when_rediscovering_then_node_identity_cha project_files=_HELPER_PACKAGE_FILES, expected_dependency_path="python/helpers/clean.py", ), + RestoredPythonPackageIdentityTestCase( + description="helper called from a helper class method stays a dependency", + unrelated_files={"site/python/__init__.py": "", "site/python/orders.py": "X = 1\n"}, + project_files={ + "python/helpers/__init__.py": "", + "python/helpers/text.py": """ +def strip_status(value): + return value.strip().lower() +""", + "python/helpers/cleaner.py": """ +from .text import strip_status + + +class Cleaner: + def run(self, value): + return strip_status(value) +""", + "python/orders.py": """ +import python.helpers.cleaner +from sqlbuild.tasks import task + + +@task +def orders(ctx): + return python.helpers.cleaner.Cleaner().run(" Shipped ") +""", + }, + expected_dependency_path="python/helpers/text.py", + ), + RestoredPythonPackageIdentityTestCase( + description="a class attribute whose attribute lookup loops does not stop identity", + unrelated_files={"site/python/__init__.py": "", "site/python/orders.py": "X = 1\n"}, + project_files={ + "python/helpers/__init__.py": "", + "python/helpers/text.py": """ +def strip_status(value): + return value.strip().lower() +""", + "python/helpers/cleaner.py": """ +from .text import strip_status + + +class Lazy: + def __getattr__(self, name): + return self + + +class Cleaner: + lazy = Lazy() + + def run(self, value): + return strip_status(value) +""", + "python/orders.py": """ +import python.helpers.cleaner +from sqlbuild.tasks import task + + +@task +def orders(ctx): + return python.helpers.cleaner.Cleaner().run(" Shipped ") +""", + }, + expected_dependency_path="python/helpers/text.py", + ), ], ids=lambda case: case.description, ) diff --git a/tests/unit/src/sqlbuild/executor/janitor/main/_test_types.py b/tests/unit/src/sqlbuild/executor/janitor/main/_test_types.py index b891ddf619..83c394c15d 100644 --- a/tests/unit/src/sqlbuild/executor/janitor/main/_test_types.py +++ b/tests/unit/src/sqlbuild/executor/janitor/main/_test_types.py @@ -31,6 +31,15 @@ class JanitorPlanTestCase: expected_suppressed_candidate_names: tuple[str, ...] = field(default_factory=tuple) +@dataclass(frozen=True) +class JanitorTargetSchemaTestCase: + description: str + target_database: str | None + target_schema: str + location: tuple[str | None, str] + expected_inspected: bool + + @dataclass(frozen=True) class JanitorExecuteTestCase: description: str diff --git a/tests/unit/src/sqlbuild/executor/janitor/main/helpers.py b/tests/unit/src/sqlbuild/executor/janitor/main/helpers.py index 2ea022a89c..a708950eac 100644 --- a/tests/unit/src/sqlbuild/executor/janitor/main/helpers.py +++ b/tests/unit/src/sqlbuild/executor/janitor/main/helpers.py @@ -79,6 +79,7 @@ def __init__( self.dropped_view_targets: list[str] = [] self.renamed_targets: list[tuple[str, str]] = [] self.executed_sql: list[str] = [] + self.listed_locations: list[tuple[str | None, str]] = [] self.tracked_relations: tuple[tuple[str | None, str | None, str], ...] = tracked_relations self._tracked_rows: tuple[tuple[Any, ...], ...] = tuple( ( @@ -172,6 +173,7 @@ def list_relations( schemas: tuple[str, ...] | None, names: tuple[str, ...] | None = None, ) -> tuple[RelationInfo, ...]: + self.listed_locations.extend((database, schema) for schema in schemas or ()) selected: list[RelationInfo] = [] available_schemas: tuple[str, ...] = schemas or tuple( relation.schema or "" for relation in self._available_relations @@ -548,6 +550,8 @@ def build_project( destination_database: str | None = None, destination_schema: str = "analytics", migrate_from: str | None = None, + target_schema: str | None = None, + target_database: str | None = None, ) -> CompiledProjectView: source: CompiledSourceView = CompiledSource( key=CompiledObjectKey( @@ -568,6 +572,8 @@ def build_project( return CompiledProject( run_id="run-1", effective_target_name="dev", + effective_target_schema=target_schema, + effective_target_database=target_database, effective_connection={}, effective_vars={}, models=( diff --git a/tests/unit/src/sqlbuild/executor/janitor/main/test_plan.py b/tests/unit/src/sqlbuild/executor/janitor/main/test_plan.py index 4f85fd29c9..3ef55f16c1 100644 --- a/tests/unit/src/sqlbuild/executor/janitor/main/test_plan.py +++ b/tests/unit/src/sqlbuild/executor/janitor/main/test_plan.py @@ -19,6 +19,7 @@ from tests.unit.src.sqlbuild.executor.janitor.main._test_types import ( JanitorExecuteTestCase, JanitorPlanTestCase, + JanitorTargetSchemaTestCase, relation_info, ) from tests.unit.src.sqlbuild.executor.janitor.main.helpers import ( @@ -263,6 +264,52 @@ def test_given_project_and_warehouse_when_building_janitor_plan_then_returns_exp assert tuple(suppressed_candidate_names) == test_case.expected_suppressed_candidate_names +@pytest.mark.parametrize( + "test_case", + [ + JanitorTargetSchemaTestCase( + description="a real target schema is inspected for artifacts", + target_database=None, + target_schema="dev", + location=(None, "dev"), + expected_inspected=True, + ), + JanitorTargetSchemaTestCase( + description="a preserve target schema names no schema to inspect", + target_database=None, + target_schema="preserve", + location=(None, "preserve"), + expected_inspected=False, + ), + JanitorTargetSchemaTestCase( + description="a preserve target database names no database for the target schema", + target_database="preserve", + target_schema="dev", + location=(None, "dev"), + expected_inspected=True, + ), + ], + ids=lambda case: case.description, +) +def test_given_a_target_schema_when_building_janitor_plan_then_only_a_real_schema_is_inspected( + test_case: JanitorTargetSchemaTestCase, +) -> None: + adapter: FakeJanitorAdapter = FakeJanitorAdapter(relation_infos=()) + + _ = build_janitor_plan( + project=build_project( + target_schema=test_case.target_schema, target_database=test_case.target_database + ), + adapter=adapter, + connection=object(), + retention_days=7, + direct_mode=True, + ) + + assert (test_case.location in adapter.listed_locations) == test_case.expected_inspected + assert all(database != "preserve" for database, _ in adapter.listed_locations) + + @pytest.mark.parametrize( "test_case", [