Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions crates/sqlbuild-python-base/src/records/constants.rs
Original file line number Diff line number Diff line change
Expand Up @@ -273,6 +273,7 @@ const BUILD_COMMAND_REQUEST: RecordSpec = RecordSpec {
FieldSpec::new("selector_files", FieldDefault::EmptyTuple),
FieldSpec::new("test_timeout", FieldDefault::None),
FieldSpec::new("discovered_inputs", FieldDefault::None),
FieldSpec::new("compiled_project", FieldDefault::None),
],
};
const BUILD_INVOCATION: RecordSpec = RecordSpec {
Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,10 @@
//! Compile the build plan, show it, gate it and write the compile target.

use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyErr, PyResult, Python};
use pyo3::prelude::{Bound, PyAny, PyAnyMethods, PyDictMethods, PyErr, PyResult, Python};
use pyo3::types::PyDict;
use sqlbuild_config::command_connection::constants::{
CONNECTION_SOURCE_KEY, DBT_PROFILE_CONNECTION_SOURCE,
};
use sqlbuild_core::build_command::constants::{
EXECUTION_LIMIT_BUILD_NOTE, MISSING_ORIGIN_BUILD_HELP, RETENTION_DECREASE_BUILD_HELP,
SNAPSHOT_FULL_REFRESH_BUILD_HELP, TABLE_TYPE_DOWNGRADE_BUILD_HELP,
Expand All @@ -13,7 +17,7 @@ use crate::bindings::_helpers::build_command::invocation::BuildContext;
use crate::bindings::_helpers::build_command::policies::enforce_plan_policies;
use crate::bindings::_helpers::build_command::policies::model_limit;
use crate::bindings::_helpers::command_runtime::audit_failures::resolve_ignore_audit_failures;
use crate::bindings::_helpers::compile_driver::plan_pipeline::run_compile_pipeline;
use crate::bindings::_helpers::compile_driver::plan_pipeline::planned_project;
use crate::bindings::_helpers::dbt_interop::reference_resolver::resolve_external_sql_reference_resolver;
use sqlbuild_python_base::boundary::main::call_with::call_with;
use sqlbuild_python_base::boundary::main::keywords::keywords;
Expand All @@ -27,6 +31,7 @@ use sqlbuild_python_base::records::main::construct::new_record_with::new_record_
use sqlbuild_python_base::records::main::fields::field::field as native_field;
use sqlbuild_python_base::records::types::RecordId;
use sqlbuild_python_compile::manifest::main::build_manifest::build_manifest;
use sqlbuild_python_front::compile_attachment::main::build_effective_connection_config::build_effective_connection_config;
use sqlbuild_python_output::build_progress::main::with_verbose_inspection::with_verbose_inspection;
use sqlbuild_python_planner::plan_output::main::format_plan::format_plan;

Expand Down Expand Up @@ -128,22 +133,50 @@ pub(crate) fn compile_build_plan<'py>(context: &BuildContext<'py>) -> PyResult<B
],
)?;
let verbose: bool = native_field(&context.request, "verbose")?.is_truthy()?;
let options: Bound<'py, PyAny> = request_options(context)?;
let held: Option<Bound<'py, PyAny>> = held_compile(context, &options)?;
with_verbose_inspection(py, verbose, || {
run_compile_pipeline(
py,
Some(keywords(
py,
&[
("discovered_inputs", context.get("discovered_inputs")?),
("adapter", context.get("adapter")?),
("options", request_options(context)?),
("hooks", hooks),
],
)?),
planned_project(
(&context.get("discovered_inputs")?, &context.get("adapter")?),
(&options, hooks),
(held.as_ref(), None),
)
})
}

/// The compile a worker verified, when compiling again could not differ: no run variables, validation and cache on, no dbt.
fn held_compile<'py>(
context: &BuildContext<'py>,
options: &Bound<'py, PyAny>,
) -> PyResult<Option<Bound<'py, PyAny>>> {
let held: Bound<'py, PyAny> = native_field(&context.request, "compiled_project")?;
let discovered: Bound<'py, PyAny> = context.get("discovered_inputs")?;
let project_config: Bound<'py, PyAny> = native_field(&discovered, "project_config")?;
let dbt_manifest: Bound<'py, PyAny> =
native_field(&native_field(&project_config, "dbt")?, "target_path")?;
if held.is_none()
|| native_field(options, "cli_vars")?.is_truthy()?
|| native_field(options, "no_sql_validation")?.is_truthy()?
|| native_field(options, "no_cache")?.is_truthy()?
|| !dbt_manifest.is_none()
{
return Ok(None);
}
let connection: Bound<'py, PyDict> = build_effective_connection_config(
&discovered,
Some(native_field(options, "selected_target")?),
None,
false,
)?
.cast_into()?;
let from_dbt_profile: bool = connection
.get_item(CONNECTION_SOURCE_KEY)?
.map(|source| source.eq(DBT_PROFILE_CONNECTION_SOURCE))
.transpose()?
.unwrap_or(false);
Ok((!from_dbt_profile).then_some(held))
}

/// Write the formatted plan to stdout, or stderr for debug and JSON runs.
pub(crate) fn write_plan_text(
context: &BuildContext<'_>,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -345,10 +345,10 @@ pub(crate) fn run_location<'py>(
arguments.value("run_id")?.extract::<String>()?,
arguments.value("claim_id")?.extract::<String>()?,
);
let ((snapshot, blocked, _), project) = compile_location(py, &arguments, &input)?;
let ((snapshot, blocked, compiled), project) = compile_location(py, &arguments, &input)?;
return execute_worker(
py,
(&snapshot, project, blocked),
((&snapshot, &compiled), project, blocked),
(&arguments, &input),
(&bearer, identity, recall),
);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -147,12 +147,12 @@ fn job_run_settings(
Ok(settings)
}

/// The frozen command's request and warehouse scope; build carries the verified inputs in its request.
/// The frozen command's request and warehouse scope; build carries the verified inputs and compiled project in its request.
fn command_request<'py>(
py: Python<'py>,
discovered: &Bound<'py, PyAny>,
arguments: &KeywordArguments<'py>,
worker: &Bound<'py, NativeOrchestrationWorker>,
(worker, compiled): (&Bound<'py, NativeOrchestrationWorker>, &Bound<'py, PyAny>),
) -> PyResult<(Bound<'py, PyAny>, Bound<'py, PyAny>)> {
let execution = worker.get().execution(py)?;
let frozen = execution.claim.request;
Expand Down Expand Up @@ -230,7 +230,13 @@ fn command_request<'py>(
}
let mut request = Invocation::new(&args)?.value(route.arguments[0].source)?;
if command == RunCommand::Build {
request = replaced(&request, &[("discovered_inputs", discovered.clone())])?;
request = replaced(
&request,
&[
("discovered_inputs", discovered.clone()),
("compiled_project", compiled.clone()),
],
)?;
}
Ok((request, warehouse_scope(&args)?))
}
Expand Down Expand Up @@ -343,7 +349,11 @@ fn record_compiled_vars<'py>(

pub(super) fn execute_worker<'py>(
py: Python<'py>,
(snapshot, project, blocked): (&NativeOrchestrationSnapshot, ProjectAdapter<'py>, NodeKeys),
((snapshot, compiled), project, blocked): (
(&NativeOrchestrationSnapshot, &Bound<'py, PyAny>),
ProjectAdapter<'py>,
NodeKeys,
),
(arguments, input): (&KeywordArguments<'py>, &SnapshotInput),
(bearer, identity, recall): (&str, (String, String), Option<&InvocationRecall>),
) -> PyResult<i32> {
Expand Down Expand Up @@ -385,7 +395,7 @@ pub(super) fn execute_worker<'py>(
return Err(PyValueError::new_err(reason));
}
record_compiled_vars(py, (&owner, &frozen), (arguments, input, &project))?;
let (request, scope) = command_request(py, &discovered, arguments, &owner)?;
let (request, scope) = command_request(py, &discovered, arguments, (&owner, compiled))?;
write_stderr(py, "Run inputs verified. Executing frozen selection...\n")?;
let _rules = RulesAtLoadScope::enter();
within(&scope, |_| {
Expand Down
2 changes: 2 additions & 0 deletions src/sqlbuild/_native.pyi
Original file line number Diff line number Diff line change
Expand Up @@ -6595,6 +6595,7 @@ class BuildCommandRequest(NativeRecord):
selector_files: tuple[SelectorFileSummary, ...]
test_timeout: str | None
discovered_inputs: DiscoveredProjectInputs | None
compiled_project: object | None
def __init__(
self,
project_dir: Path | None = None,
Expand Down Expand Up @@ -6637,6 +6638,7 @@ class BuildCommandRequest(NativeRecord):
selector_files: tuple[SelectorFileSummary, ...] = (),
test_timeout: str | None = None,
discovered_inputs: DiscoveredProjectInputs | None = None,
compiled_project: object | None = None,
) -> None: ...

class CloneCommandRequest(NativeRecord):
Expand Down
3 changes: 1 addition & 2 deletions src/sqlbuild/adapters/snowflake/classes/snowflake_adapter.py
Original file line number Diff line number Diff line change
Expand Up @@ -3434,10 +3434,9 @@ def load_seed(
) -> None:
del infer_types
if replace:
self.drop(
self.drop_table(
connection=connection,
destination=destination,
if_exists=True,
statement_recorder=statement_recorder,
)
column_defs: str = ", ".join(f"{col.name} {col.type}" for col in columns)
Expand Down
1 change: 1 addition & 0 deletions src/sqlbuild/executor/build/classes/build_scheduler.py
Original file line number Diff line number Diff line change
Expand Up @@ -1031,6 +1031,7 @@ def _execute_seed_node(self, *, key: CompiledObjectKey, connection: Any) -> Seed
run_id=self._run_id,
query_change_tracking=self._query_change_tracking,
schema_prepared=self._schema_prepared,
fingerprint_writes=self._fingerprint_writes,
)
if result.status == ExecutionStatus.SUCCESS:
result = self._with_seed_audits(key=key, result=result, connection=connection)
Expand Down
28 changes: 19 additions & 9 deletions src/sqlbuild/executor/pipeline/_helpers/seeding.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@

from __future__ import annotations

import queue
import time
from collections.abc import Callable
from concurrent.futures import Future, ThreadPoolExecutor
Expand All @@ -15,6 +16,7 @@
from sqlbuild.executor.pipeline.main.raise_warehouse_connection_failure import (
raise_warehouse_connection_failure,
)
from sqlbuild.executor.run.classes.fingerprint_writes import FingerprintWrites
from sqlbuild.executor.scheduling.types import ExecutionStatus
from sqlbuild.executor.seed.main._execute import execute_seed
from sqlbuild.runtime.contracts.types import ConnectionElapsedCallback
Expand Down Expand Up @@ -59,6 +61,11 @@ def run_seed_pipeline(
if on_connection_complete is not None:
on_connection_complete(effective_concurrency, elapsed_seconds=time.monotonic() - start)

pending: queue.SimpleQueue[tuple[int, SeedPlanEntry]] = queue.SimpleQueue()
indexed_entry: tuple[int, SeedPlanEntry]
for indexed_entry in enumerate(seed_entries):
pending.put(indexed_entry)
fingerprint_writes: FingerprintWrites = FingerprintWrites()
try:
with ThreadPoolExecutor(max_workers=effective_concurrency) as pool:
futures: tuple[Future[tuple[tuple[int, SeedExecutionResult], ...]], ...] = tuple(
Expand All @@ -67,9 +74,8 @@ def run_seed_pipeline(
pool.submit(
copy_context().run,
_run_seed_worker,
worker_index=worker_index,
effective_concurrency=effective_concurrency,
seed_entries=seed_entries,
pending=pending,
fingerprint_writes=fingerprint_writes,
adapter=adapter,
connection=connections[worker_index],
run_id=run_id,
Expand Down Expand Up @@ -97,21 +103,24 @@ def run_seed_pipeline(

def _run_seed_worker(
*,
worker_index: int,
effective_concurrency: int,
seed_entries: tuple[SeedPlanEntry, ...],
pending: queue.SimpleQueue[tuple[int, SeedPlanEntry]],
fingerprint_writes: FingerprintWrites,
adapter: BaseAdapter,
connection: Any,
run_id: str,
query_change_tracking: bool,
on_seed_complete: Callable[[SeedExecutionResult], None] | None,
) -> tuple[tuple[int, SeedExecutionResult], ...]:
"""Load seeds from the shared queue until it is empty, so a slow seed holds up only itself."""

indexed_results: list[tuple[int, SeedExecutionResult]] = []
seed_index: int
entry: SeedPlanEntry
for seed_index, entry in enumerate(seed_entries):
if seed_index % effective_concurrency != worker_index:
continue
while True:
try:
seed_index, entry = pending.get_nowait()
except queue.Empty:
break
with ResourceAttemptLifecycle(
resource_id=f"seed:{entry.name}",
resource_kind="seed",
Expand All @@ -125,6 +134,7 @@ def _run_seed_worker(
statement_recorder=StatementRecorder(),
run_id=run_id,
query_change_tracking=query_change_tracking,
fingerprint_writes=fingerprint_writes,
)
if result.status == ExecutionStatus.FAILED:
lifecycle.failed(error_code=result.error_code)
Expand Down
42 changes: 29 additions & 13 deletions src/sqlbuild/executor/seed/_helpers/fingerprinting.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,11 +3,14 @@
from __future__ import annotations

import hashlib
from contextlib import AbstractContextManager, nullcontext
from datetime import UTC, datetime
from typing import Any

from sqlbuild._native import NODE_TYPE_SEED, Fingerprint, SeedPlanEntry, write_fingerprint
from sqlbuild.adapter.contract.classes.base_adapter import BaseAdapter
from sqlbuild.executor.run.classes.fingerprint_writes import FingerprintWrites
from sqlbuild.executor.run.types import FingerprintTableLocation


def try_write_seed_fingerprint(
Expand All @@ -17,8 +20,9 @@ def try_write_seed_fingerprint(
connection: Any,
run_id: str,
query_change_tracking: bool,
fingerprint_writes: FingerprintWrites | None = None,
) -> tuple[str, ...]:
"""Write active-target seed identity after a successful load, returning warnings."""
"""Write a loaded seed's identity on the active target, setting each table up once per run."""

if not query_change_tracking:
return ()
Expand Down Expand Up @@ -48,19 +52,31 @@ def try_write_seed_fingerprint(
metadata_json=seed_entry.fingerprint_metadata_json,
ts=datetime.now(tz=UTC),
)
_ = write_fingerprint(
connection=connection,
execute=adapter.execute,
database=seed_entry.destination.database,
schema=target_schema,
fingerprint=fingerprint,
render_qualified_name=adapter.render_qualified_name,
render_framework_type=adapter.render_framework_type,
read_columns=adapter.get_columns,
render_add_columns=adapter.render_add_columns,
render_create_table_sql=adapter.render_create_fingerprint_table_sql,
render_create_index_sqls=adapter.render_create_fingerprint_index_sqls,
location: FingerprintTableLocation = (seed_entry.destination.database, target_schema)
setting_up: AbstractContextManager[bool] = (
nullcontext(True)
if fingerprint_writes is None
else fingerprint_writes.setting_up(location=location)
)
with setting_up as requested_setup:
set_up: bool = write_fingerprint(
ensure_table=requested_setup,
connection=connection,
execute=adapter.execute,
database=seed_entry.destination.database,
schema=target_schema,
fingerprint=fingerprint,
render_qualified_name=adapter.render_qualified_name,
render_framework_type=adapter.render_framework_type,
read_columns=adapter.get_columns,
render_add_columns=adapter.render_add_columns,
render_create_table_sql=adapter.render_create_fingerprint_table_sql,
render_create_index_sqls=adapter.render_create_fingerprint_index_sqls,
)
if fingerprint_writes is not None:
fingerprint_writes.written(
location=location, requested_setup=requested_setup, set_up=set_up
)
except Exception as exc:
return (
f"fingerprint write failed for seed '{seed_entry.name}'; "
Expand Down
3 changes: 3 additions & 0 deletions src/sqlbuild/executor/seed/main/_execute.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@
resolve_relation_location_qualified_name,
)
from sqlbuild.executor.build.models import SeedExecutionResult
from sqlbuild.executor.run.classes.fingerprint_writes import FingerprintWrites
from sqlbuild.executor.scheduling.types import ExecutionStatus
from sqlbuild.executor.seed._helpers.fingerprinting import try_write_seed_fingerprint
from sqlbuild.executor.seed.constants import SEED_LOAD_FAILED_CODE
Expand All @@ -26,6 +27,7 @@ def execute_seed(
run_id: str = "",
query_change_tracking: bool = False,
schema_prepared: bool = False,
fingerprint_writes: FingerprintWrites | None = None,
) -> SeedExecutionResult:
"""Load one seed into the warehouse."""

Expand Down Expand Up @@ -56,6 +58,7 @@ def execute_seed(
connection=connection,
run_id=run_id,
query_change_tracking=query_change_tracking,
fingerprint_writes=fingerprint_writes,
)
except Exception as exc:
return SeedExecutionResult(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -142,6 +142,8 @@ class WorkerExecutionTestCase:
expected_run_nodes: list[tuple[str, str, str | None, str | None, str | None]] = field(
default_factory=list
)
# Whether the build compiles again instead of planning from the compile the worker verified.
expected_compiles_again: bool = False


@dataclass(frozen=True)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -1037,6 +1037,17 @@ def worker_result(worker: ScheduledWorker, test_case: WorkerExecutionTestCase) -
).fetchall()
assert contexts, log.read_text()
assert all(context is not None for (context,) in contexts)
printed: str = "".join(
str(data)
for (data,) in control.execute(
"SELECT data FROM recall_output WHERE invocation_id LIKE 'worker/%' "
"ORDER BY occurred_at, sequence"
).fetchall()
)
assert ("Planning from the published compiled project." not in printed) == (
test_case.expected_compiles_again
), printed
assert ("Compiling project..." in printed) == test_case.expected_compiles_again, printed
expected_run_ids: set[str] = {
run_id
for (run_id,) in control.execute(
Expand Down
Loading
Loading