diff --git a/benchmarks/duckshop/edge_cases/orchestration/worker_failed_seed_audit.yml b/benchmarks/duckshop/edge_cases/orchestration/worker_failed_seed_audit.yml new file mode 100644 index 0000000000..56598946dc --- /dev/null +++ b/benchmarks/duckshop/edge_cases/orchestration/worker_failed_seed_audit.yml @@ -0,0 +1,9 @@ +seeds: + - name: order_codes + description: Order codes, one of which the audit does not accept. + columns: + - name: code + type: VARCHAR + audits: + - accepted_values: + values: [A, B] diff --git a/clients/typescript/src/schema.d.ts b/clients/typescript/src/schema.d.ts index ed4b1c4350..cde7ee2d37 100644 --- a/clients/typescript/src/schema.d.ts +++ b/clients/typescript/src/schema.d.ts @@ -1450,7 +1450,10 @@ export interface components { /** Format: int64 */ priority: number; reason: string; - /** Format: uuid */ + /** + * Format: uuid + * @description A new random (UUID v4) or time-ordered (UUID v7) ID; name-based (v5) IDs are refused as the server derives those. + */ run_id: string; target: string; }; @@ -1883,7 +1886,10 @@ export interface components { /** Format: int64 */ priority: number; reason: string; - /** Format: uuid */ + /** + * Format: uuid + * @description A new random (UUID v4) or time-ordered (UUID v7) ID; name-based (v5) IDs are refused as the server derives those. + */ run_id: string; /** @description Narrow the job with `sqb build` selectors; only nodes already in the job can remain. */ select?: string[]; @@ -2684,7 +2690,10 @@ export interface components { /** Format: int64 */ priority: number; reason: string; - /** Format: uuid */ + /** + * Format: uuid + * @description A new random (UUID v4) or time-ordered (UUID v7) ID; name-based (v5) IDs are refused as the server derives those. + */ run_id: string; /** Format: uuid */ source_run_id: string; @@ -6040,6 +6049,14 @@ export interface operations { "application/json": components["schemas"]["MutationResponse"]; }; }; + 400: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["ApiErrorBody"]; + }; + }; /** @description Session rejected */ 401: { headers: { @@ -6200,6 +6217,14 @@ export interface operations { "application/json": components["schemas"]["MutationResponse"]; }; }; + 400: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["ApiErrorBody"]; + }; + }; /** @description Session rejected */ 401: { headers: { diff --git a/crates/sqlbuild-orchestration/src/api/_helpers/resources/person_operations.rs b/crates/sqlbuild-orchestration/src/api/_helpers/resources/person_operations.rs index fd51e5d8cc..05566e385e 100644 --- a/crates/sqlbuild-orchestration/src/api/_helpers/resources/person_operations.rs +++ b/crates/sqlbuild-orchestration/src/api/_helpers/resources/person_operations.rs @@ -67,7 +67,8 @@ pub(crate) async fn statistics( #[utoipa::path(post, path = "/api/control/runs", operation_id = "requestJobRun", params(("x-sqb-csrf" = String, Header)), request_body = JobRunRequest, - responses((status = 200, body = MutationResponse), (status = 401, description = "Session rejected"), + responses((status = 200, body = MutationResponse), + (status = 400, body = crate::api::models::ApiErrorBody), (status = 401, description = "Session rejected"), (status = 403, body = crate::api::models::ApiErrorBody)), security(("session" = []), ("cliSession" = [])))] pub(crate) async fn request_job( (State(store), Extension(context)): (State, Extension), @@ -148,7 +149,8 @@ pub(crate) async fn cancel_run( #[utoipa::path(post, path = "/api/control/runs/rerun", operation_id = "rerunPersonRun", params(("x-sqb-csrf" = String, Header)), request_body = RerunRequest, - responses((status = 200, body = MutationResponse), (status = 401, description = "Session rejected"), + responses((status = 200, body = MutationResponse), + (status = 400, body = crate::api::models::ApiErrorBody), (status = 401, description = "Session rejected"), (status = 403, body = crate::api::models::ApiErrorBody)), security(("session" = []), ("cliSession" = [])))] pub(crate) async fn rerun( (State(store), Extension(context)): (State, Extension), diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/requests/adhoc.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/requests/adhoc.rs index bcb1eddb60..7a7d6be746 100644 --- a/crates/sqlbuild-orchestration/src/runs/_helpers/requests/adhoc.rs +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/requests/adhoc.rs @@ -14,6 +14,7 @@ use crate::runs::_helpers::requests::permissions::{authorize_execution, executio use crate::runs::_helpers::requests::publication::{ publish_request, record_authored, request_retry, }; +use crate::runs::_helpers::requests::run_id::refuse_derived_run_id; use crate::runs::constants::ADHOC_AUDIT_TAG; use crate::runs::models::{ AdhocCheckRequest, AdhocPermission, AdhocRunRequest, CommandPlan, PlannerResource, RunCommand, @@ -222,6 +223,7 @@ pub(in crate::runs) async fn request_adhoc_run_in( (context, principal_id): (&AuditContext, Uuid), (planner, request): (&dyn CommandPlanner, &AdhocRunRequest), ) -> Result<(ControlTransaction, bool), ControlError> { + refuse_derived_run_id(request.run_id)?; let (transaction, context) = authorize_listing( transaction, (context, principal_id), diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/requests/manual_job.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/requests/manual_job.rs index 19c55a45f3..5d84876841 100644 --- a/crates/sqlbuild-orchestration/src/runs/_helpers/requests/manual_job.rs +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/requests/manual_job.rs @@ -12,6 +12,7 @@ use crate::runs::_helpers::requests::permissions::authorize_execution; use crate::runs::_helpers::requests::publication::{ publish_request, record_authored, request_retry, }; +use crate::runs::_helpers::requests::run_id::refuse_derived_run_id; use crate::runs::constants::MANUAL_JOB_AUDIT_TAG; use crate::runs::main::frozen_job_request::frozen_job_request; use crate::runs::models::{JobRunRequest, RunWindow}; @@ -26,6 +27,7 @@ pub(crate) async fn request_job_run_in( (context, credentials): (&AuditContext, Option), (planner, request): (Option<&dyn CommandPlanner>, &JobRunRequest), ) -> Result<(ControlTransaction, bool), ControlError> { + refuse_derived_run_id(request.run_id)?; let mut context = context.clone(); let scope = PermissionScope { action: Action::JobRun, diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/requests/manual_rerun.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/requests/manual_rerun.rs index bd338a9933..37b046d5c9 100644 --- a/crates/sqlbuild-orchestration/src/runs/_helpers/requests/manual_rerun.rs +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/requests/manual_rerun.rs @@ -10,6 +10,7 @@ use crate::runs::_helpers::requests::publication::{ publish_request, record_authored, request_retry, }; use crate::runs::_helpers::requests::rerun::frozen_rerun; +use crate::runs::_helpers::requests::run_id::refuse_derived_run_id; use crate::runs::constants::RERUN_AUDIT_TAG; use crate::runs::models::{RerunRequest, RunRequest}; use crate::store::errors::ControlError; @@ -22,6 +23,7 @@ pub(crate) async fn request_rerun_in( (context, credentials): (&AuditContext, Option), request: &RerunRequest, ) -> Result<(ControlTransaction, bool), ControlError> { + refuse_derived_run_id(request.run_id)?; let mut context = context.clone(); let mut prepared = None; if let Some(credentials) = credentials { diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/requests/mod.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/requests/mod.rs index 85c7ebfc6d..419cf76bf9 100644 --- a/crates/sqlbuild-orchestration/src/runs/_helpers/requests/mod.rs +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/requests/mod.rs @@ -7,3 +7,4 @@ pub(in crate::runs) mod permissions; pub(in crate::runs) mod plan; pub(in crate::runs) mod publication; pub(in crate::runs) mod rerun; +pub(in crate::runs) mod run_id; diff --git a/crates/sqlbuild-orchestration/src/runs/_helpers/requests/run_id.rs b/crates/sqlbuild-orchestration/src/runs/_helpers/requests/run_id.rs new file mode 100644 index 0000000000..a96443fbfb --- /dev/null +++ b/crates/sqlbuild-orchestration/src/runs/_helpers/requests/run_id.rs @@ -0,0 +1,18 @@ +//! A run ID a person supplies must not be of the kind the server derives for its own runs. + +use uuid::Uuid; + +use crate::runs::constants::DERIVED_RUN_ID_VERSION; +use crate::store::errors::ControlError; + +/// Refuses a name-based run ID, the kind the server derives for group members and retries. +pub(in crate::runs) fn refuse_derived_run_id(run_id: Uuid) -> Result<(), ControlError> { + if run_id.get_version_num() != DERIVED_RUN_ID_VERSION { + return Ok(()); + } + Err(ControlError::Invalid(format!( + "run_id {run_id} is a name-based UUID (version 5), which the server reserves for the IDs it \ + derives for group members and retries; send a random (version 4) or time-ordered \ + (version 7) UUID" + ))) +} diff --git a/crates/sqlbuild-orchestration/src/runs/constants.rs b/crates/sqlbuild-orchestration/src/runs/constants.rs index 574237df98..3a18b7d850 100644 --- a/crates/sqlbuild-orchestration/src/runs/constants.rs +++ b/crates/sqlbuild-orchestration/src/runs/constants.rs @@ -38,3 +38,5 @@ pub(crate) const DEFAULT_RUN_LOG_RETENTION_COUNT: u16 = 200; pub(crate) const RUN_LOG_RETENTION_BATCH: usize = 10; /// Artifact digests checked for remaining references per statement. pub(crate) const RUN_LOG_DIGEST_CHUNK: usize = 500; +/// The UUID version of the run IDs the server derives from other IDs, which people may not supply. +pub(crate) const DERIVED_RUN_ID_VERSION: usize = 5; diff --git a/crates/sqlbuild-orchestration/src/runs/models.rs b/crates/sqlbuild-orchestration/src/runs/models.rs index 6b92161618..90423bd549 100644 --- a/crates/sqlbuild-orchestration/src/runs/models.rs +++ b/crates/sqlbuild-orchestration/src/runs/models.rs @@ -188,6 +188,7 @@ impl RunRequest { #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] #[serde(deny_unknown_fields)] pub struct JobRunRequest { + /// A new random (UUID v4) or time-ordered (UUID v7) ID; name-based (v5) IDs are refused as the server derives those. pub run_id: Uuid, pub location_id: Uuid, pub target: String, @@ -211,6 +212,7 @@ pub struct JobRunRequest { #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] #[serde(deny_unknown_fields)] pub struct RerunRequest { + /// A new random (UUID v4) or time-ordered (UUID v7) ID; name-based (v5) IDs are refused as the server derives those. pub run_id: Uuid, pub source_run_id: Uuid, pub from_failure: bool, @@ -742,6 +744,7 @@ pub struct RunObservation { #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)] #[serde(deny_unknown_fields)] pub struct AdhocRunRequest { + /// A new random (UUID v4) or time-ordered (UUID v7) ID; name-based (v5) IDs are refused as the server derives those. pub run_id: Uuid, pub location_id: Uuid, pub target: String, diff --git a/crates/sqlbuild-orchestration/tests/api/test_contracts.rs b/crates/sqlbuild-orchestration/tests/api/test_contracts.rs index b3297708b9..d7e3e17264 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_contracts.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_contracts.rs @@ -88,6 +88,8 @@ mod test_publish_encoding; mod test_reads; #[path = "test_resource_costs.rs"] mod test_resource_costs; +#[path = "test_run_ids.rs"] +mod test_run_ids; #[path = "test_server_errors.rs"] mod test_server_errors; #[path = "test_statistics_recovery.rs"] diff --git a/crates/sqlbuild-orchestration/tests/api/test_run_ids.rs b/crates/sqlbuild-orchestration/tests/api/test_run_ids.rs new file mode 100644 index 0000000000..87aa51f573 --- /dev/null +++ b/crates/sqlbuild-orchestration/tests/api/test_run_ids.rs @@ -0,0 +1,141 @@ +//! People cannot start runs under the name-based IDs the server derives for its own runs. + +use serde_json::{Value, json}; +use uuid::{Builder, Uuid}; + +use crate::api::models::{ApiAuthentication, HumanLogin}; +use crate::api::tests::denials_helpers::{job, person_send}; +use crate::api::tests::helpers::{start_api_with_authentication, stop_api}; +use crate::api::tests::test_types::RunIdRefusalTestCase; +use crate::credentials::tests::role_helpers::role_fixture; +use crate::runs::main::request_job_run::request_job_run; +use crate::runs::models::JobRunRequest; +use crate::runs::tests::helpers::snapshot; +use crate::snapshots::main::publish_snapshot::publish_snapshot; +use crate::store::main::migrate_store::migrate_store; +use crate::store::tests::store_helpers::{close_test_store, open_test_store, run}; +use crate::store::tests::store_test_types::Backend; + +#[test] +fn given_a_name_based_run_id_when_a_person_requests_a_run_rerun_or_adhoc_run_then_it_is_refused() { + let test_cases = [ + RunIdRefusalTestCase { + description: "SQLite run IDs", + backend: Backend::Sqlite, + expected_refusal_status: 400, + expected_random_status: 200, + }, + RunIdRefusalTestCase { + description: "PostgreSQL run IDs", + backend: Backend::Postgres, + expected_refusal_status: 400, + expected_random_status: 200, + }, + ]; + for test_case in test_cases { + run(async { + let Some(fixture) = open_test_store(test_case.backend).await else { + return; + }; + let store = &fixture.store; + migrate_store(store).await.expect("migrate"); + let roles = role_fixture(store).await; + let mut published = snapshot(roles.location_id); + published.generation = 2; + published.documents.insert( + "jobs".into(), + json!({"orders": {"description": "Build orders", "selection": ["orders"]}}), + ); + publish_snapshot(store, &published).await.expect("snapshot"); + let source = JobRunRequest { + run_id: Uuid::now_v7(), + location_id: roles.location_id, + target: "development".into(), + job_key: "orders".into(), + reason: "Refresh orders".into(), + priority: 0, + window: None, + full_refresh: false, + select: Vec::new(), + exclude: Vec::new(), + }; + assert!( + request_job_run(store, (&roles.context, None), &source) + .await + .expect("source run") + ); + let api = start_api_with_authentication( + store, + ApiAuthentication { + login: HumanLogin::Password("https://control.example.com".into()), + ..ApiAuthentication::default() + }, + ) + .await; + let client = reqwest::Client::new(); + let session = ( + roles.administrator.bearer.as_str(), + roles.administrator.csrf.as_str(), + ); + let derived = Uuid::new_v5(&source.run_id, b"orders"); + let mut job_run = job(roles.location_id, "orders"); + job_run["run_id"] = json!(derived); + let requests = [ + ("runs", job_run), + ( + "runs/adhoc", + json!({"run_id": derived, "location_id": roles.location_id, + "target": "development", "command": ["build", "--select", "orders"], + "reason": "Refresh orders", "priority": 0}), + ), + ( + "runs/rerun", + json!({"run_id": derived, "source_run_id": source.run_id, + "from_failure": false, "reason": "Retry orders", "priority": 0}), + ), + ]; + let mut refusals = Vec::new(); + for (path, body) in requests { + let url = format!("{}/api/control/{path}", api.url); + let response = person_send(&client, (&url, session.0, session.1), Some(body)).await; + let status = response.status().as_u16(); + let body: Value = response.json().await.expect("JSON refusal"); + refusals.push((path, status, body["error"].clone())); + } + let reason = json!(format!( + "run_id {derived} is a name-based UUID (version 5), which the server reserves for \ + the IDs it derives for group members and retries; send a random (version 4) or \ + time-ordered (version 7) UUID" + )); + assert_eq!( + refusals, + vec![ + ("runs", test_case.expected_refusal_status, reason.clone()), + ( + "runs/adhoc", + test_case.expected_refusal_status, + reason.clone() + ), + ("runs/rerun", test_case.expected_refusal_status, reason), + ], + "{}: every person run request refuses a name-based run ID", + test_case.description + ); + let mut random = job(roles.location_id, "orders"); + random["run_id"] = json!(Builder::from_random_bytes([7; 16]).into_uuid()); + let url = format!("{}/api/control/runs", api.url); + let response = person_send(&client, (&url, session.0, session.1), Some(random)).await; + assert_eq!( + ( + response.status().as_u16(), + response.json::().await.expect("JSON") + ), + (test_case.expected_random_status, json!({"applied": true})), + "{}: a random run ID is still accepted", + test_case.description + ); + stop_api(api).await; + close_test_store(fixture).await; + }); + } +} diff --git a/crates/sqlbuild-orchestration/tests/api/test_types.rs b/crates/sqlbuild-orchestration/tests/api/test_types.rs index abbef0d410..fd9e7a97d2 100644 --- a/crates/sqlbuild-orchestration/tests/api/test_types.rs +++ b/crates/sqlbuild-orchestration/tests/api/test_types.rs @@ -169,6 +169,13 @@ pub(crate) struct DenialTestCase { pub(crate) expected_status: u16, } +pub(crate) struct RunIdRefusalTestCase { + pub(crate) description: &'static str, + pub(crate) backend: Backend, + pub(crate) expected_refusal_status: u16, + pub(crate) expected_random_status: u16, +} + pub(crate) struct CustomRoleApiTestCase { pub(crate) description: &'static str, pub(crate) backend: Backend, diff --git a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/server.md b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/server.md index 7cc1efbdff..7ac6503de3 100644 --- a/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/server.md +++ b/src/sqlbuild/.agents/skills/sqlbuild/references/docs/cli/server.md @@ -368,12 +368,14 @@ command selects (`selection`), the nodes it writes (`write_set`), the options th command that can't run whatever your permissions, and `launchable` is `true` when the launch would be accepted. A dry run queues nothing and records no audit entry. -To launch, `POST /api/control/runs/adhoc` with the same fields plus a new `run_id` and a -`reason`. The response is `{"applied": true}`; sending the same `run_id` again returns -`{"applied": false}` and starts nothing. The server resolves selectors against the current -publication, and the worker runs exactly the resolved nodes, from the project it verified against -that publication; files that change on the location after that check are not used. The target is -always the one in the request. +To launch, `POST /api/control/runs/adhoc` with the same fields plus a `reason` and a new `run_id`, +a random (version 4) or time-ordered (version 7) UUID. The response is `{"applied": true}`; +sending the same `run_id` again returns `{"applied": false}` and starts nothing. Job runs and +reruns take a `run_id` the same way. The server refuses a name-based (version 5) `run_id` with a +400, because it derives those IDs for the runs it starts itself. The server resolves selectors +against the current publication, and the worker runs exactly the resolved nodes, from the project +it verified against that publication; files that change on the location after that check are not +used. The target is always the one in the request. | Verb | What the worker runs | | --- | --- | diff --git a/src/sqlbuild/executor/build/classes/build_scheduler.py b/src/sqlbuild/executor/build/classes/build_scheduler.py index 0231a20d37..5f55987d43 100644 --- a/src/sqlbuild/executor/build/classes/build_scheduler.py +++ b/src/sqlbuild/executor/build/classes/build_scheduler.py @@ -104,6 +104,7 @@ from sqlbuild.executor.python_nodes.types import PythonIdentityRecorder from sqlbuild.executor.run.classes.fingerprint_writes import FingerprintWrites from sqlbuild.executor.run.classes.runtime_watermark_resolver import RuntimeWatermarkResolver +from sqlbuild.executor.run.constants import RUN_AUDIT_FAILED_CODE from sqlbuild.executor.run.models import ( BatchWindow, BoundViewGuard, @@ -258,6 +259,7 @@ def __init__( self._load_results: list[LoadExecutionResult] = list(initial_state.initial_load_results) self._test_results: list[SqlTestExecutionResult] = [] self._source_audit_results: list[AuditExecutionResult] = [] + self._seed_audit_results: dict[CompiledObjectKey, tuple[AuditExecutionResult, ...]] = {} self._executed_source_audits: set[str] = set() self._failed_sources: set[str] = set() @@ -1004,6 +1006,8 @@ def _execute_seed_node(self, *, key: CompiledObjectKey, connection: Any) -> Seed query_change_tracking=self._query_change_tracking, schema_prepared=self._schema_prepared, ) + if result.status == ExecutionStatus.SUCCESS: + result = self._with_seed_audits(key=key, result=result, connection=connection) duration: int = int((time.monotonic() - start) * 1000) completed_result: SeedExecutionResult = dataclasses.replace(result, duration_ms=duration) log_debug_event( @@ -1346,12 +1350,11 @@ def _handle_completion( failed: bool = False if isinstance(result, SeedExecutionResult): + self._source_audit_results.extend(self._seed_audit_results.pop(key, ())) self._seed_results.append(result) if self._on_node_complete is not None: self._on_node_complete(result) failed = result.status == ExecutionStatus.FAILED - if result.status == ExecutionStatus.SUCCESS and self._run_seed_audits(key=key): - failed = True elif isinstance(result, SqlTestExecutionResult): self._test_results.append(result) @@ -1404,26 +1407,38 @@ def _handle_completion( self._stop = True self._mark_complete(key) - def _run_seed_audits(self, *, key: CompiledObjectKey) -> bool: - """Run a loaded seed's audits and return whether an error-severity audit failed.""" + def _with_seed_audits( + self, *, key: CompiledObjectKey, result: SeedExecutionResult, connection: Any + ) -> SeedExecutionResult: + """Run a loaded seed's audits within its attempt; an error-severity failure fails it.""" if not self._run_audits: - return False + return result audits: tuple[AuditPlanEntry, ...] = self._indexes.seed_audits_by_seed.get(key.name, ()) if not audits: - return False + return result results: tuple[AuditExecutionResult, ...] = run_end_audits( end_audits=audits, adapter=self._adapter, - connection=self._scheduler_connection, + connection=connection, model_locations=self._plan.model_locations, seed_locations=self._plan.seed_locations, source_map=self._plan.source_map, run_id=self._run_id, quality_scope="seed", ) - self._source_audit_results.extend(results) - return any(result.outcome == AuditOutcome.ERROR for result in results) + self._seed_audit_results[key] = results + if not any(audit.outcome == AuditOutcome.ERROR for audit in results): + return result + return dataclasses.replace( + result, + status=ExecutionStatus.FAILED, + error_code=RUN_AUDIT_FAILED_CODE, + error_message=( + f"audit for seed '{result.seed_name}' failed after loading " + "with severity level: error" + ), + ) def _mark_complete(self, key: CompiledObjectKey) -> None: """Mark a key complete and enqueue any downstream that become ready.""" diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py index ae15ae5ad1..4cb4545bfb 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/_test_types.py @@ -1042,6 +1042,14 @@ class AttachedAuditGateSummaryE2ETestCase: expected_summary: str +@dataclass(frozen=True) +class AttachedAuditGateAttemptE2ETestCase: + description: str + target_kind: str + order_code: str + expected_target_event: tuple[str, str | None] + + @dataclass(frozen=True) class AttachedAuditGateCycleE2ETestCase: description: str diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/helpers.py index fc8da19a0e..774bce1499 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/helpers.py @@ -639,6 +639,15 @@ def build_check_outcomes(stdout: str) -> dict[tuple[object, object], tuple[objec } +def integration_attempt_outcomes(records_path: Path) -> dict[object, tuple[object, object]]: + """Map each resource ID to its attempt's event type and error code from integration records.""" + + return { + record["resource_id"]: (record["event_type"], record.get("error_code")) + for record in map(json.loads, records_path.read_text(encoding="utf-8").splitlines()) + } + + def build_asset_names(stdout: str) -> tuple[str, ...]: """Return the asset names reported by build JSON output.""" diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_attached_audit_gates.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_attached_audit_gates.py index a147362453..3bdcee6635 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_attached_audit_gates.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_attached_audit_gates.py @@ -8,6 +8,7 @@ import pytest from tests.e2e.src.sqlbuild.cli.commands.main.build._test_types import ( + AttachedAuditGateAttemptE2ETestCase, AttachedAuditGateBuildE2ETestCase, AttachedAuditGateCycleE2ETestCase, AttachedAuditGateNoAuditsE2ETestCase, @@ -20,6 +21,7 @@ ATTACHED_AUDIT_GATE_DATABASE, build_asset_names, build_check_outcomes, + integration_attempt_outcomes, prepare_attached_audit_gate_project, prepare_audit_read_plan_project, prepare_nested_source_gate_project, @@ -182,11 +184,11 @@ def test_given_attached_audit_reading_another_resource_when_building_then_it_gat expected_summary="PASS=1 WARN=0 FAIL=1 INSUFFICIENT=0 SKIP=1 TOTAL=3", ), AttachedAuditGateSummaryE2ETestCase( - description="failing seed audit counts as a failure", + description="failing seed audit fails its seed", target_kind="seed", order_code="Z", expected_exit_code=1, - expected_summary="PASS=2 WARN=0 FAIL=1 INSUFFICIENT=0 SKIP=1 TOTAL=4", + expected_summary="PASS=1 WARN=0 FAIL=2 INSUFFICIENT=0 SKIP=1 TOTAL=4", ), ], ids=lambda case: case.description, @@ -208,6 +210,48 @@ def test_given_attached_audit_outcome_when_building_in_terminal_then_summary_cou assert test_case.expected_summary in output +@pytest.mark.parametrize( + "test_case", + [ + AttachedAuditGateAttemptE2ETestCase( + description="passing seed audit completes the seed's attempt", + target_kind="seed", + order_code="A", + expected_target_event=("resource_attempt_completed", None), + ), + AttachedAuditGateAttemptE2ETestCase( + description="failing seed audit fails the seed's attempt", + target_kind="seed", + order_code="Z", + expected_target_event=("resource_attempt_failed", "R006"), + ), + ], + ids=lambda case: case.description, +) +def test_given_attached_audit_outcome_when_building_then_the_target_attempt_records_it( + test_case: AttachedAuditGateAttemptE2ETestCase, + tmp_path: Path, +) -> None: + project_dir: Path = prepare_attached_audit_gate_project( + tmp_path=tmp_path, target_kind=test_case.target_kind, order_code=test_case.order_code + ) + records_path: Path = tmp_path / "integration_results.jsonl" + + result: subprocess.CompletedProcess[str] = run_sqb( + command=("--no-color", "build"), + project_dir=project_dir, + env={"SQLBUILD_INTEGRATION_RESULT_PATH": str(records_path)}, + ) + + assert ( + integration_attempt_outcomes(records_path).get("seed:order_codes") + == test_case.expected_target_event + ), ( + test_case.description, + result.stdout + result.stderr, + ) + + @pytest.mark.parametrize( "test_case", [ diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_audit_execution_errors.py b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_audit_execution_errors.py index d8f87abd44..f908457e93 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/build/test_audit_execution_errors.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/build/test_audit_execution_errors.py @@ -61,7 +61,7 @@ severity="error", concurrency=1, expected_attachment_kind="seed", - expected_asset_statuses={"order_codes": "success", "coded_orders": "skipped"}, + expected_asset_statuses={"order_codes": "failed", "coded_orders": "skipped"}, ), AuditExecutionErrorE2ETestCase( description="seed audit with error severity at concurrency 4", @@ -69,7 +69,7 @@ severity="error", concurrency=4, expected_attachment_kind="seed", - expected_asset_statuses={"order_codes": "success", "coded_orders": "skipped"}, + expected_asset_statuses={"order_codes": "failed", "coded_orders": "skipped"}, ), AuditExecutionErrorE2ETestCase( description="end audit with error severity at concurrency 1", @@ -133,13 +133,14 @@ def test_given_audit_query_raises_when_building_then_it_reports_a_failed_audit( ), ), AuditExecutionErrorTerminalE2ETestCase( - description="seed audit error is listed and counted as a failure", + description="seed audit error is listed and fails its seed", audit_kind="seed", expected_fragments=( "Failures:", "broken_check on order_codes (audit)", + "order_codes (seed)", "missing_lookup", - "PASS=1 WARN=0 FAIL=1 INSUFFICIENT=0 SKIP=1 TOTAL=3", + "PASS=0 WARN=0 FAIL=2 INSUFFICIENT=0 SKIP=1 TOTAL=3", ), ), AuditExecutionErrorTerminalE2ETestCase( diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/seed/test_seed_yaml_audits.py b/tests/e2e/src/sqlbuild/cli/commands/main/seed/test_seed_yaml_audits.py index 9eda44997d..19271f26cf 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/seed/test_seed_yaml_audits.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/seed/test_seed_yaml_audits.py @@ -49,7 +49,11 @@ audit_sql=_RELATION_AUDIT_SQL, seed_csv="waffle_type_id,price_cents\n1,450\n2,-1\n", expected_exit_code=1, - expected_output_fragments=("expression_is_true (waffle_types)", "FAIL=1"), + expected_output_fragments=( + "expression_is_true (waffle_types)", + "waffle_types (seed)", + "FAIL=2", + ), ), SeedYamlAuditE2ETestCase( description="model-only audit on a seed points at @relation", diff --git a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_worker.py b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_worker.py index ff3cfb341e..b228c4f5d0 100644 --- a/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_worker.py +++ b/tests/integration/src/sqlbuild/compiler/compiled_project/publication/test_worker.py @@ -263,6 +263,68 @@ def test_given_a_stale_source_when_a_worker_builds_then_blocked_nodes_finish_ski assert warehouse.sql(test_case.warehouse_query).fetchall() == test_case.expected_rows +@pytest.mark.parametrize( + "test_case", + [ + WorkerExecutionTestCase( + description="a failing seed audit fails its seed, and what reads it is skipped", + model_sql="SELECT 3 AS amount", + warehouse_query="SELECT count(*) FROM duckdb_tables() WHERE table_name='coded_orders'", + expected_status="partial", + expected_rows=[(0,)], + files={ + "seeds/order_codes.csv": "code\nA\nZ\n", + "seeds/order_codes.yml": Path( + "benchmarks/duckshop/edge_cases/orchestration/worker_failed_seed_audit.yml" + ).read_text(), + "models/coded_orders.sql": ( + "MODEL (description 'Coded orders.', materialized table);\n" + 'SELECT code FROM __seed("order_codes")\n' + ), + "orchestration/orders.py": ( + "from sqlbuild.orchestration import job, schedule\n" + 'annual_orders = job("annual_orders", ' + 'select="orders order_codes coded_orders", ' + 'description="Build orders.", ' + 'schedule=schedule(cron="0 0 * * *", timezone="UTC", ' + 'starts_at="2000-01-01T00:00:00Z"))\n' + ), + }, + ) + ], + ids=lambda case: case.description, +) +def test_given_a_failing_seed_audit_when_a_worker_builds_then_its_seed_fails_with_the_audit_error( + test_case: WorkerExecutionTestCase, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + with scheduled_worker(tmp_path, monkeypatch, test_case) as worker: + run: dict[str, Any] = wait_for_run( + worker.api, worker.process, worker.log, job_key="annual_orders" + ) + assert run["status"] == test_case.expected_status, worker.log.read_text() + with sqlite3.connect(worker.control_store) as control: + nodes: list[tuple[str, str, str | None, str | None]] = control.execute( + "SELECT n.node_key, n.status, n.error_code, " + "json_extract(e.payload, '$.change.change.error_message') " + "FROM sqb_run_nodes n LEFT JOIN recall_events e " + "ON e.storage_id=n.finished_storage_id " + "WHERE n.run_id=? AND n.kind='node' ORDER BY n.node_key", + (run["run_id"],), + ).fetchall() + assert nodes == [ + ("model:coded_orders", "skipped", None, None), + ("model:orders", "success", None, None), + ( + "seed:order_codes", + "failed", + "R006", + "audit for seed 'order_codes' failed after loading with severity level: error", + ), + ], worker.log.read_text() + with duckdb.connect(str(worker.project / "warehouse.duckdb"), read_only=True) as warehouse: + assert warehouse.sql(test_case.warehouse_query).fetchall() == test_case.expected_rows + + @pytest.mark.parametrize( "test_case", [ diff --git a/website/src/content/docs/docs/cli/server.mdx b/website/src/content/docs/docs/cli/server.mdx index 2037da023e..48db7bffd6 100644 --- a/website/src/content/docs/docs/cli/server.mdx +++ b/website/src/content/docs/docs/cli/server.mdx @@ -353,12 +353,14 @@ command selects (`selection`), the nodes it writes (`write_set`), the options th command that can't run whatever your permissions, and `launchable` is `true` when the launch would be accepted. A dry run queues nothing and records no audit entry. -To launch, `POST /api/control/runs/adhoc` with the same fields plus a new `run_id` and a -`reason`. The response is `{"applied": true}`; sending the same `run_id` again returns -`{"applied": false}` and starts nothing. The server resolves selectors against the current -publication, and the worker runs exactly the resolved nodes, from the project it verified against -that publication; files that change on the location after that check are not used. The target is -always the one in the request. +To launch, `POST /api/control/runs/adhoc` with the same fields plus a `reason` and a new `run_id`, +a random (version 4) or time-ordered (version 7) UUID. The response is `{"applied": true}`; +sending the same `run_id` again returns `{"applied": false}` and starts nothing. Job runs and +reruns take a `run_id` the same way. The server refuses a name-based (version 5) `run_id` with a +400, because it derives those IDs for the runs it starts itself. The server resolves selectors +against the current publication, and the worker runs exactly the resolved nodes, from the project +it verified against that publication; files that change on the location after that check are not +used. The target is always the one in the request. | Verb | What the worker runs | | --- | --- |