diff --git a/crates/sqlbuild-python/src/bindings/_helpers/event_exporting/dispatcher.rs b/crates/sqlbuild-python-output/src/event_exporting/_helpers/dispatcher.rs similarity index 55% rename from crates/sqlbuild-python/src/bindings/_helpers/event_exporting/dispatcher.rs rename to crates/sqlbuild-python-output/src/event_exporting/_helpers/dispatcher.rs index 39a80d892f..7d1020f38c 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/event_exporting/dispatcher.rs +++ b/crates/sqlbuild-python-output/src/event_exporting/_helpers/dispatcher.rs @@ -5,42 +5,31 @@ use std::sync::{Arc, Mutex, MutexGuard}; use std::time::{Duration, Instant}; use pyo3::exceptions::PyRuntimeError; -use pyo3::prelude::{ - Bound, Py, PyAny, PyAnyMethods, PyResult, Python, pyclass, pyfunction, pymethods, -}; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python, pyfunction}; use pyo3::types::{PyBool, PyDict, PyFloat, PyInt, PyString}; -use sqlbuild_core::command_output::constants::COMMAND_OUTPUT_RECORD_TYPE; use sqlbuild_core::event_export::constants::{ DEFAULT_EVENT_EXPORT_HEALTH_INTERVAL_SECONDS, DEFAULT_EVENT_EXPORT_INVOCATION_TIMEOUT_SECONDS, DEFAULT_EVENT_EXPORT_NOTIFICATION_QUEUE_CAPACITY, DEFAULT_EVENT_EXPORT_QUEUE_CAPACITY, - DEFAULT_EVENT_EXPORT_SHUTDOWN_TIMEOUT_SECONDS, INVOCATION_START_EVENT_TYPE, - INVOCATION_TERMINAL_EVENT_TYPES, OUTCOME_EVENT_TYPES, OUTCOME_RESERVE_FRACTION, + DEFAULT_EVENT_EXPORT_SHUTDOWN_TIMEOUT_SECONDS, }; -use sqlbuild_core::event_export::main::drop_lifecycle_attempts::drop_lifecycle_attempts; use sqlbuild_core::event_export::main::export_snapshot::export_snapshot; -use sqlbuild_core::event_export::main::queue_lifecycle_event::queue_lifecycle_event; -use sqlbuild_core::event_export::main::route_lifecycle_event::route_lifecycle_event; use sqlbuild_core::event_export::models::{ - EventQueue, ExportPolicy, ExportSeverity, ExportSnapshot, ExporterCounts, ExporterFilter, - QueuedEvent, SnapshotInputs, + EventQueue, ExportSeverity, ExportSnapshot, ExporterCounts, ExporterFilter, SnapshotInputs, }; use sqlbuild_host::entries::models::UserCodeHost; -use crate::bindings::_helpers::event_exporting::workers::{DispatcherWorker, Lane, WorkerRole}; +use crate::event_exporting::_helpers::workers::Lane; +use crate::event_exporting::_helpers::workers::WorkerRole; +use crate::event_exporting::models::DispatcherWorker; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::boundary::models::KeywordArguments; -use sqlbuild_python_base::event_exporting::constants::DISPATCHER_THREAD_NAME; use sqlbuild_python_base::event_exporting::constants::EXIT_JOIN_SECONDS; -use sqlbuild_python_base::event_exporting::constants::HEALTH_THREAD_NAME; use sqlbuild_python_base::event_exporting::constants::IDLE_WAIT_SECONDS; -use sqlbuild_python_base::event_exporting::constants::JOIN_GRACE_SECONDS; use sqlbuild_python_base::event_exporting::constants::NO_SECONDS; -use sqlbuild_python_base::event_exporting::constants::NOTIFIER_THREAD_NAME; use sqlbuild_python_base::event_exporting::constants::OUTPUT_SINK_THREAD_NAME; use sqlbuild_python_base::event_exporting::constants::POLL_SECONDS; use sqlbuild_python_base::event_exporting::constants::WORKER_ATTRIBUTE; -use sqlbuild_python_base::event_exporting::main::export_policy::export_policy; use sqlbuild_python_base::event_exporting::main::input_error::input_error; use sqlbuild_python_base::event_exporting::main::severity_from::severity_from; use sqlbuild_python_base::event_exporting::main::state_error::state_error; @@ -54,6 +43,7 @@ use sqlbuild_python_base::native_threads::main::signal_interpreter_exit::signal_ use sqlbuild_python_base::records::main::fields::field::field as native_field; /// One bound lifecycle-event sink. +#[derive(Debug)] pub(crate) struct Exporter { pub(crate) name: String, pub(crate) function: Py, @@ -61,13 +51,15 @@ pub(crate) struct Exporter { } /// One bound command-output sink and the streams it accepts. +#[derive(Debug)] pub(crate) struct OutputSink { - function: Py, - provider_arguments: Py, - streams: Vec, + pub(crate) function: Py, + pub(crate) provider_arguments: Py, + pub(crate) streams: Vec, } /// A queued event with the policy dimensions its failures report. +#[derive(Debug)] pub(crate) struct Queued { pub(crate) event: Py, pub(crate) kind: String, @@ -75,17 +67,18 @@ pub(crate) struct Queued { } /// Delivery state, read and written only while holding the dispatcher's Python condition. +#[derive(Debug)] pub(crate) struct State { pub(crate) bound: bool, pub(crate) exporters: Vec>, pub(crate) filters: Vec, pub(crate) counts: Vec, - output_sinks: Arc>, + pub(crate) output_sinks: Arc>, /// Calls queued to the single command-output sink thread, once it has started. - output_lane: Option>, + pub(crate) output_lane: Option>, pub(crate) queue: EventQueue, - sequence: u64, - accepting: bool, + pub(crate) sequence: u64, + pub(crate) accepting: bool, pub(crate) deadline: Option, /// When only outcome events remain deliverable, ahead of the deadline. pub(crate) reserve_at: Option, @@ -112,10 +105,10 @@ pub(crate) struct State { pub(crate) last_charged: Option, pub(crate) live_invocations: HashSet, pub(crate) next_invocation: u64, - dispatcher_running: bool, - shutdown_started: bool, - shutdown_complete: bool, - final_summary: Option>, + pub(crate) dispatcher_running: bool, + pub(crate) shutdown_started: bool, + pub(crate) shutdown_complete: bool, + pub(crate) final_summary: Option>, pub(crate) failures: VecDeque, pub(crate) notification_stopping: bool, pub(crate) health_stopping: bool, @@ -127,31 +120,21 @@ pub(crate) struct State { } /// What the dispatcher and its threads share. +#[derive(Debug)] pub(crate) struct Shared { - condition: Py, + pub(crate) condition: Py, /// The `SimpleQueue` every delivery thread reports `(token, error)` outcomes to. pub(crate) results: Py, /// Attempt tokens a delivery thread claimed (`True`) before running, or the dispatcher cancelled (`False`). pub(crate) claims: Py, - state: Mutex, + pub(crate) state: Mutex, pub(crate) invocation_timeout: Duration, - shutdown_timeout: Duration, + pub(crate) shutdown_timeout: Duration, pub(crate) health_interval: Duration, - notification_capacity: usize, + pub(crate) notification_capacity: usize, pub(crate) failure_callback: Option>, pub(crate) summary_callback: Option>, - notifications: bool, -} - -/// Deliver filtered lifecycle events off execution threads with bounded memory. -#[pyclass(module = "sqlbuild._native", frozen)] -pub(crate) struct EventExporterDispatcher { - shared: Arc, - /// Event kinds this command never exports, such as compile's phase operations. - withheld_kinds: Vec, - thread: Py, - notification_thread: Option>, - health_thread: Option>, + pub(crate) notifications: bool, } /// The dispatcher's Python condition, entered while alive and exited on drop. @@ -166,12 +149,12 @@ pub(crate) enum Delivery { } /// Validated dispatcher limits, in seconds and items. -struct Limits { - queue_capacity: usize, - shutdown_timeout: Duration, - invocation_timeout: Duration, - notification_capacity: usize, - health_interval: Duration, +pub(crate) struct Limits { + pub(crate) queue_capacity: usize, + pub(crate) shutdown_timeout: Duration, + pub(crate) invocation_timeout: Duration, + pub(crate) notification_capacity: usize, + pub(crate) health_interval: Duration, } impl Drop for Held<'_> { @@ -188,14 +171,14 @@ fn seconds(value: f64) -> Duration { Duration::try_from_secs_f64(value.max(NO_SECONDS)).unwrap_or(Duration::MAX) } -fn remaining_seconds(deadline: Instant) -> f64 { +pub(crate) fn remaining_seconds(deadline: Instant) -> f64 { deadline .saturating_duration_since(Instant::now()) .as_secs_f64() } /// End a retired command-output sink thread once it has run every call queued before. -fn retire_output_lane(py: Python<'_>, retired: Option>) -> PyResult<()> { +pub(crate) fn retire_output_lane(py: Python<'_>, retired: Option>) -> PyResult<()> { if let Some(calls) = retired { calls.bind(py).call_method1("put", (py.None(),))?; } @@ -203,7 +186,7 @@ fn retire_output_lane(py: Python<'_>, retired: Option>) -> PyResult<() } /// `thread.join(timeout=seconds)` when the thread is alive. -fn join(thread: &Bound<'_, PyAny>, seconds: f64) -> PyResult<()> { +pub(crate) fn join(thread: &Bound<'_, PyAny>, seconds: f64) -> PyResult<()> { if thread.call_method0("is_alive")?.is_truthy()? { call_with( &native_field(thread, "join")?, @@ -214,7 +197,7 @@ fn join(thread: &Bound<'_, PyAny>, seconds: f64) -> PyResult<()> { } /// Bound exporters read from `BoundEventExporter` values or objects with the same attributes. -fn read_exporters( +pub(crate) fn read_exporters( exporters: &Bound<'_, PyAny>, ) -> PyResult<(Vec>, Vec)> { let mut bound: Vec> = Vec::new(); @@ -247,7 +230,7 @@ fn read_exporters( Ok((bound, filters)) } -fn read_output_sinks(sinks: &Bound<'_, PyAny>) -> PyResult> { +pub(crate) fn read_output_sinks(sinks: &Bound<'_, PyAny>) -> PyResult> { let mut bound: Vec = Vec::new(); for sink in sinks.try_iter()? { let sink: Bound<'_, PyAny> = sink?; @@ -264,7 +247,7 @@ fn read_output_sinks(sinks: &Bound<'_, PyAny>) -> PyResult> { } /// `health_interval_seconds` when it is a positive, finite non-boolean number. -fn health_interval(value: &Bound<'_, PyAny>) -> PyResult { +pub(crate) fn health_interval(value: &Bound<'_, PyAny>) -> PyResult { let py: Python<'_> = value.py(); if value.is_none() { return Ok(DEFAULT_EVENT_EXPORT_HEALTH_INTERVAL_SECONDS); @@ -288,7 +271,7 @@ fn health_interval(value: &Bound<'_, PyAny>) -> PyResult { } /// The constructor's limits, validated in the Python dispatcher's order. -fn limits(options: &KeywordArguments<'_>) -> PyResult { +pub(crate) fn limits(options: &KeywordArguments<'_>) -> PyResult { let integer = |name: &str, default: i64| -> PyResult { let value = options.value(name)?; if value.is_none() { @@ -425,7 +408,7 @@ impl Shared { .is_some_and(|deadline| Instant::now() >= deadline) } - fn snapshot(state: &State, final_snapshot: bool) -> ExportSnapshot { + pub(crate) fn snapshot(state: &State, final_snapshot: bool) -> ExportSnapshot { let exporters: Vec<(String, ExporterCounts)> = state .exporters .iter() @@ -456,7 +439,7 @@ impl Shared { } /// Forget one finished invocation and wake a caller waiting for delivery to become idle. - fn invocation_finished(&self, py: Python<'_>, invocation: u64) -> PyResult<()> { + pub(crate) fn invocation_finished(&self, py: Python<'_>, invocation: u64) -> PyResult<()> { let retired: Option> = { let _held = self.hold(py)?; let mut state = self.locked()?; @@ -481,12 +464,12 @@ impl Shared { } /// Whether the dispatcher thread and every sink call have finished. - fn idle(state: &State) -> bool { + pub(crate) fn idle(state: &State) -> bool { state.live_invocations.is_empty() && !state.dispatcher_running } /// Wait until delivery and every sink call are idle, the idle wait ends or interpreter exit begins. - fn await_idle(&self, py: Python<'_>) -> PyResult<()> { + pub(crate) fn await_idle(&self, py: Python<'_>) -> PyResult<()> { let deadline: Instant = Instant::now() + self.invocation_timeout.min(seconds(IDLE_WAIT_SECONDS)); let threads: Vec> = { @@ -518,7 +501,7 @@ impl Shared { } /// Register a running invocation, returning its identity. - fn begin_invocation(&self, py: Python<'_>) -> PyResult { + pub(crate) fn begin_invocation(&self, py: Python<'_>) -> PyResult { let _held = self.hold(py)?; let mut state = self.locked()?; state.next_invocation += 1; @@ -528,7 +511,7 @@ impl Shared { } /// Stop acceptance, delivery, health and notifications so every long-lived thread returns. - fn stop_for_exit(&self, py: Python<'_>) -> PyResult<()> { + pub(crate) fn stop_for_exit(&self, py: Python<'_>) -> PyResult<()> { let _held = self.hold(py)?; let mut state = self.locked()?; state.accepting = false; @@ -541,7 +524,7 @@ impl Shared { } /// The queue feeding this dispatcher's single command-output sink thread, started on first use. - fn output_lane<'py>(&self, py: Python<'py>) -> PyResult> { + pub(crate) fn output_lane<'py>(&self, py: Python<'py>) -> PyResult> { let _held = self.hold(py)?; let existing: Option> = self .locked()? @@ -576,7 +559,7 @@ impl Shared { } /// Call one command-output sink on the sink thread and wait until it returns or exit begins. - fn call_output_sink( + pub(crate) fn call_output_sink( &self, py: Python<'_>, sink: &OutputSink, @@ -613,7 +596,7 @@ impl Shared { } /// The final summary once another caller's shutdown completes. - fn await_shutdown(&self, py: Python<'_>) -> PyResult> { + pub(crate) fn await_shutdown(&self, py: Python<'_>) -> PyResult> { let _held = self.hold(py)?; loop { let state = self.locked()?; @@ -633,7 +616,7 @@ impl Shared { } /// A started daemon thread running `worker`, tagged so the exit hook can stop it. -fn start_worker<'py>( +pub(crate) fn start_worker<'py>( py: Python<'py>, shared: &Arc, (role, name): (WorkerRole, &str), @@ -652,372 +635,3 @@ fn start_worker<'py>( thread.setattr(WORKER_ATTRIBUTE, worker)?; Ok(thread) } - -#[pymethods] -impl EventExporterDispatcher { - #[new] - #[pyo3(signature = (**options))] - pub(crate) fn new(py: Python<'_>, options: Option>) -> PyResult { - let options: KeywordArguments<'_> = KeywordArguments::read( - py, - "EventExporterDispatcher", - options, - ( - &[], - &[ - "exporters", - "queue_capacity", - "shutdown_timeout_seconds", - "invocation_timeout_seconds", - "notification_queue_capacity", - "health_interval_seconds", - "failure_callback", - "summary_callback", - "withheld_kinds", - ], - ), - )?; - let withheld: Bound<'_, PyAny> = options.value("withheld_kinds")?; - let withheld_kinds: Vec = if withheld.is_none() { - Vec::new() - } else { - withheld.extract()? - }; - let limits: Limits = limits(&options)?; - let exporters: Bound<'_, PyAny> = options.value("exporters")?; - let (bound_exporters, filters) = if exporters.is_none() { - (Vec::new(), Vec::new()) - } else { - read_exporters(&exporters)? - }; - let callback = |name: &str| -> PyResult>> { - let value: Bound<'_, PyAny> = options.value(name)?; - Ok((!value.is_none()).then(|| value.unbind())) - }; - let failure_callback: Option> = callback("failure_callback")?; - let summary_callback: Option> = callback("summary_callback")?; - let notifications: bool = failure_callback.is_some() || summary_callback.is_some(); - let has_summary: bool = summary_callback.is_some(); - let shared = Arc::new(Shared { - condition: imported(py, "threading", "Condition")?.call0()?.unbind(), - results: imported(py, "queue", "SimpleQueue")?.call0()?.unbind(), - claims: PyDict::new(py).unbind(), - state: Mutex::new(State { - bound: !exporters.is_none(), - counts: vec![ExporterCounts::default(); bound_exporters.len()], - exporters: bound_exporters, - filters, - output_sinks: Arc::new(Vec::new()), - output_lane: None, - queue: EventQueue::new(limits.queue_capacity), - sequence: 0, - accepting: true, - deadline: None, - reserve_at: None, - reserved: false, - lanes: Vec::new(), - tokens: HashMap::new(), - next_token: 0, - retired: Vec::new(), - abandoned_calls: false, - stopping: false, - force_stop: false, - blocked: HashSet::new(), - stalled: HashMap::new(), - held: Vec::new(), - newly_blocked: Vec::new(), - last_charged: None, - live_invocations: HashSet::new(), - next_invocation: 0, - dispatcher_running: true, - shutdown_started: false, - shutdown_complete: false, - final_summary: None, - failures: VecDeque::new(), - notification_stopping: false, - health_stopping: false, - pending_periodic: None, - periodic_inflight: None, - last_published_periodic: [ - 0, - 0, - 0, - 0, - 0, - 0, - u64::try_from(limits.queue_capacity).unwrap_or(u64::MAX), - ], - final_notification: None, - final_delivered: false, - }), - invocation_timeout: limits.invocation_timeout, - shutdown_timeout: limits.shutdown_timeout, - health_interval: limits.health_interval, - notification_capacity: limits.notification_capacity, - failure_callback, - summary_callback, - notifications, - }); - let notification_thread: Option> = notifications - .then(|| start_worker(py, &shared, (WorkerRole::Notify, NOTIFIER_THREAD_NAME))) - .transpose()?; - let health_thread: Option> = has_summary - .then(|| start_worker(py, &shared, (WorkerRole::Health, HEALTH_THREAD_NAME))) - .transpose()?; - for thread in notification_thread.iter().chain(&health_thread) { - thread.call_method0("start")?; - } - let thread = start_worker(py, &shared, (WorkerRole::Dispatch, DISPATCHER_THREAD_NAME))?; - thread.call_method0("start")?; - Ok(Self { - shared, - withheld_kinds, - thread: thread.unbind(), - notification_thread: notification_thread.map(Bound::unbind), - health_thread: health_thread.map(Bound::unbind), - }) - } - - #[getter] - pub(crate) fn thread(&self, py: Python<'_>) -> Py { - self.thread.clone_ref(py) - } - - #[getter] - pub(crate) fn notification_thread(&self, py: Python<'_>) -> Option> { - self.notification_thread - .as_ref() - .map(|thread| thread.clone_ref(py)) - } - - /// Filter and enqueue eligible exporter attempts without waiting. - fn enqueue(&self, py: Python<'_>, event: &Bound<'_, PyAny>) -> PyResult<()> { - let event_type: String = native_field(event, "event_type")?.extract()?; - let policy: ExportPolicy = export_policy(py, &event_type)?; - if self.withheld_kinds.contains(&policy.kind) { - return Ok(()); - } - let invocation_id: Option = match native_field(event, "invocation_id") { - Ok(value) => value.extract()?, - Err(_) => None, - }; - let shared = &self.shared; - let _held = shared.hold(py)?; - let mut state = shared.locked()?; - if !state.bound { - drop(state); - return Err(state_error( - py, - "event exporters must be bound before enqueue", - )); - } - let counts: Vec = std::mem::take(&mut state.counts); - let (counts, eligible) = route_lifecycle_event(&policy, &state.filters, counts); - state.counts = counts; - if eligible.is_empty() { - return Ok(()); - } - if !state.accepting { - state.counts = drop_lifecycle_attempts(std::mem::take(&mut state.counts), &eligible); - return Ok(()); - } - let sequence: u64 = state.sequence; - state.sequence += 1; - let capacity: usize = state.queue.capacity; - let mut outcome = queue_lifecycle_event( - std::mem::replace(&mut state.queue, EventQueue::new(capacity)), - QueuedEvent { - sequence, - item: Queued { - event: event.clone().unbind(), - kind: policy.kind, - severity: policy.severity, - }, - priority: policy.priority, - invocation_terminal: INVOCATION_TERMINAL_EVENT_TYPES.contains(&event_type.as_str()), - outcome: OUTCOME_EVENT_TYPES.contains(&event_type.as_str()), - invocation_id, - invocation_start: event_type == INVOCATION_START_EVENT_TYPE, - eligible_exporters: eligible, - }, - ); - state.queue = std::mem::replace(&mut outcome.queue, EventQueue::new(capacity)); - for lost in outcome.displaced.iter().chain(&outcome.rejected) { - state.counts = drop_lifecycle_attempts( - std::mem::take(&mut state.counts), - &lost.eligible_exporters, - ); - } - drop(state); - shared.notify_all(py)?; - drop(outcome); - Ok(()) - } - - /// Bind validated exporters once before lifecycle publication starts. - fn bind(&self, py: Python<'_>, exporters: &Bound<'_, PyAny>) -> PyResult<()> { - let (exporters, filters) = read_exporters(exporters)?; - let shared = &self.shared; - let _held = shared.hold(py)?; - let mut state = shared.locked()?; - if state.bound { - drop(state); - return Err(state_error(py, "event exporters are already bound")); - } - state.counts = vec![ExporterCounts::default(); exporters.len()]; - state.exporters = exporters; - state.filters = filters; - state.bound = true; - drop(state); - shared.notify_all(py) - } - - /// Bind explicit command-output sinks before capture starts. - fn bind_command_output_sinks(&self, py: Python<'_>, sinks: &Bound<'_, PyAny>) -> PyResult<()> { - let sinks: Arc> = Arc::new(read_output_sinks(sinks)?); - let previous = { - let _held = self.shared.hold(py)?; - std::mem::replace(&mut self.shared.locked()?.output_sinks, sinks) - }; - drop(previous); - Ok(()) - } - - /// Deliver output records through the already-bound destination providers. - fn export_output(&self, py: Python<'_>, records: &Bound<'_, PyAny>) -> PyResult<()> { - let shared = &self.shared; - let invocation: u64 = shared.begin_invocation(py)?; - let sinks: Arc> = { - let _held = shared.hold(py)?; - Arc::clone(&shared.locked()?.output_sinks) - }; - let mut failed: bool = false; - let delivered = (|| -> PyResult { - for record in records.try_iter()? { - let record: Bound<'_, PyAny> = record?; - let is_output: bool = - native_field(&record, "record_type")?.eq(COMMAND_OUTPUT_RECORD_TYPE)?; - let stream: Option = if is_output { - Some(native_field(&record, "stream")?.extract()?) - } else { - None - }; - for sink in sinks.iter() { - if stream - .as_ref() - .is_some_and(|stream| !sink.streams.contains(stream)) - { - continue; - } - match shared.call_output_sink(py, sink, &record)? { - Delivery::Delivered(sink_failed) => failed |= sink_failed, - Delivery::Abandoned => return Ok(Delivery::Abandoned), - } - } - } - Ok(Delivery::Delivered(failed)) - })(); - if let Ok(Delivery::Abandoned) = delivered { - return Err(state_error( - py, - "output export stopped because the interpreter is exiting", - )); - } - shared.invocation_finished(py, invocation)?; - delivered?; - if failed { - return Err(state_error(py, "one or more output export attempts failed")); - } - Ok(()) - } - - /// Stop acceptance and drain until the fixed shutdown deadline. - fn shutdown(&self, py: Python<'_>) -> PyResult> { - let shared = &self.shared; - let (owns, deadline) = { - let _held = shared.hold(py)?; - let mut state = shared.locked()?; - let owns: bool = !state.shutdown_started; - if owns { - state.shutdown_started = true; - state.accepting = false; - let started: Instant = Instant::now(); - state.deadline = Some(started + shared.shutdown_timeout); - state.reserve_at = Some( - started - + shared - .shutdown_timeout - .mul_f64(1.0 - OUTCOME_RESERVE_FRACTION), - ); - state.stopping = true; - state.health_stopping = true; - } - let deadline: Instant = state.deadline.unwrap_or_else(Instant::now); - drop(state); - shared.notify_all(py)?; - (owns, deadline) - }; - if !owns { - return shared.await_shutdown(py); - } - let thread: &Bound<'_, PyAny> = self.thread.bind(py); - call_with( - &native_field(thread, "join")?, - &[( - "timeout", - PyFloat::new(py, remaining_seconds(deadline) + JOIN_GRACE_SECONDS).into_any(), - )], - )?; - if thread.call_method0("is_alive")?.is_truthy()? { - { - let _held = shared.hold(py)?; - shared.locked()?.force_stop = true; - shared.notify_all(py)?; - } - thread.call_method0("join")?; - } - let snapshot: ExportSnapshot = { - let _held = shared.hold(py)?; - Shared::snapshot(&*shared.locked()?, true) - }; - let summary: Py = EventExportSummary::object(py, snapshot)?; - { - let _held = shared.hold(py)?; - let mut state = shared.locked()?; - state.final_summary = Some(summary.clone_ref(py)); - state.shutdown_complete = true; - if shared.summary_callback.is_some() - && state.final_notification.is_none() - && !state.final_delivered - { - state.final_notification = Some(summary.clone_ref(py)); - } - state.notification_stopping = true; - let retired: Option> = if state.live_invocations.is_empty() { - state.output_lane.take() - } else { - None - }; - drop(state); - shared.notify_all(py)?; - retire_output_lane(py, retired)?; - } - if let Some(health) = &self.health_thread { - join(health.bind(py), EXIT_JOIN_SECONDS)?; - } - Ok(summary) - } - - /// Run `finalizer` on the calling thread, never a daemon delivery thread, once delivery is idle or the idle wait ends. - fn finalize_when_idle(&self, py: Python<'_>, finalizer: Py) -> PyResult<()> { - self.shared.await_idle(py)?; - drop(finalizer.bind(py).call0()); - Ok(()) - } - - /// Return a thread-safe immutable point-in-time snapshot. - fn summary(&self, py: Python<'_>) -> PyResult> { - let snapshot: ExportSnapshot = self.shared.current_snapshot(py)?; - EventExportSummary::object(py, snapshot) - } -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/event_exporting/getters.rs b/crates/sqlbuild-python-output/src/event_exporting/_helpers/getters.rs similarity index 80% rename from crates/sqlbuild-python/src/bindings/_helpers/event_exporting/getters.rs rename to crates/sqlbuild-python-output/src/event_exporting/_helpers/getters.rs index ee7ad4da9a..1248fe68ec 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/event_exporting/getters.rs +++ b/crates/sqlbuild-python-output/src/event_exporting/_helpers/getters.rs @@ -3,49 +3,15 @@ use pyo3::IntoPyObjectExt; use pyo3::prelude::{Bound, PyAny, PyResult, Python}; -use crate::bindings::_helpers::event_exporting::dispatcher::EventExporterDispatcher; +use crate::event_exporting::models::EventExporterDispatcher; use sqlbuild_python_base::event_exporting::models::BoundEventExporter; use sqlbuild_python_base::event_exporting::models::EventExportSummary; use sqlbuild_python_base::event_exporting::models::EventExporterAccounting; use sqlbuild_python_base::event_exporting::models::EventExporterCommandScope; use sqlbuild_python_base::event_exporting::models::EventExporterCounts; use sqlbuild_python_base::event_exporting::models::EventExporterFailure; -use sqlbuild_python_base::records::main::registry::class_of::class_of; -use sqlbuild_python_base::records::models::GetterEntry; -/// The getter readers of this module's native pyclasses. -pub(crate) const GETTERS: &[GetterEntry] = &[ - GetterEntry { - class: class_of::, - read: read_event_exporter_dispatcher, - }, - GetterEntry { - class: class_of::, - read: read_bound_event_exporter, - }, - GetterEntry { - class: class_of::, - read: read_event_export_summary, - }, - GetterEntry { - class: class_of::, - read: read_event_exporter_accounting, - }, - GetterEntry { - class: class_of::, - read: read_event_exporter_counts, - }, - GetterEntry { - class: class_of::, - read: read_event_exporter_failure, - }, - GetterEntry { - class: class_of::, - read: read_event_exporter_command_scope, - }, -]; - -fn read_event_exporter_dispatcher<'py>( +pub(crate) fn read_event_exporter_dispatcher<'py>( value: &Bound<'py, PyAny>, name: &str, ) -> Option>> { @@ -63,7 +29,7 @@ fn read_event_exporter_dispatcher<'py>( }) } -fn read_bound_event_exporter<'py>( +pub(crate) fn read_bound_event_exporter<'py>( value: &Bound<'py, PyAny>, name: &str, ) -> Option>> { @@ -88,7 +54,7 @@ fn read_bound_event_exporter<'py>( }) } -fn read_event_export_summary<'py>( +pub(crate) fn read_event_export_summary<'py>( value: &Bound<'py, PyAny>, name: &str, ) -> Option>> { @@ -116,7 +82,7 @@ fn read_event_export_summary<'py>( }) } -fn read_event_exporter_accounting<'py>( +pub(crate) fn read_event_exporter_accounting<'py>( value: &Bound<'py, PyAny>, name: &str, ) -> Option>> { @@ -134,7 +100,7 @@ fn read_event_exporter_accounting<'py>( }) } -fn read_event_exporter_counts<'py>( +pub(crate) fn read_event_exporter_counts<'py>( value: &Bound<'py, PyAny>, name: &str, ) -> Option>> { @@ -155,7 +121,7 @@ fn read_event_exporter_counts<'py>( }) } -fn read_event_exporter_failure<'py>( +pub(crate) fn read_event_exporter_failure<'py>( value: &Bound<'py, PyAny>, name: &str, ) -> Option>> { @@ -177,7 +143,7 @@ fn read_event_exporter_failure<'py>( }) } -fn read_event_exporter_command_scope<'py>( +pub(crate) fn read_event_exporter_command_scope<'py>( value: &Bound<'py, PyAny>, name: &str, ) -> Option>> { diff --git a/crates/sqlbuild-python-output/src/event_exporting/_helpers/mod.rs b/crates/sqlbuild-python-output/src/event_exporting/_helpers/mod.rs new file mode 100644 index 0000000000..94c999df39 --- /dev/null +++ b/crates/sqlbuild-python-output/src/event_exporting/_helpers/mod.rs @@ -0,0 +1,4 @@ +pub(crate) mod dispatcher; +pub(crate) mod getters; +pub(crate) mod policy; +pub(crate) mod workers; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/event_exporting/policy.rs b/crates/sqlbuild-python-output/src/event_exporting/_helpers/policy.rs similarity index 77% rename from crates/sqlbuild-python/src/bindings/_helpers/event_exporting/policy.rs rename to crates/sqlbuild-python-output/src/event_exporting/_helpers/policy.rs index e5b4153519..17d9066c29 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/event_exporting/policy.rs +++ b/crates/sqlbuild-python-output/src/event_exporting/_helpers/policy.rs @@ -1,8 +1,6 @@ //! Lifecycle export policy, severity comparison and narrowing for Python callers. -use pyo3::prelude::{ - Bound, PyAny, PyModule, PyModuleMethods, PyResult, Python, pyfunction, wrap_pyfunction, -}; +use pyo3::prelude::{Bound, PyAny, PyResult, Python, pyfunction}; use pyo3::types::{PyDict, PyDictMethods}; use sqlbuild_core::event_export::main::severity_at_least::severity_at_least; use sqlbuild_core::event_export::main::stricter_severity::stricter_severity; @@ -16,7 +14,7 @@ use sqlbuild_python_base::event_exporting::main::severity_object::severity_objec /// `{event_type: (kind, severity, priority)}` for every catalogued lifecycle event, in catalogue order. #[pyfunction] -fn lifecycle_export_policy_catalog(py: Python<'_>) -> PyResult> { +pub(crate) fn lifecycle_export_policy_catalog(py: Python<'_>) -> PyResult> { let policies: Bound<'_, PyDict> = PyDict::new(py); for definition in catalog(CURRENT_LIFECYCLE_EVENT_SCHEMA_VERSION).unwrap_or_default() { let event_type: &str = definition.event_type; @@ -36,7 +34,7 @@ fn lifecycle_export_policy_catalog(py: Python<'_>) -> PyResult /// Whether a catalogued severity meets a configured minimum. #[pyfunction] #[pyo3(signature = (*, severity, minimum))] -fn event_export_severity_at_least( +pub(crate) fn event_export_severity_at_least( severity: &Bound<'_, PyAny>, minimum: &Bound<'_, PyAny>, ) -> PyResult { @@ -49,16 +47,10 @@ fn event_export_severity_at_least( /// The stricter of a required and an optional minimum severity, as `EventExportSeverity`. #[pyfunction] #[pyo3(signature = (*, first, second))] -fn stricter_event_export_severity<'py>( +pub(crate) fn stricter_event_export_severity<'py>( first: &Bound<'py, PyAny>, second: Option<&Bound<'py, PyAny>>, ) -> PyResult> { let second: Option = second.map(severity_from).transpose()?; severity_object(first.py(), stricter_severity(severity_from(first)?, second)) } - -pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add_function(wrap_pyfunction!(lifecycle_export_policy_catalog, module)?)?; - module.add_function(wrap_pyfunction!(event_export_severity_at_least, module)?)?; - module.add_function(wrap_pyfunction!(stricter_event_export_severity, module)?) -} diff --git a/crates/sqlbuild-python/src/bindings/_helpers/event_exporting/workers.rs b/crates/sqlbuild-python-output/src/event_exporting/_helpers/workers.rs similarity index 62% rename from crates/sqlbuild-python/src/bindings/_helpers/event_exporting/workers.rs rename to crates/sqlbuild-python-output/src/event_exporting/_helpers/workers.rs index f2c3a7d1e9..8d78b78057 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/event_exporting/workers.rs +++ b/crates/sqlbuild-python-output/src/event_exporting/_helpers/workers.rs @@ -4,28 +4,29 @@ use std::collections::VecDeque; use std::sync::Arc; use std::time::{Duration, Instant}; -use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyErr, PyResult, Python, pyclass, pymethods}; +use pyo3::prelude::{Bound, Py, PyAny, PyAnyMethods, PyResult, Python}; use pyo3::types::{PyBool, PyDict, PyDictMethods, PyString}; use sqlbuild_core::event_export::constants::TIMEOUT_ERROR_TYPE; use sqlbuild_core::event_export::main::drop_lifecycle_attempts::drop_lifecycle_attempts; use sqlbuild_core::event_export::main::retain_outcome_events::retain_outcome_events; use sqlbuild_core::event_export::main::take_lifecycle_event::take_lifecycle_event; -use sqlbuild_core::event_export::models::{EventQueue, ExportSnapshot, QueuedEvent}; +use sqlbuild_core::event_export::models::{EventQueue, ExportSnapshot}; use sqlbuild_host::entries::models::UserCodeHost; -use crate::bindings::_helpers::event_exporting::dispatcher::{Exporter, Queued, Shared, State}; +use crate::event_exporting::_helpers::dispatcher::Exporter; +use crate::event_exporting::_helpers::dispatcher::Queued; +use crate::event_exporting::_helpers::dispatcher::Shared; +use crate::event_exporting::_helpers::dispatcher::State; +use crate::recall::main::write_stderr::write_stderr; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::event_exporting::constants::CHARGE_LIMIT_SECONDS; use sqlbuild_python_base::event_exporting::constants::INVOCATION_THREAD_PREFIX; use sqlbuild_python_base::event_exporting::constants::LANE_WINDOW; -use sqlbuild_python_base::event_exporting::constants::NO_SECONDS; -use sqlbuild_python_base::event_exporting::constants::POLL_SECONDS; use sqlbuild_python_base::event_exporting::main::error_type_of::error_type_of; use sqlbuild_python_base::event_exporting::models::EventExportSummary; use sqlbuild_python_base::event_exporting::models::EventExporterFailure; use sqlbuild_python_base::native_host::main::host_entry::host_entry; -use sqlbuild_python_output::recall::main::write_stderr::write_stderr; /// Which long-lived dispatcher thread a worker runs. #[derive(Clone, Copy)] @@ -35,17 +36,11 @@ pub(crate) enum WorkerRole { Health, } -/// A long-lived dispatcher thread's target. -#[pyclass(module = "sqlbuild._native", frozen)] -pub(crate) struct DispatcherWorker { - shared: Arc, - role: WorkerRole, -} - /// Outcome events held for blocked exporters, each with its exporter index. type HeldOutcomes = Vec<(usize, Queued)>; /// One attempt handed to a lane and not yet reported. +#[derive(Debug)] pub(crate) struct Attempt { token: u64, queued: Queued, @@ -53,6 +48,7 @@ pub(crate) struct Attempt { } /// One exporter's long-lived delivery thread, its call queue and the attempts it still owes. +#[derive(Debug)] pub(crate) struct Lane { calls: Py, thread: Py, @@ -64,12 +60,12 @@ pub(crate) struct Lane { /// What one dispatch round reads from the dispatcher besides its state. #[derive(Clone, Copy)] pub(crate) struct Round<'a> { - shared: &'a Shared, - diagnostics: Option, + pub(crate) shared: &'a Shared, + pub(crate) diagnostics: Option, } /// The next thing the notification thread delivers. -enum Notification { +pub(crate) enum Notification { Periodic(ExportSnapshot), Final(Py), Failure(EventExporterFailure), @@ -123,7 +119,7 @@ impl Lane { impl State { /// Account reported attempts against their lanes, returning whether a diagnostic was queued. - fn record_reports( + pub(crate) fn record_reports( &mut self, py: Python<'_>, reports: Vec<(u64, Option)>, @@ -213,7 +209,7 @@ impl State { } /// Retire `index`'s lane: claimed calls fail as timed out and block it until they return. - fn abandon_lane( + pub(crate) fn abandon_lane( &mut self, py: Python<'_>, (index, timed_out): (usize, bool), @@ -259,7 +255,7 @@ impl State { } /// Abandon lanes whose started call outlived `timeout`, charging each round at most `CHARGE_LIMIT_SECONDS`. - fn expire_lanes( + pub(crate) fn expire_lanes( &mut self, py: Python<'_>, timeout: Duration, @@ -293,7 +289,11 @@ impl State { } /// Drop every queued and lane-waiting event that is not an outcome, counting each attempt. - fn reserve_for_outcomes(&mut self, py: Python<'_>, claims: &Py) -> PyResult<()> { + pub(crate) fn reserve_for_outcomes( + &mut self, + py: Python<'_>, + claims: &Py, + ) -> PyResult<()> { let capacity: usize = self.queue.capacity; let (queue, dropped) = retain_outcome_events(std::mem::replace( &mut self.queue, @@ -323,7 +323,7 @@ impl State { } /// Move queued events onto exporter lanes while every eligible lane has room. - fn feed(&mut self, py: Python<'_>, round: Round<'_>) -> PyResult { + pub(crate) fn feed(&mut self, py: Python<'_>, round: Round<'_>) -> PyResult { let mut notify: bool = false; while !self.queue.items.is_empty() { let capacity: usize = self.queue.capacity; @@ -422,7 +422,7 @@ impl State { } /// Whether any lane still owes a reported attempt. - fn busy(&self) -> bool { + pub(crate) fn busy(&self) -> bool { self.lanes .iter() .flatten() @@ -430,7 +430,7 @@ impl State { } /// End every lane once it has run what was queued before, keeping its thread to join. - fn retire_lanes(&mut self, py: Python<'_>) -> PyResult<()> { + pub(crate) fn retire_lanes(&mut self, py: Python<'_>) -> PyResult<()> { let lanes: Vec = self.lanes.drain(..).flatten().collect(); for lane in lanes { lane.retire(py)?; @@ -454,7 +454,7 @@ fn cancel(claims: &Bound<'_, PyDict>, token: u64) -> PyResult<()> { } /// Warn once per stall that a sink call outlived its timeout; best-effort, ignoring write errors. -fn warn_blocked(py: Python<'_>, name: &str, timeout: Duration) { +pub(crate) fn warn_blocked(py: Python<'_>, name: &str, timeout: Duration) { drop(write_stderr( py, &format!( @@ -464,230 +464,3 @@ fn warn_blocked(py: Python<'_>, name: &str, timeout: Duration) { ), )); } - -impl DispatcherWorker { - pub(crate) fn new(shared: Arc, role: WorkerRole) -> Self { - Self { shared, role } - } - - /// The state this worker's dispatcher shares. - pub(crate) fn shared(&self) -> &Shared { - &self.shared - } - - /// Wait until bound, returning `false` when stopped first. - fn await_bound(&self, py: Python<'_>) -> PyResult { - let shared = &self.shared; - let _held = shared.hold(py)?; - loop { - let state = shared.locked()?; - if state.bound { - return Ok(true); - } - if state.stopping || state.force_stop { - return Ok(false); - } - drop(state); - shared.wait(py, POLL_SECONDS)?; - } - } - - fn run_dispatch(&self, py: Python<'_>) -> PyResult<()> { - let result = (|| -> PyResult<()> { - if !self.await_bound(py)? { - return Ok(()); - } - while self.dispatch_round(py)? {} - Ok(()) - })(); - let retired: PyResult<()> = (|| { - let _held = self.shared.hold(py)?; - self.shared.locked()?.retire_lanes(py) - })(); - self.shared.dispatcher_finished(py)?; - retired?; - result - } - - /// Record reported outcomes, expire, trim and feed lanes, then wait; `false` once delivery stops. - fn dispatch_round(&self, py: Python<'_>) -> PyResult { - let shared = &self.shared; - let round = Round { - shared, - diagnostics: shared.diagnostics(), - }; - let _held = shared.hold(py)?; - let reports: Vec<(u64, Option)> = self.reported(py)?; - let mut state = shared.locked()?; - let mut notify: bool = state.record_reports(py, reports, round)?; - let stopped: bool = if Shared::past_deadline(&state) || state.force_stop { - let drained: Vec> = std::mem::take(&mut state.queue.items); - for item in &drained { - state.counts = drop_lifecycle_attempts( - std::mem::take(&mut state.counts), - &item.eligible_exporters, - ); - } - for index in 0..state.lanes.len() { - notify |= state.abandon_lane(py, (index, false), round)?; - } - for (index, _) in std::mem::take(&mut state.held) { - state.counts = drop_lifecycle_attempts(std::mem::take(&mut state.counts), &[index]); - } - drop(drained); - true - } else { - notify |= state.expire_lanes(py, shared.invocation_timeout, round)?; - if !state.reserved && state.reserve_at.is_some_and(|at| Instant::now() >= at) { - state.reserved = true; - state.reserve_for_outcomes(py, &shared.claims)?; - } - notify |= state.feed(py, round)?; - state.stopping && !state.busy() && state.queue.items.is_empty() && state.held.is_empty() - }; - let blocked: Vec = std::mem::take(&mut state.newly_blocked); - drop(state); - for name in blocked { - warn_blocked(py, &name, shared.invocation_timeout); - } - if notify { - shared.notify_all(py)?; - } - if !stopped { - shared.wait(py, POLL_SECONDS)?; - } - Ok(!stopped) - } - - /// Every `(token, sanitized error type)` delivery threads reported since the last round. - fn reported(&self, py: Python<'_>) -> PyResult)>> { - let results: &Bound<'_, PyAny> = self.shared.results.bind(py); - let mut reports: Vec<(u64, Option)> = Vec::new(); - while !results.call_method0("empty")?.is_truthy()? { - let (token, error): (u64, Bound<'_, PyAny>) = - results.call_method0("get_nowait")?.extract()?; - let error_type: Option = - (!error.is_none()).then(|| error_type_of(py, &PyErr::from_value(error))); - reports.push((token, error_type)); - } - Ok(reports) - } - - fn next_notification(&self, py: Python<'_>) -> PyResult { - let shared = &self.shared; - let _held = shared.hold(py)?; - loop { - let mut state = shared.locked()?; - if let Some(snapshot) = state.pending_periodic.take() { - state.periodic_inflight = Some(snapshot.health_signature()); - return Ok(Notification::Periodic(snapshot)); - } - state.periodic_inflight = None; - if !state.final_delivered - && let Some(summary) = &state.final_notification - { - return Ok(Notification::Final(summary.clone_ref(py))); - } - if let Some(failure) = state.failures.pop_front() { - return Ok(Notification::Failure(failure)); - } - if state.notification_stopping { - return Ok(Notification::Stop); - } - drop(state); - shared.wait(py, POLL_SECONDS)?; - } - } - - fn run_notifications(&self, py: Python<'_>) -> PyResult<()> { - let shared = &self.shared; - loop { - match self.next_notification(py)? { - Notification::Periodic(snapshot) => { - let signature: [u64; 7] = snapshot.health_signature(); - if let Some(callback) = &shared.summary_callback { - let summary = EventExportSummary::object(py, snapshot)?; - drop(callback.bind(py).call1((summary,))); - } - let _held = shared.hold(py)?; - let mut state = shared.locked()?; - state.last_published_periodic = signature; - state.periodic_inflight = None; - } - Notification::Final(summary) => { - if let Some(callback) = &shared.summary_callback { - drop(callback.bind(py).call1((summary,))); - } - let released = { - let _held = shared.hold(py)?; - let mut state = shared.locked()?; - state.final_delivered = true; - state.final_notification.take() - }; - drop(released); - return Ok(()); - } - Notification::Failure(failure) => { - if let Some(callback) = &shared.failure_callback { - let failure = Py::new(py, failure)?; - drop(callback.bind(py).call1((failure,))); - } - } - Notification::Stop => return Ok(()), - } - } - } - - /// Wait one health interval, returning `false` once health publication stops. - fn await_interval(&self, py: Python<'_>) -> PyResult { - let shared = &self.shared; - let deadline: Instant = Instant::now() + shared.health_interval; - let _held = shared.hold(py)?; - loop { - if shared.locked()?.health_stopping { - return Ok(false); - } - let remaining: f64 = deadline - .saturating_duration_since(Instant::now()) - .as_secs_f64(); - if remaining <= NO_SECONDS { - return Ok(true); - } - shared.wait(py, remaining)?; - } - } - - fn run_health(&self, py: Python<'_>) -> PyResult<()> { - let shared = &self.shared; - while self.await_interval(py)? { - let snapshot: ExportSnapshot = shared.current_snapshot(py)?; - let signature: [u64; 7] = snapshot.health_signature(); - let _held = shared.hold(py)?; - let mut state = shared.locked()?; - if state.health_stopping || state.final_notification.is_some() { - return Ok(()); - } - if signature == state.last_published_periodic - || state.periodic_inflight == Some(signature) - { - state.pending_periodic = None; - continue; - } - state.pending_periodic = Some(snapshot); - drop(state); - shared.notify_all(py)?; - } - Ok(()) - } -} - -#[pymethods] -impl DispatcherWorker { - fn __call__(&self, py: Python<'_>) -> PyResult<()> { - match self.role { - WorkerRole::Dispatch => self.run_dispatch(py), - WorkerRole::Notify => self.run_notifications(py), - WorkerRole::Health => self.run_health(py), - } - } -} diff --git a/crates/sqlbuild-python-output/src/event_exporting/constants.rs b/crates/sqlbuild-python-output/src/event_exporting/constants.rs new file mode 100644 index 0000000000..c74a8cb0b7 --- /dev/null +++ b/crates/sqlbuild-python-output/src/event_exporting/constants.rs @@ -0,0 +1,50 @@ +//! Public constants of event exporting. + +use crate::event_exporting::_helpers::getters::read_bound_event_exporter; +use crate::event_exporting::_helpers::getters::read_event_export_summary; +use crate::event_exporting::_helpers::getters::read_event_exporter_accounting; +use crate::event_exporting::_helpers::getters::read_event_exporter_command_scope; +use crate::event_exporting::_helpers::getters::read_event_exporter_counts; +use crate::event_exporting::_helpers::getters::read_event_exporter_dispatcher; +use crate::event_exporting::_helpers::getters::read_event_exporter_failure; +use crate::event_exporting::models::EventExporterDispatcher; +use sqlbuild_python_base::event_exporting::models::BoundEventExporter; +use sqlbuild_python_base::event_exporting::models::EventExportSummary; +use sqlbuild_python_base::event_exporting::models::EventExporterAccounting; +use sqlbuild_python_base::event_exporting::models::EventExporterCommandScope; +use sqlbuild_python_base::event_exporting::models::EventExporterCounts; +use sqlbuild_python_base::event_exporting::models::EventExporterFailure; +use sqlbuild_python_base::records::main::registry::class_of::class_of; +use sqlbuild_python_base::records::models::GetterEntry; + +/// The getter readers of this module's native pyclasses. +pub const GETTERS: &[GetterEntry] = &[ + GetterEntry { + class: class_of::, + read: read_event_exporter_dispatcher, + }, + GetterEntry { + class: class_of::, + read: read_bound_event_exporter, + }, + GetterEntry { + class: class_of::, + read: read_event_export_summary, + }, + GetterEntry { + class: class_of::, + read: read_event_exporter_accounting, + }, + GetterEntry { + class: class_of::, + read: read_event_exporter_counts, + }, + GetterEntry { + class: class_of::, + read: read_event_exporter_failure, + }, + GetterEntry { + class: class_of::, + read: read_event_exporter_command_scope, + }, +]; diff --git a/crates/sqlbuild-python-output/src/event_exporting/main/mod.rs b/crates/sqlbuild-python-output/src/event_exporting/main/mod.rs new file mode 100644 index 0000000000..72044fb874 --- /dev/null +++ b/crates/sqlbuild-python-output/src/event_exporting/main/mod.rs @@ -0,0 +1,2 @@ +pub mod register_policy; +pub mod stop_event_exporter_threads_function; diff --git a/crates/sqlbuild-python-output/src/event_exporting/main/register_policy.rs b/crates/sqlbuild-python-output/src/event_exporting/main/register_policy.rs new file mode 100644 index 0000000000..65ef310ab1 --- /dev/null +++ b/crates/sqlbuild-python-output/src/event_exporting/main/register_policy.rs @@ -0,0 +1,13 @@ +//! Register the policy functions and classes. + +use crate::event_exporting::_helpers::policy::event_export_severity_at_least; +use crate::event_exporting::_helpers::policy::lifecycle_export_policy_catalog; +use crate::event_exporting::_helpers::policy::stricter_event_export_severity; +use pyo3::prelude::{Bound, PyModule, PyModuleMethods, PyResult, wrap_pyfunction}; + +/// Register the policy functions and classes. +pub fn register_policy(module: &Bound<'_, PyModule>) -> PyResult<()> { + module.add_function(wrap_pyfunction!(lifecycle_export_policy_catalog, module)?)?; + module.add_function(wrap_pyfunction!(event_export_severity_at_least, module)?)?; + module.add_function(wrap_pyfunction!(stricter_event_export_severity, module)?) +} diff --git a/crates/sqlbuild-python-output/src/event_exporting/main/stop_event_exporter_threads_function.rs b/crates/sqlbuild-python-output/src/event_exporting/main/stop_event_exporter_threads_function.rs new file mode 100644 index 0000000000..254346fa44 --- /dev/null +++ b/crates/sqlbuild-python-output/src/event_exporting/main/stop_event_exporter_threads_function.rs @@ -0,0 +1,10 @@ +//! `stop_event_exporter_threads` as a function of `module`. + +use crate::event_exporting::_helpers::dispatcher::stop_event_exporter_threads; + +/// `stop_event_exporter_threads` as a function of `module`. +pub fn stop_event_exporter_threads_function<'py>( + module: &pyo3::Bound<'py, pyo3::types::PyModule>, +) -> pyo3::PyResult> { + pyo3::wrap_pyfunction!(stop_event_exporter_threads, module) +} diff --git a/crates/sqlbuild-python-output/src/event_exporting/mod.rs b/crates/sqlbuild-python-output/src/event_exporting/mod.rs new file mode 100644 index 0000000000..0070ad7544 --- /dev/null +++ b/crates/sqlbuild-python-output/src/event_exporting/mod.rs @@ -0,0 +1,6 @@ +//! Event exporting: the dispatcher that queues build events, the delivery workers and the export policy. + +pub(crate) mod _helpers; +pub mod constants; +pub mod main; +pub mod models; diff --git a/crates/sqlbuild-python-output/src/event_exporting/models.rs b/crates/sqlbuild-python-output/src/event_exporting/models.rs new file mode 100644 index 0000000000..e8b23c67bd --- /dev/null +++ b/crates/sqlbuild-python-output/src/event_exporting/models.rs @@ -0,0 +1,691 @@ +//! Public records of event exporting. + +use crate::event_exporting::_helpers::dispatcher::Delivery; +use crate::event_exporting::_helpers::dispatcher::Limits; +use crate::event_exporting::_helpers::dispatcher::OutputSink; +use crate::event_exporting::_helpers::dispatcher::Queued; +use crate::event_exporting::_helpers::dispatcher::Shared; +use crate::event_exporting::_helpers::dispatcher::State; +use crate::event_exporting::_helpers::dispatcher::join; +use crate::event_exporting::_helpers::dispatcher::limits; +use crate::event_exporting::_helpers::dispatcher::read_exporters; +use crate::event_exporting::_helpers::dispatcher::read_output_sinks; +use crate::event_exporting::_helpers::dispatcher::remaining_seconds; +use crate::event_exporting::_helpers::dispatcher::retire_output_lane; +use crate::event_exporting::_helpers::dispatcher::start_worker; +use crate::event_exporting::_helpers::workers::Notification; +use crate::event_exporting::_helpers::workers::Round; +use crate::event_exporting::_helpers::workers::WorkerRole; +use crate::event_exporting::_helpers::workers::warn_blocked; +use pyo3::prelude::Bound; +use pyo3::prelude::Py; +use pyo3::prelude::PyAny; +use pyo3::prelude::PyAnyMethods; +use pyo3::prelude::PyErr; +use pyo3::prelude::PyResult; +use pyo3::prelude::Python; +use pyo3::prelude::pyclass; +use pyo3::prelude::pymethods; +use pyo3::types::PyDict; +use pyo3::types::PyFloat; +use sqlbuild_core::command_output::constants::COMMAND_OUTPUT_RECORD_TYPE; +use sqlbuild_core::event_export::constants::INVOCATION_START_EVENT_TYPE; +use sqlbuild_core::event_export::constants::INVOCATION_TERMINAL_EVENT_TYPES; +use sqlbuild_core::event_export::constants::OUTCOME_EVENT_TYPES; +use sqlbuild_core::event_export::constants::OUTCOME_RESERVE_FRACTION; +use sqlbuild_core::event_export::main::drop_lifecycle_attempts::drop_lifecycle_attempts; +use sqlbuild_core::event_export::main::queue_lifecycle_event::queue_lifecycle_event; +use sqlbuild_core::event_export::main::route_lifecycle_event::route_lifecycle_event; +use sqlbuild_core::event_export::models::EventQueue; +use sqlbuild_core::event_export::models::ExportPolicy; +use sqlbuild_core::event_export::models::ExportSnapshot; +use sqlbuild_core::event_export::models::ExporterCounts; +use sqlbuild_core::event_export::models::QueuedEvent; +use sqlbuild_python_base::boundary::main::call_with::call_with; +use sqlbuild_python_base::boundary::main::imported::imported; +use sqlbuild_python_base::boundary::models::KeywordArguments; +use sqlbuild_python_base::event_exporting::constants::DISPATCHER_THREAD_NAME; +use sqlbuild_python_base::event_exporting::constants::EXIT_JOIN_SECONDS; +use sqlbuild_python_base::event_exporting::constants::HEALTH_THREAD_NAME; +use sqlbuild_python_base::event_exporting::constants::JOIN_GRACE_SECONDS; +use sqlbuild_python_base::event_exporting::constants::NO_SECONDS; +use sqlbuild_python_base::event_exporting::constants::NOTIFIER_THREAD_NAME; +use sqlbuild_python_base::event_exporting::constants::POLL_SECONDS; +use sqlbuild_python_base::event_exporting::main::error_type_of::error_type_of; +use sqlbuild_python_base::event_exporting::main::export_policy::export_policy; +use sqlbuild_python_base::event_exporting::main::state_error::state_error; +use sqlbuild_python_base::event_exporting::models::EventExportSummary; +use sqlbuild_python_base::records::main::fields::field::field as native_field; +use std::collections::HashMap; +use std::collections::HashSet; +use std::collections::VecDeque; +use std::sync::Arc; +use std::sync::Mutex; +use std::time::Instant; + +/// Deliver filtered lifecycle events off execution threads with bounded memory. +#[pyclass(module = "sqlbuild._native", frozen)] +#[derive(Debug)] +pub struct EventExporterDispatcher { + pub(crate) shared: Arc, + /// Event kinds this command never exports, such as compile's phase operations. + pub(crate) withheld_kinds: Vec, + pub(crate) thread: Py, + pub(crate) notification_thread: Option>, + pub(crate) health_thread: Option>, +} + +#[pymethods] +impl EventExporterDispatcher { + #[new] + #[pyo3(signature = (**options))] + pub fn new(py: Python<'_>, options: Option>) -> PyResult { + let options: KeywordArguments<'_> = KeywordArguments::read( + py, + "EventExporterDispatcher", + options, + ( + &[], + &[ + "exporters", + "queue_capacity", + "shutdown_timeout_seconds", + "invocation_timeout_seconds", + "notification_queue_capacity", + "health_interval_seconds", + "failure_callback", + "summary_callback", + "withheld_kinds", + ], + ), + )?; + let withheld: Bound<'_, PyAny> = options.value("withheld_kinds")?; + let withheld_kinds: Vec = if withheld.is_none() { + Vec::new() + } else { + withheld.extract()? + }; + let limits: Limits = limits(&options)?; + let exporters: Bound<'_, PyAny> = options.value("exporters")?; + let (bound_exporters, filters) = if exporters.is_none() { + (Vec::new(), Vec::new()) + } else { + read_exporters(&exporters)? + }; + let callback = |name: &str| -> PyResult>> { + let value: Bound<'_, PyAny> = options.value(name)?; + Ok((!value.is_none()).then(|| value.unbind())) + }; + let failure_callback: Option> = callback("failure_callback")?; + let summary_callback: Option> = callback("summary_callback")?; + let notifications: bool = failure_callback.is_some() || summary_callback.is_some(); + let has_summary: bool = summary_callback.is_some(); + let shared = Arc::new(Shared { + condition: imported(py, "threading", "Condition")?.call0()?.unbind(), + results: imported(py, "queue", "SimpleQueue")?.call0()?.unbind(), + claims: PyDict::new(py).unbind(), + state: Mutex::new(State { + bound: !exporters.is_none(), + counts: vec![ExporterCounts::default(); bound_exporters.len()], + exporters: bound_exporters, + filters, + output_sinks: Arc::new(Vec::new()), + output_lane: None, + queue: EventQueue::new(limits.queue_capacity), + sequence: 0, + accepting: true, + deadline: None, + reserve_at: None, + reserved: false, + lanes: Vec::new(), + tokens: HashMap::new(), + next_token: 0, + retired: Vec::new(), + abandoned_calls: false, + stopping: false, + force_stop: false, + blocked: HashSet::new(), + stalled: HashMap::new(), + held: Vec::new(), + newly_blocked: Vec::new(), + last_charged: None, + live_invocations: HashSet::new(), + next_invocation: 0, + dispatcher_running: true, + shutdown_started: false, + shutdown_complete: false, + final_summary: None, + failures: VecDeque::new(), + notification_stopping: false, + health_stopping: false, + pending_periodic: None, + periodic_inflight: None, + last_published_periodic: [ + 0, + 0, + 0, + 0, + 0, + 0, + u64::try_from(limits.queue_capacity).unwrap_or(u64::MAX), + ], + final_notification: None, + final_delivered: false, + }), + invocation_timeout: limits.invocation_timeout, + shutdown_timeout: limits.shutdown_timeout, + health_interval: limits.health_interval, + notification_capacity: limits.notification_capacity, + failure_callback, + summary_callback, + notifications, + }); + let notification_thread: Option> = notifications + .then(|| start_worker(py, &shared, (WorkerRole::Notify, NOTIFIER_THREAD_NAME))) + .transpose()?; + let health_thread: Option> = has_summary + .then(|| start_worker(py, &shared, (WorkerRole::Health, HEALTH_THREAD_NAME))) + .transpose()?; + for thread in notification_thread.iter().chain(&health_thread) { + thread.call_method0("start")?; + } + let thread = start_worker(py, &shared, (WorkerRole::Dispatch, DISPATCHER_THREAD_NAME))?; + thread.call_method0("start")?; + Ok(Self { + shared, + withheld_kinds, + thread: thread.unbind(), + notification_thread: notification_thread.map(Bound::unbind), + health_thread: health_thread.map(Bound::unbind), + }) + } + + #[getter] + pub fn thread(&self, py: Python<'_>) -> Py { + self.thread.clone_ref(py) + } + + #[getter] + pub fn notification_thread(&self, py: Python<'_>) -> Option> { + self.notification_thread + .as_ref() + .map(|thread| thread.clone_ref(py)) + } + + /// Filter and enqueue eligible exporter attempts without waiting. + pub(crate) fn enqueue(&self, py: Python<'_>, event: &Bound<'_, PyAny>) -> PyResult<()> { + let event_type: String = native_field(event, "event_type")?.extract()?; + let policy: ExportPolicy = export_policy(py, &event_type)?; + if self.withheld_kinds.contains(&policy.kind) { + return Ok(()); + } + let invocation_id: Option = match native_field(event, "invocation_id") { + Ok(value) => value.extract()?, + Err(_) => None, + }; + let shared = &self.shared; + let _held = shared.hold(py)?; + let mut state = shared.locked()?; + if !state.bound { + drop(state); + return Err(state_error( + py, + "event exporters must be bound before enqueue", + )); + } + let counts: Vec = std::mem::take(&mut state.counts); + let (counts, eligible) = route_lifecycle_event(&policy, &state.filters, counts); + state.counts = counts; + if eligible.is_empty() { + return Ok(()); + } + if !state.accepting { + state.counts = drop_lifecycle_attempts(std::mem::take(&mut state.counts), &eligible); + return Ok(()); + } + let sequence: u64 = state.sequence; + state.sequence += 1; + let capacity: usize = state.queue.capacity; + let mut outcome = queue_lifecycle_event( + std::mem::replace(&mut state.queue, EventQueue::new(capacity)), + QueuedEvent { + sequence, + item: Queued { + event: event.clone().unbind(), + kind: policy.kind, + severity: policy.severity, + }, + priority: policy.priority, + invocation_terminal: INVOCATION_TERMINAL_EVENT_TYPES.contains(&event_type.as_str()), + outcome: OUTCOME_EVENT_TYPES.contains(&event_type.as_str()), + invocation_id, + invocation_start: event_type == INVOCATION_START_EVENT_TYPE, + eligible_exporters: eligible, + }, + ); + state.queue = std::mem::replace(&mut outcome.queue, EventQueue::new(capacity)); + for lost in outcome.displaced.iter().chain(&outcome.rejected) { + state.counts = drop_lifecycle_attempts( + std::mem::take(&mut state.counts), + &lost.eligible_exporters, + ); + } + drop(state); + shared.notify_all(py)?; + drop(outcome); + Ok(()) + } + + /// Bind validated exporters once before lifecycle publication starts. + pub(crate) fn bind(&self, py: Python<'_>, exporters: &Bound<'_, PyAny>) -> PyResult<()> { + let (exporters, filters) = read_exporters(exporters)?; + let shared = &self.shared; + let _held = shared.hold(py)?; + let mut state = shared.locked()?; + if state.bound { + drop(state); + return Err(state_error(py, "event exporters are already bound")); + } + state.counts = vec![ExporterCounts::default(); exporters.len()]; + state.exporters = exporters; + state.filters = filters; + state.bound = true; + drop(state); + shared.notify_all(py) + } + + /// Bind explicit command-output sinks before capture starts. + pub(crate) fn bind_command_output_sinks( + &self, + py: Python<'_>, + sinks: &Bound<'_, PyAny>, + ) -> PyResult<()> { + let sinks: Arc> = Arc::new(read_output_sinks(sinks)?); + let previous = { + let _held = self.shared.hold(py)?; + std::mem::replace(&mut self.shared.locked()?.output_sinks, sinks) + }; + drop(previous); + Ok(()) + } + + /// Deliver output records through the already-bound destination providers. + pub(crate) fn export_output(&self, py: Python<'_>, records: &Bound<'_, PyAny>) -> PyResult<()> { + let shared = &self.shared; + let invocation: u64 = shared.begin_invocation(py)?; + let sinks: Arc> = { + let _held = shared.hold(py)?; + Arc::clone(&shared.locked()?.output_sinks) + }; + let mut failed: bool = false; + let delivered = (|| -> PyResult { + for record in records.try_iter()? { + let record: Bound<'_, PyAny> = record?; + let is_output: bool = + native_field(&record, "record_type")?.eq(COMMAND_OUTPUT_RECORD_TYPE)?; + let stream: Option = if is_output { + Some(native_field(&record, "stream")?.extract()?) + } else { + None + }; + for sink in sinks.iter() { + if stream + .as_ref() + .is_some_and(|stream| !sink.streams.contains(stream)) + { + continue; + } + match shared.call_output_sink(py, sink, &record)? { + Delivery::Delivered(sink_failed) => failed |= sink_failed, + Delivery::Abandoned => return Ok(Delivery::Abandoned), + } + } + } + Ok(Delivery::Delivered(failed)) + })(); + if let Ok(Delivery::Abandoned) = delivered { + return Err(state_error( + py, + "output export stopped because the interpreter is exiting", + )); + } + shared.invocation_finished(py, invocation)?; + delivered?; + if failed { + return Err(state_error(py, "one or more output export attempts failed")); + } + Ok(()) + } + + /// Stop acceptance and drain until the fixed shutdown deadline. + pub(crate) fn shutdown(&self, py: Python<'_>) -> PyResult> { + let shared = &self.shared; + let (owns, deadline) = { + let _held = shared.hold(py)?; + let mut state = shared.locked()?; + let owns: bool = !state.shutdown_started; + if owns { + state.shutdown_started = true; + state.accepting = false; + let started: Instant = Instant::now(); + state.deadline = Some(started + shared.shutdown_timeout); + state.reserve_at = Some( + started + + shared + .shutdown_timeout + .mul_f64(1.0 - OUTCOME_RESERVE_FRACTION), + ); + state.stopping = true; + state.health_stopping = true; + } + let deadline: Instant = state.deadline.unwrap_or_else(Instant::now); + drop(state); + shared.notify_all(py)?; + (owns, deadline) + }; + if !owns { + return shared.await_shutdown(py); + } + let thread: &Bound<'_, PyAny> = self.thread.bind(py); + call_with( + &native_field(thread, "join")?, + &[( + "timeout", + PyFloat::new(py, remaining_seconds(deadline) + JOIN_GRACE_SECONDS).into_any(), + )], + )?; + if thread.call_method0("is_alive")?.is_truthy()? { + { + let _held = shared.hold(py)?; + shared.locked()?.force_stop = true; + shared.notify_all(py)?; + } + thread.call_method0("join")?; + } + let snapshot: ExportSnapshot = { + let _held = shared.hold(py)?; + Shared::snapshot(&*shared.locked()?, true) + }; + let summary: Py = EventExportSummary::object(py, snapshot)?; + { + let _held = shared.hold(py)?; + let mut state = shared.locked()?; + state.final_summary = Some(summary.clone_ref(py)); + state.shutdown_complete = true; + if shared.summary_callback.is_some() + && state.final_notification.is_none() + && !state.final_delivered + { + state.final_notification = Some(summary.clone_ref(py)); + } + state.notification_stopping = true; + let retired: Option> = if state.live_invocations.is_empty() { + state.output_lane.take() + } else { + None + }; + drop(state); + shared.notify_all(py)?; + retire_output_lane(py, retired)?; + } + if let Some(health) = &self.health_thread { + join(health.bind(py), EXIT_JOIN_SECONDS)?; + } + Ok(summary) + } + + /// Run `finalizer` on the calling thread, never a daemon delivery thread, once delivery is idle or the idle wait ends. + pub(crate) fn finalize_when_idle(&self, py: Python<'_>, finalizer: Py) -> PyResult<()> { + self.shared.await_idle(py)?; + drop(finalizer.bind(py).call0()); + Ok(()) + } + + /// Return a thread-safe immutable point-in-time snapshot. + pub(crate) fn summary(&self, py: Python<'_>) -> PyResult> { + let snapshot: ExportSnapshot = self.shared.current_snapshot(py)?; + EventExportSummary::object(py, snapshot) + } +} + +/// A long-lived dispatcher thread's target. +#[pyclass(module = "sqlbuild._native", frozen)] +pub struct DispatcherWorker { + pub(crate) shared: Arc, + pub(crate) role: WorkerRole, +} + +impl std::fmt::Debug for DispatcherWorker { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("DispatcherWorker") + .finish_non_exhaustive() + } +} + +impl DispatcherWorker { + pub(crate) fn new(shared: Arc, role: WorkerRole) -> Self { + Self { shared, role } + } + + /// The state this worker's dispatcher shares. + pub(crate) fn shared(&self) -> &Shared { + &self.shared + } + + /// Wait until bound, returning `false` when stopped first. + pub(crate) fn await_bound(&self, py: Python<'_>) -> PyResult { + let shared = &self.shared; + let _held = shared.hold(py)?; + loop { + let state = shared.locked()?; + if state.bound { + return Ok(true); + } + if state.stopping || state.force_stop { + return Ok(false); + } + drop(state); + shared.wait(py, POLL_SECONDS)?; + } + } + + pub(crate) fn run_dispatch(&self, py: Python<'_>) -> PyResult<()> { + let result = (|| -> PyResult<()> { + if !self.await_bound(py)? { + return Ok(()); + } + while self.dispatch_round(py)? {} + Ok(()) + })(); + let retired: PyResult<()> = (|| { + let _held = self.shared.hold(py)?; + self.shared.locked()?.retire_lanes(py) + })(); + self.shared.dispatcher_finished(py)?; + retired?; + result + } + + /// Record reported outcomes, expire, trim and feed lanes, then wait; `false` once delivery stops. + pub(crate) fn dispatch_round(&self, py: Python<'_>) -> PyResult { + let shared = &self.shared; + let round = Round { + shared, + diagnostics: shared.diagnostics(), + }; + let _held = shared.hold(py)?; + let reports: Vec<(u64, Option)> = self.reported(py)?; + let mut state = shared.locked()?; + let mut notify: bool = state.record_reports(py, reports, round)?; + let stopped: bool = if Shared::past_deadline(&state) || state.force_stop { + let drained: Vec> = std::mem::take(&mut state.queue.items); + for item in &drained { + state.counts = drop_lifecycle_attempts( + std::mem::take(&mut state.counts), + &item.eligible_exporters, + ); + } + for index in 0..state.lanes.len() { + notify |= state.abandon_lane(py, (index, false), round)?; + } + for (index, _) in std::mem::take(&mut state.held) { + state.counts = drop_lifecycle_attempts(std::mem::take(&mut state.counts), &[index]); + } + drop(drained); + true + } else { + notify |= state.expire_lanes(py, shared.invocation_timeout, round)?; + if !state.reserved && state.reserve_at.is_some_and(|at| Instant::now() >= at) { + state.reserved = true; + state.reserve_for_outcomes(py, &shared.claims)?; + } + notify |= state.feed(py, round)?; + state.stopping && !state.busy() && state.queue.items.is_empty() && state.held.is_empty() + }; + let blocked: Vec = std::mem::take(&mut state.newly_blocked); + drop(state); + for name in blocked { + warn_blocked(py, &name, shared.invocation_timeout); + } + if notify { + shared.notify_all(py)?; + } + if !stopped { + shared.wait(py, POLL_SECONDS)?; + } + Ok(!stopped) + } + + /// Every `(token, sanitized error type)` delivery threads reported since the last round. + pub(crate) fn reported(&self, py: Python<'_>) -> PyResult)>> { + let results: &Bound<'_, PyAny> = self.shared.results.bind(py); + let mut reports: Vec<(u64, Option)> = Vec::new(); + while !results.call_method0("empty")?.is_truthy()? { + let (token, error): (u64, Bound<'_, PyAny>) = + results.call_method0("get_nowait")?.extract()?; + let error_type: Option = + (!error.is_none()).then(|| error_type_of(py, &PyErr::from_value(error))); + reports.push((token, error_type)); + } + Ok(reports) + } + + pub(crate) fn next_notification(&self, py: Python<'_>) -> PyResult { + let shared = &self.shared; + let _held = shared.hold(py)?; + loop { + let mut state = shared.locked()?; + if let Some(snapshot) = state.pending_periodic.take() { + state.periodic_inflight = Some(snapshot.health_signature()); + return Ok(Notification::Periodic(snapshot)); + } + state.periodic_inflight = None; + if !state.final_delivered + && let Some(summary) = &state.final_notification + { + return Ok(Notification::Final(summary.clone_ref(py))); + } + if let Some(failure) = state.failures.pop_front() { + return Ok(Notification::Failure(failure)); + } + if state.notification_stopping { + return Ok(Notification::Stop); + } + drop(state); + shared.wait(py, POLL_SECONDS)?; + } + } + + pub(crate) fn run_notifications(&self, py: Python<'_>) -> PyResult<()> { + let shared = &self.shared; + loop { + match self.next_notification(py)? { + Notification::Periodic(snapshot) => { + let signature: [u64; 7] = snapshot.health_signature(); + if let Some(callback) = &shared.summary_callback { + let summary = EventExportSummary::object(py, snapshot)?; + drop(callback.bind(py).call1((summary,))); + } + let _held = shared.hold(py)?; + let mut state = shared.locked()?; + state.last_published_periodic = signature; + state.periodic_inflight = None; + } + Notification::Final(summary) => { + if let Some(callback) = &shared.summary_callback { + drop(callback.bind(py).call1((summary,))); + } + let released = { + let _held = shared.hold(py)?; + let mut state = shared.locked()?; + state.final_delivered = true; + state.final_notification.take() + }; + drop(released); + return Ok(()); + } + Notification::Failure(failure) => { + if let Some(callback) = &shared.failure_callback { + let failure = Py::new(py, failure)?; + drop(callback.bind(py).call1((failure,))); + } + } + Notification::Stop => return Ok(()), + } + } + } + + /// Wait one health interval, returning `false` once health publication stops. + pub(crate) fn await_interval(&self, py: Python<'_>) -> PyResult { + let shared = &self.shared; + let deadline: Instant = Instant::now() + shared.health_interval; + let _held = shared.hold(py)?; + loop { + if shared.locked()?.health_stopping { + return Ok(false); + } + let remaining: f64 = deadline + .saturating_duration_since(Instant::now()) + .as_secs_f64(); + if remaining <= NO_SECONDS { + return Ok(true); + } + shared.wait(py, remaining)?; + } + } + + pub(crate) fn run_health(&self, py: Python<'_>) -> PyResult<()> { + let shared = &self.shared; + while self.await_interval(py)? { + let snapshot: ExportSnapshot = shared.current_snapshot(py)?; + let signature: [u64; 7] = snapshot.health_signature(); + let _held = shared.hold(py)?; + let mut state = shared.locked()?; + if state.health_stopping || state.final_notification.is_some() { + return Ok(()); + } + if signature == state.last_published_periodic + || state.periodic_inflight == Some(signature) + { + state.pending_periodic = None; + continue; + } + state.pending_periodic = Some(snapshot); + drop(state); + shared.notify_all(py)?; + } + Ok(()) + } +} + +#[pymethods] +impl DispatcherWorker { + pub(crate) fn __call__(&self, py: Python<'_>) -> PyResult<()> { + match self.role { + WorkerRole::Dispatch => self.run_dispatch(py), + WorkerRole::Notify => self.run_notifications(py), + WorkerRole::Health => self.run_health(py), + } + } +} diff --git a/crates/sqlbuild-python-output/src/lib.rs b/crates/sqlbuild-python-output/src/lib.rs index 10d1540e1b..a3704154f9 100644 --- a/crates/sqlbuild-python-output/src/lib.rs +++ b/crates/sqlbuild-python-output/src/lib.rs @@ -5,6 +5,7 @@ pub mod build_progress; pub mod compute_logs; pub mod cost_telemetry; +pub mod event_exporting; pub mod output_capture; pub mod recall; pub mod result_output; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/cli/observability.rs b/crates/sqlbuild-python/src/bindings/_helpers/cli/observability.rs index 84d84dc90b..be8bef7517 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/cli/observability.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/cli/observability.rs @@ -15,7 +15,6 @@ use sqlbuild_cli::dispatch::models::{ExportSummary, ExporterLoss}; use sqlbuild_core::event_export::constants::DEFAULT_EVENT_EXPORT_SHUTDOWN_TIMEOUT_SECONDS; use sqlbuild_host::entries::models::{ExecutorHost, UserCodeHost}; -use crate::bindings::_helpers::event_exporting::dispatcher::EventExporterDispatcher; use crate::bindings::_helpers::native_host::setup_context::invocation_setup_context; use sqlbuild_python_base::boundary::main::call_with::call_with; use sqlbuild_python_base::boundary::main::imported::imported; @@ -30,6 +29,7 @@ use sqlbuild_python_base::native_host::main::host_entry::host_entry; use sqlbuild_python_base::native_host::main::redirect_stdout_to_stderr::redirect_stdout_to_stderr; use sqlbuild_python_base::presentation_text::main::stdout_supports_color::stdout_supports_color; use sqlbuild_python_base::records::main::fields::field::field as native_field; +use sqlbuild_python_output::event_exporting::models::EventExporterDispatcher; use sqlbuild_python_output::recall::main::write_stderr::write_stderr; use sqlbuild_python_output::result_output::main::events::terminal_event_index_scope::terminal_event_index_scope; use sqlbuild_python_output::result_output::models::TerminalEventIndex; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/event_exporting/mod.rs b/crates/sqlbuild-python/src/bindings/_helpers/event_exporting/mod.rs index 16eba7bfb1..7444bd0374 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/event_exporting/mod.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/event_exporting/mod.rs @@ -1,5 +1 @@ -pub(crate) mod dispatcher; -pub(crate) mod getters; -pub(crate) mod policy; pub(crate) mod registration; -pub(crate) mod workers; diff --git a/crates/sqlbuild-python/src/bindings/_helpers/event_exporting/registration.rs b/crates/sqlbuild-python/src/bindings/_helpers/event_exporting/registration.rs index 6335096ee9..0bbb7d92e3 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/event_exporting/registration.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/event_exporting/registration.rs @@ -1,12 +1,7 @@ //! Register native lifecycle-event export and stop its threads at interpreter exit. -use pyo3::prelude::{Bound, PyAnyMethods, PyModule, PyModuleMethods, PyResult, wrap_pyfunction}; +use pyo3::prelude::{Bound, PyAnyMethods, PyModule, PyModuleMethods, PyResult}; -use crate::bindings::_helpers::event_exporting::dispatcher::{ - EventExporterDispatcher, stop_event_exporter_threads, -}; -use crate::bindings::_helpers::event_exporting::policy; -use crate::bindings::_helpers::event_exporting::workers::DispatcherWorker; use sqlbuild_python_base::boundary::main::imported::imported; use sqlbuild_python_base::event_exporting::main::register_scope::register_scope; use sqlbuild_python_base::event_exporting::models::BoundEventExporter; @@ -14,6 +9,10 @@ use sqlbuild_python_base::event_exporting::models::EventExportSummary; use sqlbuild_python_base::event_exporting::models::EventExporterAccounting; use sqlbuild_python_base::event_exporting::models::EventExporterCounts; use sqlbuild_python_base::event_exporting::models::EventExporterFailure; +use sqlbuild_python_output::event_exporting::main::register_policy::register_policy; +use sqlbuild_python_output::event_exporting::main::stop_event_exporter_threads_function::stop_event_exporter_threads_function; +use sqlbuild_python_output::event_exporting::models::DispatcherWorker; +use sqlbuild_python_output::event_exporting::models::EventExporterDispatcher; pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_class::()?; @@ -23,9 +22,9 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add_class::()?; module.add_class::()?; module.add_class::()?; - let hook = wrap_pyfunction!(stop_event_exporter_threads, module)?; + let hook = stop_event_exporter_threads_function(module)?; module.add_function(hook.clone())?; imported(module.py(), "atexit", "register")?.call1((hook,))?; - policy::register(module)?; + register_policy(module)?; register_scope(module) } diff --git a/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/constants.rs b/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/constants.rs index d3a487ceda..d8e63e1987 100644 --- a/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/constants.rs +++ b/crates/sqlbuild-python/src/bindings/_helpers/record_behaviours/constants.rs @@ -68,7 +68,7 @@ use crate::bindings::_helpers::record_behaviours::scopes::{ use crate::bindings::_helpers::record_behaviours::values::{ finish_fingerprint_set, logical_type_display_name, sql_value_kind, }; -use crate::bindings::_helpers::{dbt_events, dbt_interop, event_exporting}; +use crate::bindings::_helpers::{dbt_events, dbt_interop}; use sqlbuild_python_base::records::models::{GetterEntry, RecordBehaviours}; use sqlbuild_python_base::records::types::ClassMethod; use sqlbuild_python_base::records::types::ClassProperty; @@ -364,7 +364,7 @@ pub(crate) const GETTER_TABLES: &[&[GetterEntry]] = &[ dbt_interop::getters::GETTERS, sqlbuild_python_base::diagnostics::constants::GETTERS, sqlbuild_python_front::discovery::constants::GETTERS, - event_exporting::getters::GETTERS, + sqlbuild_python_output::event_exporting::constants::GETTERS, sqlbuild_python_front::execution_history::constants::GETTERS, sqlbuild_python_project::graph::constants::GETTERS, sqlbuild_python_front::model_config::constants::GETTERS, diff --git a/fensu.toml b/fensu.toml index bb7e60942e..333605e565 100644 --- a/fensu.toml +++ b/fensu.toml @@ -1675,7 +1675,7 @@ select = ["FPRS", "XSB072", "XSB073", "XSB075", "XSB076", "XSB077", "XSB078", "X [[targets.rust.threshold_overrides]] paths = ["crates/sqlbuild-python/Cargo.toml"] -thresholds = { max_crate_code_lines = 52134 } +thresholds = { max_crate_code_lines = 50333 } reason = "The Python bindings crate is above the code-line limit; this caps it at its current size until it is split into crates by area. Lower the cap as code moves out." [[targets.rust.threshold_overrides]] @@ -1895,6 +1895,12 @@ path = "crates/sqlbuild-python-output/src/recall/models.rs" symbols = ["RecallReplacements"] reason = "RecallReplacements implement Debug manually because they cache Python projections in PyOnceLocks, directly or through the projections they hold, which do not implement Debug; other models derive it." +[[targets.rust.rule_exceptions]] +rule = "FPRSS201" +path = "crates/sqlbuild-python-output/src/event_exporting/models.rs" +symbols = ["DispatcherWorker"] +reason = "DispatcherWorker implements Debug manually because its WorkerRole does not implement Debug; other models derive it." + [[targets.rust.rule_exceptions]] rule = "FPRSR307" path = "crates/sqlbuild-rules/src/scope_metadata/main.rs" @@ -2482,6 +2488,9 @@ pending_budgets = [ "crates/sqlbuild-python-output/src/cost_telemetry/_helpers/allocation.rs=1", "crates/sqlbuild-python-output/src/cost_telemetry/_helpers/ledger.rs=13", "crates/sqlbuild-python-output/src/cost_telemetry/_helpers/snowflake.rs=23", + "crates/sqlbuild-python-output/src/event_exporting/_helpers/dispatcher.rs=19", + "crates/sqlbuild-python-output/src/event_exporting/_helpers/workers.rs=6", + "crates/sqlbuild-python-output/src/event_exporting/models.rs=14", "crates/sqlbuild-python-output/src/output_capture/_helpers/context.rs=5", "crates/sqlbuild-python-output/src/output_capture/_helpers/dispatcher.rs=29", "crates/sqlbuild-python-output/src/output_capture/_helpers/records.rs=13", @@ -2767,9 +2776,7 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/diff_report/query.rs=17", "crates/sqlbuild-python/src/bindings/_helpers/diff_report/support.rs=5", "crates/sqlbuild-python/src/bindings/_helpers/diff_report/tolerances.rs=5", - "crates/sqlbuild-python/src/bindings/_helpers/event_exporting/dispatcher.rs=28", "crates/sqlbuild-python/src/bindings/_helpers/event_exporting/registration.rs=2", - "crates/sqlbuild-python/src/bindings/_helpers/event_exporting/workers.rs=11", "crates/sqlbuild-python/src/bindings/_helpers/exported_constants/registration.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/format_command/command.rs=15", "crates/sqlbuild-python/src/bindings/_helpers/format_command/config.rs=1", @@ -2963,6 +2970,9 @@ pending_budgets = [ "crates/sqlbuild-python-output/src/build_progress/_helpers/nested_progress.rs=2", "crates/sqlbuild-python-output/src/build_progress/models.rs=18", "crates/sqlbuild-python-output/src/cost_telemetry/_helpers/snowflake.rs=7", + "crates/sqlbuild-python-output/src/event_exporting/_helpers/dispatcher.rs=11", + "crates/sqlbuild-python-output/src/event_exporting/_helpers/workers.rs=2", + "crates/sqlbuild-python-output/src/event_exporting/models.rs=3", "crates/sqlbuild-python-output/src/output_capture/_helpers/context.rs=2", "crates/sqlbuild-python-output/src/output_capture/_helpers/dispatcher.rs=9", "crates/sqlbuild-python-output/src/recall/_helpers/replacements.rs=3", @@ -3053,8 +3063,6 @@ pending_budgets = [ "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/runner.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/debug_command/run_debug.rs=4", "crates/sqlbuild-python/src/bindings/_helpers/diff_report/query.rs=4", - "crates/sqlbuild-python/src/bindings/_helpers/event_exporting/dispatcher.rs=14", - "crates/sqlbuild-python/src/bindings/_helpers/event_exporting/workers.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/format_command/command.rs=2", "crates/sqlbuild-python/src/bindings/_helpers/format_command/files.rs=1", "crates/sqlbuild-python/src/bindings/_helpers/format_command/run.rs=6", @@ -3127,7 +3135,7 @@ module_layers = [ "diagnostics=registry", "diff_report=commands", "discovery=registry", - "event_exporting=output", + "event_exporting=registry", "execution_history=registry", "exported_constants=registry", "fingerprints=registry", @@ -3409,7 +3417,7 @@ paths = ["crates/sqlbuild-python-base/src/warehouse_access/main/read_catalog.rs" reason = "Each public entry forwards to its own helper with the same arguments." [[dupes.allowlist]] -paths = ["crates/sqlbuild-python-base/src/boundary/_helpers/expected_errors.rs", "crates/sqlbuild-python-front/src/discovery/main/headers/register_header_values.rs", "crates/sqlbuild-python-front/src/macro_bridge/main/register_module_digests.rs", "crates/sqlbuild-python-project/src/semantic_shapes/main/register_shapes.rs", "crates/sqlbuild-python/src/bindings/_helpers/command_runtime/connection.rs", "crates/sqlbuild-python-compile/src/compile_command/_helpers/invocation/report.rs", "crates/sqlbuild-python-compile/src/compile_driver/_helpers/setup/project.rs", "crates/sqlbuild-python-compile/src/compile_driver/_helpers/setup/python_plan_entries.rs", "crates/sqlbuild-python/src/bindings/_helpers/cost_report/degradation.rs", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/pipeline.rs", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/plan_output.rs", "crates/sqlbuild-python/src/bindings/_helpers/event_exporting/policy.rs", "crates/sqlbuild-python-planner/src/microbatches/_helpers/projection.rs", "crates/sqlbuild-python-planner/src/planner_changes/_helpers/detect.rs", "crates/sqlbuild-python-planner/src/planner_changes/_helpers/metadata.rs", "crates/sqlbuild-python-planner/src/planner_resolution/_helpers/cursor/cursor_ranges.rs", "crates/sqlbuild-python-planner/src/planner_resolution/_helpers/references/locations.rs", "crates/sqlbuild-python-planner/src/planner_resolution/_helpers/references/orchestration.rs", "crates/sqlbuild-python-planner/src/planner_strategy/_helpers/audits.rs"] +paths = ["crates/sqlbuild-python-base/src/boundary/_helpers/expected_errors.rs", "crates/sqlbuild-python-front/src/discovery/main/headers/register_header_values.rs", "crates/sqlbuild-python-front/src/macro_bridge/main/register_module_digests.rs", "crates/sqlbuild-python-project/src/semantic_shapes/main/register_shapes.rs", "crates/sqlbuild-python/src/bindings/_helpers/command_runtime/connection.rs", "crates/sqlbuild-python-compile/src/compile_command/_helpers/invocation/report.rs", "crates/sqlbuild-python-compile/src/compile_driver/_helpers/setup/project.rs", "crates/sqlbuild-python-compile/src/compile_driver/_helpers/setup/python_plan_entries.rs", "crates/sqlbuild-python/src/bindings/_helpers/cost_report/degradation.rs", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/pipeline.rs", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/plan_output.rs", "crates/sqlbuild-python-planner/src/microbatches/_helpers/projection.rs", "crates/sqlbuild-python-planner/src/planner_changes/_helpers/detect.rs", "crates/sqlbuild-python-planner/src/planner_changes/_helpers/metadata.rs", "crates/sqlbuild-python-planner/src/planner_resolution/_helpers/cursor/cursor_ranges.rs", "crates/sqlbuild-python-planner/src/planner_resolution/_helpers/references/locations.rs", "crates/sqlbuild-python-planner/src/planner_resolution/_helpers/references/orchestration.rs", "crates/sqlbuild-python-planner/src/planner_strategy/_helpers/audits.rs"] reason = "Each registers its own module's Python functions; only the add-function calls are alike." [[dupes.allowlist]] @@ -3685,7 +3693,7 @@ paths = ["crates/sqlbuild-python-base/src/boundary/_helpers/oracles.rs", "crates reason = "Each registers its own module's Python functions; only the add-function calls are alike." [[dupes.allowlist]] -paths = ["crates/sqlbuild-python-base/src/boundary/_helpers/expected_errors.rs", "crates/sqlbuild-python-front/src/discovery/main/headers/register_header_values.rs", "crates/sqlbuild-python-front/src/macro_bridge/main/register_module_digests.rs", "crates/sqlbuild-python-planner/src/microbatches/main/register_projection.rs", "crates/sqlbuild-python-planner/src/planner_changes/main/register_detect.rs", "crates/sqlbuild-python-planner/src/planner_changes/main/register_metadata.rs", "crates/sqlbuild-python-planner/src/planner_resolution/main/cursor/register_cursor_ranges.rs", "crates/sqlbuild-python-planner/src/planner_resolution/main/references/register_locations.rs", "crates/sqlbuild-python-planner/src/planner_resolution/main/references/register_orchestration.rs", "crates/sqlbuild-python-planner/src/planner_strategy/main/audits/register_audits.rs", "crates/sqlbuild-python-project/src/semantic_shapes/main/register_shapes.rs", "crates/sqlbuild-python/src/bindings/_helpers/command_runtime/connection.rs", "crates/sqlbuild-python-compile/src/compile_command/_helpers/invocation/report.rs", "crates/sqlbuild-python-compile/src/compile_driver/_helpers/setup/project.rs", "crates/sqlbuild-python-compile/src/compile_driver/_helpers/setup/python_plan_entries.rs", "crates/sqlbuild-python/src/bindings/_helpers/cost_report/degradation.rs", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/pipeline.rs", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/plan_output.rs", "crates/sqlbuild-python/src/bindings/_helpers/event_exporting/policy.rs"] +paths = ["crates/sqlbuild-python-base/src/boundary/_helpers/expected_errors.rs", "crates/sqlbuild-python-front/src/discovery/main/headers/register_header_values.rs", "crates/sqlbuild-python-front/src/macro_bridge/main/register_module_digests.rs", "crates/sqlbuild-python-planner/src/microbatches/main/register_projection.rs", "crates/sqlbuild-python-planner/src/planner_changes/main/register_detect.rs", "crates/sqlbuild-python-planner/src/planner_changes/main/register_metadata.rs", "crates/sqlbuild-python-planner/src/planner_resolution/main/cursor/register_cursor_ranges.rs", "crates/sqlbuild-python-planner/src/planner_resolution/main/references/register_locations.rs", "crates/sqlbuild-python-planner/src/planner_resolution/main/references/register_orchestration.rs", "crates/sqlbuild-python-planner/src/planner_strategy/main/audits/register_audits.rs", "crates/sqlbuild-python-project/src/semantic_shapes/main/register_shapes.rs", "crates/sqlbuild-python/src/bindings/_helpers/command_runtime/connection.rs", "crates/sqlbuild-python-compile/src/compile_command/_helpers/invocation/report.rs", "crates/sqlbuild-python-compile/src/compile_driver/_helpers/setup/project.rs", "crates/sqlbuild-python-compile/src/compile_driver/_helpers/setup/python_plan_entries.rs", "crates/sqlbuild-python/src/bindings/_helpers/cost_report/degradation.rs", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/pipeline.rs", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/plan_output.rs"] reason = "Each registers its own module's Python functions; only the add-function calls are alike." [[dupes.allowlist]] @@ -3789,7 +3797,7 @@ paths = ["crates/sqlbuild-python-compile/src/compile_driver/main/stages/register reason = "Each registers its own module's Python functions; only the add-function calls are alike." [[dupes.allowlist]] -paths = ["crates/sqlbuild-python-base/src/boundary/_helpers/expected_errors.rs", "crates/sqlbuild-python-compile/src/compile_command/main/invocation/register_report.rs", "crates/sqlbuild-python-compile/src/compile_driver/main/setup/register_project.rs", "crates/sqlbuild-python-compile/src/compile_driver/main/setup/register_python_plan_entries.rs", "crates/sqlbuild-python-front/src/discovery/main/headers/register_header_values.rs", "crates/sqlbuild-python-front/src/macro_bridge/main/register_module_digests.rs", "crates/sqlbuild-python-planner/src/microbatches/main/register_projection.rs", "crates/sqlbuild-python-planner/src/planner_changes/main/register_detect.rs", "crates/sqlbuild-python-planner/src/planner_changes/main/register_metadata.rs", "crates/sqlbuild-python-planner/src/planner_resolution/main/cursor/register_cursor_ranges.rs", "crates/sqlbuild-python-planner/src/planner_resolution/main/references/register_locations.rs", "crates/sqlbuild-python-planner/src/planner_resolution/main/references/register_orchestration.rs", "crates/sqlbuild-python-planner/src/planner_strategy/main/audits/register_audits.rs", "crates/sqlbuild-python-project/src/semantic_shapes/main/register_shapes.rs", "crates/sqlbuild-python/src/bindings/_helpers/command_runtime/connection.rs", "crates/sqlbuild-python/src/bindings/_helpers/cost_report/degradation.rs", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/pipeline.rs", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/plan_output.rs", "crates/sqlbuild-python/src/bindings/_helpers/event_exporting/policy.rs"] +paths = ["crates/sqlbuild-python-base/src/boundary/_helpers/expected_errors.rs", "crates/sqlbuild-python-compile/src/compile_command/main/invocation/register_report.rs", "crates/sqlbuild-python-compile/src/compile_driver/main/setup/register_project.rs", "crates/sqlbuild-python-compile/src/compile_driver/main/setup/register_python_plan_entries.rs", "crates/sqlbuild-python-front/src/discovery/main/headers/register_header_values.rs", "crates/sqlbuild-python-front/src/macro_bridge/main/register_module_digests.rs", "crates/sqlbuild-python-planner/src/microbatches/main/register_projection.rs", "crates/sqlbuild-python-planner/src/planner_changes/main/register_detect.rs", "crates/sqlbuild-python-planner/src/planner_changes/main/register_metadata.rs", "crates/sqlbuild-python-planner/src/planner_resolution/main/cursor/register_cursor_ranges.rs", "crates/sqlbuild-python-planner/src/planner_resolution/main/references/register_locations.rs", "crates/sqlbuild-python-planner/src/planner_resolution/main/references/register_orchestration.rs", "crates/sqlbuild-python-planner/src/planner_strategy/main/audits/register_audits.rs", "crates/sqlbuild-python-project/src/semantic_shapes/main/register_shapes.rs", "crates/sqlbuild-python/src/bindings/_helpers/command_runtime/connection.rs", "crates/sqlbuild-python/src/bindings/_helpers/cost_report/degradation.rs", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/pipeline.rs", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/plan_output.rs"] reason = "Each registers its own module's Python functions; only the add-function calls are alike." [[dupes.allowlist]] @@ -3984,6 +3992,14 @@ reason = "Each reporter clears its own transient line under its own lock; sharin paths = ["crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/cli.rs"] reason = "Each is its own Python entry point with its own keyword contract and dbt command shape; both end in the shared argv builder." +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-output/src/event_exporting/_helpers/getters.rs"] +reason = "The export summary and the per-exporter counts are separate Python classes that each expose the same delivery counters; each getter reads its own class's fields." + +[[dupes.allowlist]] +paths = ["crates/sqlbuild-python-base/src/boundary/_helpers/expected_errors.rs", "crates/sqlbuild-python-compile/src/compile_command/main/invocation/register_report.rs", "crates/sqlbuild-python-compile/src/compile_driver/main/setup/register_project.rs", "crates/sqlbuild-python-compile/src/compile_driver/main/setup/register_python_plan_entries.rs", "crates/sqlbuild-python-front/src/discovery/main/headers/register_header_values.rs", "crates/sqlbuild-python-front/src/macro_bridge/main/register_module_digests.rs", "crates/sqlbuild-python-output/src/event_exporting/main/register_policy.rs", "crates/sqlbuild-python-planner/src/microbatches/main/register_projection.rs", "crates/sqlbuild-python-planner/src/planner_changes/main/register_detect.rs", "crates/sqlbuild-python-planner/src/planner_changes/main/register_metadata.rs", "crates/sqlbuild-python-planner/src/planner_resolution/main/cursor/register_cursor_ranges.rs", "crates/sqlbuild-python-planner/src/planner_resolution/main/references/register_locations.rs", "crates/sqlbuild-python-planner/src/planner_resolution/main/references/register_orchestration.rs", "crates/sqlbuild-python-planner/src/planner_strategy/main/audits/register_audits.rs", "crates/sqlbuild-python-project/src/semantic_shapes/main/register_shapes.rs", "crates/sqlbuild-python/src/bindings/_helpers/command_runtime/connection.rs", "crates/sqlbuild-python/src/bindings/_helpers/cost_report/degradation.rs", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/pipeline.rs", "crates/sqlbuild-python/src/bindings/_helpers/dbt_interop/plan_output.rs"] +reason = "Each registers its own module's Python functions; only the add-function calls are alike." + [[dupes.allowlist]] paths = ["crates/sqlbuild-analysis/src/sql_tree/_helpers/source_map.rs"] reason = "source_start and source_end apply opposite boundary rules: a start inside a rewrite maps to the start of the replaced text, while an exclusive end inside or closing one maps to its end, and an end at a removal maps to where the removed text began. Only the partition-and-shift shape is alike."