diff --git a/Cargo.toml b/Cargo.toml index bfc5666be3..8a3c63a00d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -138,7 +138,7 @@ errno = "0.3" windows-sys = { version = "0.61", default-features = false, features = ["Win32_System_Diagnostics_Debug"] } # Recall's local store: SQLite compiled in, so every build has the same engine version. rusqlite = { version = "0.37", features = ["bundled", "column_decltype"] } -rustix = { version = "1.1.4", default-features = false, features = ["fs", "process", "std", "termios"] } +rustix = { version = "1.1.4", default-features = false, features = ["fs", "process", "std", "stdio", "termios"] } saphyr-parser = "0.1" regex = "1" reqwest = { version = "0.12", default-features = false, features = ["json", "rustls-tls"] } diff --git a/clients/typescript/src/schema.d.ts b/clients/typescript/src/schema.d.ts index 15e3989cfa..ef9ad3ebf4 100644 --- a/clients/typescript/src/schema.d.ts +++ b/clients/typescript/src/schema.d.ts @@ -1914,7 +1914,7 @@ export interface components { name: string; target: string; }; - /** @description Locations report bounded facts, never connection strings or user exception text. */ + /** @description Locations report bounded facts, never connection strings; a failed warehouse read carries only its redacted error text. */ LocationHealth: { /** @enum {string} */ kind: "heartbeat"; @@ -1939,6 +1939,8 @@ export interface components { } | { /** Format: int64 */ elapsed_millis: number; + /** @description A failed read's warehouse error, redacted and bounded; absent otherwise. */ + error?: string | null; failure?: null | components["schemas"]["WarehouseSyncFailure"]; /** Format: int64 */ generation: number; @@ -3587,7 +3589,7 @@ export interface components { session_id: string; }; /** - * @description Closed failure stages keep warehouse errors and credentials out of health observations. + * @description The stage a warehouse sync failed at; only a failed read also reports its redacted error. * @enum {string} */ WarehouseSyncFailure: "watermark" | "read" | "delivery" | "busy"; diff --git a/crates/sqlbuild-cli/src/dev_screen/_helpers/apply_screen_message.rs b/crates/sqlbuild-cli/src/dev_screen/_helpers/apply_screen_message.rs new file mode 100644 index 0000000000..5e68c57115 --- /dev/null +++ b/crates/sqlbuild-cli/src/dev_screen/_helpers/apply_screen_message.rs @@ -0,0 +1,83 @@ +//! Apply one message to the `sqb dev` screen. + +use std::time::Instant; + +use sqlbuild_core::screen_text::models::PhaseState; + +use crate::dev_screen::_helpers::lines::{compose, ready_lines}; +use crate::dev_screen::_helpers::transitions::{ + child_event, client_text, key_pressed, located, logged, run_reported, step_finished, + step_started, warehouse_settled, +}; +use crate::dev_screen::models::{DevScreen, DevStep, ScreenAction, ScreenMessage, ScreenOutput}; + +/// `screen` changed by `message` at `now` (local wall time `clock`), and the text to write. +pub(crate) fn apply_screen_message( + screen: DevScreen, + message: ScreenMessage, + now: Instant, + clock: &str, +) -> (DevScreen, ScreenOutput) { + let mut action: ScreenAction = ScreenAction::None; + let mut raw: String = String::new(); + let (mut screen, mut lines): (DevScreen, Vec) = match message { + ScreenMessage::Started { step, detail } => step_started(screen, (step, detail), now), + ScreenMessage::Finished { step, detail } => { + step_finished(screen, (step, PhaseState::Ok), detail, now) + } + ScreenMessage::Child(event) => child_event(screen, event, now, clock), + ScreenMessage::Location(location) => located(screen, location), + ScreenMessage::Triggers(triggers) => (DevScreen { triggers, ..screen }, Vec::new()), + ScreenMessage::Warehouse { adapter, state } => { + let (screen, closing, lines) = warehouse_settled(screen, (adapter, state), now, clock); + raw = closing; + (screen, lines) + } + ScreenMessage::ClientOutput(text) => { + let (screen, text) = client_text(screen, &text, now); + return (screen, ScreenOutput { text, action }); + } + ScreenMessage::Run { + name, + detail, + finished, + } => { + let lines: Vec = run_reported(&screen, (name, detail), finished, clock); + (screen, lines) + } + ScreenMessage::Log(line) => logged(screen, line), + ScreenMessage::Key(key) => { + let (screen, shown, asked) = key_pressed(screen, key); + action = asked; + (screen, shown) + } + ScreenMessage::Tick => { + if !screen.live || screen.ready || screen.client_output || connecting(&screen) { + return ( + screen, + ScreenOutput { + text: String::new(), + action, + }, + ); + } + let frame: usize = screen.frame + 1; + (DevScreen { frame, ..screen }, Vec::new()) + } + }; + if screen.event_gap && !lines.is_empty() { + screen.event_gap = false; + lines.insert(0, String::new()); + } + let (screen, ready) = ready_lines(screen, now); + lines.extend(ready); + let (screen, text) = compose(screen, (&raw, &lines), now); + (screen, ScreenOutput { text, action }) +} + +/// The foreground connection may prompt on the terminal itself, so nothing redraws over it. +fn connecting(screen: &DevScreen) -> bool { + screen.steps.iter().any(|row| { + row.step == DevStep::Warehouse && matches!(row.state, PhaseState::Running { .. }) + }) +} diff --git a/crates/sqlbuild-cli/src/dev_screen/_helpers/capture.rs b/crates/sqlbuild-cli/src/dev_screen/_helpers/capture.rs new file mode 100644 index 0000000000..cd853f9508 --- /dev/null +++ b/crates/sqlbuild-cli/src/dev_screen/_helpers/capture.rs @@ -0,0 +1,54 @@ +//! Route this process's stderr into the screen's log while the screen owns the terminal. + +use std::fs::File; +#[cfg(unix)] +use std::io::{BufRead, BufReader}; +#[cfg(unix)] +use std::os::fd::AsFd; + +use tokio::sync::mpsc::UnboundedSender; + +use crate::dev_screen::models::ScreenMessage; + +/// Point stderr at a pipe whose lines become log messages, and return the terminal it pointed at. +#[cfg(unix)] +pub(crate) fn capture_stderr( + messages: UnboundedSender, +) -> std::io::Result> { + let terminal: File = File::from(std::io::stderr().as_fd().try_clone_to_owned()?); + let (reader, writer) = std::io::pipe()?; + rustix::stdio::dup2_stderr(&writer)?; + drop(writer); + std::thread::spawn(move || { + for line in BufReader::new(reader).split(b'\n') { + let Ok(line) = line else { + return; + }; + let text: String = String::from_utf8_lossy(&line) + .trim_end_matches('\r') + .to_owned(); + if messages.send(ScreenMessage::Log(text)).is_err() { + return; + } + } + }); + Ok(Some(terminal)) +} + +/// Without file-descriptor redirection, stderr stays on the terminal. +#[cfg(not(unix))] +pub(crate) fn capture_stderr( + _messages: UnboundedSender, +) -> std::io::Result> { + Ok(None) +} + +/// Point stderr back at the terminal. +#[cfg(unix)] +pub(crate) fn release_stderr(terminal: &File) { + let _ = rustix::stdio::dup2_stderr(terminal); +} + +/// Nothing was redirected. +#[cfg(not(unix))] +pub(crate) fn release_stderr(_terminal: &File) {} diff --git a/crates/sqlbuild-cli/src/dev_screen/_helpers/drawing.rs b/crates/sqlbuild-cli/src/dev_screen/_helpers/drawing.rs new file mode 100644 index 0000000000..6046427f86 --- /dev/null +++ b/crates/sqlbuild-cli/src/dev_screen/_helpers/drawing.rs @@ -0,0 +1,86 @@ +//! The screen task: apply each message, write its text, and leave the closing text behind. + +use std::fs::File; +use std::io::Write; +use std::time::Instant; + +use sqlbuild_core::screen_text::main::redraw_rows::redraw_rows; +use tokio::sync::mpsc::{UnboundedReceiver, UnboundedSender}; +use tokio::sync::oneshot; + +use crate::dev_screen::_helpers::apply_screen_message::apply_screen_message; +use crate::dev_screen::_helpers::capture::release_stderr; +use crate::dev_screen::_helpers::keys::KeyReader; +use crate::dev_screen::constants::{FAILURE_LOG_LINES, TICK}; +use crate::dev_screen::models::{DevScreen, ScreenAction, ScreenMessage}; + +/// Apply each message and write what it draws, until the session finishes; then leave the closing text. +pub(crate) async fn draw_screen( + mut screen: DevScreen, + (mut inbox, actions): ( + UnboundedReceiver, + UnboundedSender, + ), + mut finished: oneshot::Receiver, + (keys, terminal): (Option, Option), +) { + let mut tick = tokio::time::interval(TICK); + tick.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); + let failed: bool = loop { + let message: ScreenMessage = tokio::select! { + failed = &mut finished => break failed.unwrap_or(false), + message = inbox.recv() => match message { + Some(message) => message, + None => break false, + }, + _ = tick.tick(), if screen.live => ScreenMessage::Tick, + }; + let clock: String = chrono::Local::now().format("%H:%M:%S").to_string(); + let output; + (screen, output) = apply_screen_message(screen, message, Instant::now(), &clock); + write_screen(terminal.as_ref(), &output.text); + if output.action != ScreenAction::None { + let _ = actions.send(output.action); + } + }; + drop(keys); + if let Some(terminal) = terminal.as_ref() { + release_stderr(terminal); + } + write_screen(None, &closing_text(&screen, failed)); +} + +/// What stays on the terminal after the screen stops: the live rows cleared and, on failure, the newest log lines that explain it. +fn closing_text(screen: &DevScreen, failed: bool) -> String { + let mut text: String = if screen.live && screen.drawn > 0 { + redraw_rows(screen.drawn, &[]) + } else { + String::new() + }; + if failed && !screen.show_log { + let skip: usize = screen.log.len().saturating_sub(FAILURE_LOG_LINES); + for line in screen.log.iter().skip(skip) { + text.push_str(line); + text.push('\n'); + } + } + text +} + +/// Write to the terminal stderr was taken from, or to stderr when it was not redirected. +fn write_screen(terminal: Option<&File>, text: &str) { + if text.is_empty() { + return; + } + match terminal { + Some(mut terminal) => drop(terminal.write_all(text.as_bytes())), + None => { + let mut stderr = std::io::stderr().lock(); + drop( + stderr + .write_all(text.as_bytes()) + .and_then(|()| stderr.flush()), + ); + } + } +} diff --git a/crates/sqlbuild-cli/src/dev_screen/_helpers/keys.rs b/crates/sqlbuild-cli/src/dev_screen/_helpers/keys.rs new file mode 100644 index 0000000000..0e5c825d7f --- /dev/null +++ b/crates/sqlbuild-cli/src/dev_screen/_helpers/keys.rs @@ -0,0 +1,100 @@ +//! Hotkeys from the terminal: stdin in raw mode, handed back to the warehouse client while it prompts. + +use std::sync::Arc; +use std::sync::atomic::Ordering; +use std::thread::JoinHandle; +#[cfg(unix)] +use std::time::Duration; + +#[cfg(unix)] +use rustix::termios::{ + LocalModes, OptionalActions, SpecialCodeIndex, Termios, tcgetattr, tcsetattr, +}; +use tokio::sync::mpsc::UnboundedSender; + +#[cfg(unix)] +use crate::dev_screen::constants::KEY_POLL; +use crate::dev_screen::models::{KeyGate, ScreenMessage}; + +/// The hotkey thread; dropping it restores the terminal. +pub(crate) struct KeyReader { + gate: Arc, + thread: Option>, +} + +impl KeyReader { + /// Start reading keys into `keys`, or `None` when stdin is not a terminal. + #[cfg(unix)] + pub(crate) fn start(keys: UnboundedSender) -> Option { + let cooked: Termios = match tcgetattr(std::io::stdin()) { + Ok(cooked) => cooked, + Err(_) => return None, + }; + let gate: Arc = Arc::new(KeyGate::default()); + let shared: Arc = Arc::clone(&gate); + let thread = std::thread::spawn(move || read_keys(&shared, &cooked, &keys)); + Some(Self { + gate, + thread: Some(thread), + }) + } + + /// Hotkeys need a Unix terminal; the screen is plain elsewhere. + #[cfg(not(unix))] + pub(crate) fn start(_keys: UnboundedSender) -> Option { + None + } + + pub(crate) fn gate(&self) -> Arc { + Arc::clone(&self.gate) + } +} + +impl Drop for KeyReader { + fn drop(&mut self) { + self.gate.stop.store(true, Ordering::SeqCst); + if let Some(thread) = self.thread.take() { + drop(thread.join()); + } + } +} + +/// Keys without echo or line buffering; Ctrl-C still interrupts, and reads give up after a tenth of a second. +#[cfg(unix)] +fn raw_mode(cooked: &Termios) -> Termios { + let mut raw: Termios = cooked.clone(); + raw.local_modes -= LocalModes::ICANON | LocalModes::ECHO; + raw.special_codes[SpecialCodeIndex::VMIN] = 0; + raw.special_codes[SpecialCodeIndex::VTIME] = 1; + raw +} + +#[cfg(unix)] +fn read_keys(gate: &KeyGate, cooked: &Termios, keys: &UnboundedSender) { + let raw: Termios = raw_mode(cooked); + let mut byte: [u8; 1] = [0]; + while !gate.stop.load(Ordering::SeqCst) && !keys.is_closed() { + let reading: bool = gate.reading.load(Ordering::SeqCst); + if gate.closed.load(Ordering::SeqCst) { + if reading { + let _ = tcsetattr(std::io::stdin(), OptionalActions::Now, cooked); + gate.reading.store(false, Ordering::SeqCst); + } + std::thread::sleep(KEY_POLL); + continue; + } + if !reading { + if tcsetattr(std::io::stdin(), OptionalActions::Now, &raw).is_err() { + return; + } + gate.reading.store(true, Ordering::SeqCst); + } + match rustix::io::read(std::io::stdin(), &mut byte) { + Ok(1) => drop(keys.send(ScreenMessage::Key(char::from(byte[0]).to_ascii_lowercase()))), + Ok(_) => {} + Err(_) => std::thread::sleep(Duration::from_millis(100)), + } + } + let _ = tcsetattr(std::io::stdin(), OptionalActions::Now, cooked); + gate.reading.store(false, Ordering::SeqCst); +} diff --git a/crates/sqlbuild-cli/src/dev_screen/_helpers/lines.rs b/crates/sqlbuild-cli/src/dev_screen/_helpers/lines.rs new file mode 100644 index 0000000000..e188225929 --- /dev/null +++ b/crates/sqlbuild-cli/src/dev_screen/_helpers/lines.rs @@ -0,0 +1,106 @@ +//! The lines the `sqb dev` screen prints: event lines, the ready card, and the redraw that places them. + +use std::time::{Duration, Instant}; + +use sqlbuild_core::screen_text::main::format_plain_line::format_plain_line; +use sqlbuild_core::screen_text::main::redraw_rows::redraw_rows; +use sqlbuild_core::screen_text::main::render_event_line::render_event_line; +use sqlbuild_core::screen_text::main::render_ready_card::render_ready_card; +use sqlbuild_core::screen_text::models::{EventLine, EventMark, ReadyCard, Segment}; + +use crate::dev_screen::_helpers::rows::live_rows; +use crate::dev_screen::models::{DevScreen, DevStep}; + +/// One event: its live line, and the plain line's message. +pub(crate) struct Report<'a> { + pub(crate) clock: &'a str, + pub(crate) mark: EventMark, + pub(crate) title: String, + pub(crate) detail: String, + pub(crate) elapsed: Option, + pub(crate) notes: Vec>, + pub(crate) plain: String, +} + +/// The event's rows on a live screen, or its one plain line. +pub(crate) fn report_lines(screen: &DevScreen, report: Report<'_>) -> Vec { + if !screen.live { + return vec![format_plain_line(&report.plain, report.elapsed)]; + } + render_event_line( + &EventLine { + clock: report.clock.to_owned(), + mark: report.mark, + title: report.title, + detail: report.detail, + elapsed: report.elapsed, + notes: report.notes, + }, + &screen.facts, + ) +} + +/// The card, or the plain `ready` line, once the first generation is published and the warehouse settled. +pub(crate) fn ready_lines(mut screen: DevScreen, now: Instant) -> (DevScreen, Vec) { + let published: bool = screen + .steps + .iter() + .any(|row| row.step == DevStep::Publish && row.elapsed.is_some()); + let (Some(generation), Some(warehouse), false, true) = ( + screen.serving, + screen.warehouse.clone(), + screen.ready, + published, + ) else { + return (screen, Vec::new()); + }; + screen.ready = true; + if !screen.live { + let line: String = format_plain_line(&format!("ready {}", screen.ui_url), None); + return (screen, vec![line]); + } + let mut location: String = format!("{} \u{b7} gen {generation}", screen.header.target); + if !screen.compile_detail.is_empty() { + location = format!("{location} \u{b7} {}", screen.compile_detail); + } + let lines: Vec = render_ready_card( + &ReadyCard { + version: screen.header.version.clone(), + ready_in: now.saturating_duration_since(screen.started), + ui_url: screen.ui_url.clone(), + location, + adapter: screen.adapter.clone(), + warehouse, + triggers: screen.triggers.clone(), + watching: screen.watching.clone(), + }, + &screen.facts, + ); + screen.event_gap = true; + (screen, lines) +} + +/// `raw` client text, then `lines` printed for good, then the live rows redrawn below them. +pub(crate) fn compose( + mut screen: DevScreen, + (raw, lines): (&str, &[String]), + now: Instant, +) -> (DevScreen, String) { + let mut text: String = raw.to_owned(); + if !screen.live { + for line in lines { + text.push_str(line); + text.push('\n'); + } + return (screen, text); + } + if screen.client_output { + return (screen, text); + } + let live: Vec = live_rows(&screen, now); + let mut rows: Vec = lines.to_vec(); + rows.extend(live.iter().cloned()); + text.push_str(&redraw_rows(screen.drawn, &rows)); + screen.drawn = live.len(); + (screen, text) +} diff --git a/crates/sqlbuild-cli/src/dev_screen/_helpers/mod.rs b/crates/sqlbuild-cli/src/dev_screen/_helpers/mod.rs new file mode 100644 index 0000000000..0c724e415a --- /dev/null +++ b/crates/sqlbuild-cli/src/dev_screen/_helpers/mod.rs @@ -0,0 +1,8 @@ +pub(crate) mod apply_screen_message; +pub(crate) mod capture; +pub(crate) mod drawing; +pub(crate) mod keys; +pub(crate) mod lines; +pub(crate) mod new_dev_screen; +pub(crate) mod rows; +pub(crate) mod transitions; diff --git a/crates/sqlbuild-cli/src/dev_screen/_helpers/new_dev_screen.rs b/crates/sqlbuild-cli/src/dev_screen/_helpers/new_dev_screen.rs new file mode 100644 index 0000000000..81d20482e8 --- /dev/null +++ b/crates/sqlbuild-cli/src/dev_screen/_helpers/new_dev_screen.rs @@ -0,0 +1,38 @@ +//! A `sqb dev` screen before its first step. + +use std::collections::VecDeque; +use std::time::Instant; + +use crate::dev_screen::_helpers::transitions::pending; +use crate::dev_screen::constants::STEPS; +use crate::dev_screen::models::{DevScreen, ScreenSetup}; + +/// Every step pending, the warehouse unknown and nothing drawn yet. +pub(crate) fn new_dev_screen(setup: ScreenSetup, started: Instant) -> DevScreen { + DevScreen { + live: setup.live, + facts: setup.facts, + header: setup.header, + ui_url: String::new(), + watching: setup.watching, + triggers: String::new(), + started, + steps: STEPS.into_iter().map(pending).collect(), + ready: false, + location: String::new(), + adapter: String::new(), + warehouse: None, + generation_started: None, + pending_changes: 0, + serving: None, + compile_detail: String::new(), + project_root: setup.project_root, + client_output: false, + client_partial: false, + event_gap: false, + log: VecDeque::new(), + show_log: false, + frame: 0, + drawn: 0, + } +} diff --git a/crates/sqlbuild-cli/src/dev_screen/_helpers/rows.rs b/crates/sqlbuild-cli/src/dev_screen/_helpers/rows.rs new file mode 100644 index 0000000000..5a056c694b --- /dev/null +++ b/crates/sqlbuild-cli/src/dev_screen/_helpers/rows.rs @@ -0,0 +1,85 @@ +//! The live rows of the `sqb dev` screen: the checklist before ready, the hotkey bar after. + +use std::time::{Duration, Instant}; + +use sqlbuild_core::screen_text::main::render_dev_header::render_dev_header; +use sqlbuild_core::screen_text::main::render_frame_rule::render_frame_rule; +use sqlbuild_core::screen_text::main::render_hotkey_bar::render_hotkey_bar; +use sqlbuild_core::screen_text::main::render_phase_lines::render_phase_lines; +use sqlbuild_core::screen_text::models::{Hotkey, PhaseLayout, PhaseLine, PhaseState}; + +use crate::dev_screen::constants::{HOTKEYS, WAREHOUSE_FRAME_LABEL}; +use crate::dev_screen::models::{DevScreen, DevStep, StepRow}; + +/// The label a checklist row shows for `step`. +pub(crate) fn step_label(step: DevStep) -> &'static str { + match step { + DevStep::ControlStore => "Control store", + DevStep::Api => "API", + DevStep::Scheduler => "Scheduler", + DevStep::Discover => "Discover project", + DevStep::Compile => "Compile snapshot", + DevStep::Publish => "Publish", + DevStep::Warehouse => "Warehouse", + } +} + +/// How long ago `started` was, if it was. +pub(crate) fn elapsed_since(started: Option, now: Instant) -> Option { + started.map(|started| now.saturating_duration_since(started)) +} + +/// The rows redrawn in place: header and checklist, with the client's framed output under them. +pub(crate) fn live_rows(screen: &DevScreen, now: Instant) -> Vec { + if screen.ready { + return vec![String::new(), render_hotkey_bar(&hotkeys(), &screen.facts)]; + } + let lines: Vec = screen + .steps + .iter() + .map(|row| phase_line(row, screen.frame, now)) + .collect(); + let mut rows: Vec = vec![ + render_dev_header(&screen.header, &screen.facts), + String::new(), + ]; + rows.extend(render_phase_lines( + &lines, + PhaseLayout::TimeLast, + &screen.facts, + )); + if screen.client_output { + let label: &str = if screen.adapter.is_empty() { + WAREHOUSE_FRAME_LABEL + } else { + &screen.adapter + }; + rows.push(String::new()); + rows.push(render_frame_rule(Some(label), &screen.facts)); + } + rows +} + +/// One checklist row; a running row shows the current spinner frame and its live timer. +fn phase_line(row: &StepRow, frame: usize, now: Instant) -> PhaseLine { + PhaseLine { + state: match row.state { + PhaseState::Running { .. } => PhaseState::Running { frame }, + state => state, + }, + label: step_label(row.step).to_owned(), + detail: row.detail.clone(), + elapsed: row.elapsed.or_else(|| elapsed_since(row.started, now)), + } +} + +/// The hotkey bar's keys. +pub(crate) fn hotkeys() -> Vec { + HOTKEYS + .iter() + .map(|(key, label)| Hotkey { + key: *key, + label: (*label).to_owned(), + }) + .collect() +} diff --git a/crates/sqlbuild-cli/src/dev_screen/_helpers/transitions.rs b/crates/sqlbuild-cli/src/dev_screen/_helpers/transitions.rs new file mode 100644 index 0000000000..66debdf4f9 --- /dev/null +++ b/crates/sqlbuild-cli/src/dev_screen/_helpers/transitions.rs @@ -0,0 +1,506 @@ +//! How each message changes the `sqb dev` screen, and the lines it prints for good. + +use std::time::{Duration, Instant}; + +use sqlbuild_core::screen_text::main::format_plain_line::format_plain_line; +use sqlbuild_core::screen_text::main::redraw_rows::redraw_rows; +use sqlbuild_core::screen_text::main::render_frame_rule::render_frame_rule; +use sqlbuild_core::screen_text::main::render_log_line::render_log_line; +use sqlbuild_core::screen_text::models::{EventMark, PhaseState, Segment, Tone, WarehouseState}; +use sqlbuild_orchestration::dev_events::models::{ + DevEvent, DevPhase, DevStepState, SourcePosition, +}; + +use crate::dev_screen::_helpers::lines::{Report, report_lines}; +use crate::dev_screen::_helpers::rows::{elapsed_since, live_rows}; +use crate::dev_screen::constants::{FAILED_HINT, LOG_RING_LINES, SNAPSHOT_ID_CHARS}; +use crate::dev_screen::models::{DevScreen, DevStep, ScreenAction, StepRow}; + +/// The screen after a message, and the lines it prints for good. +pub(crate) type Printed = (DevScreen, Vec); + +/// A startup step began; discovery starts a new generation, so its later steps go back to pending. +pub(crate) fn step_started( + mut screen: DevScreen, + (step, detail): (DevStep, String), + now: Instant, +) -> Printed { + if step == DevStep::Discover { + screen.generation_started = Some(now); + } + if screen.ready { + return (screen, Vec::new()); + } + let reset: &[DevStep] = if step == DevStep::Discover { + &[DevStep::Compile, DevStep::Publish] + } else { + &[] + }; + for row in &mut screen.steps { + if reset.contains(&row.step) { + *row = pending(row.step); + } + if row.step == step { + *row = StepRow { + state: PhaseState::Running { frame: 0 }, + detail: detail.clone(), + started: Some(now), + ..pending(step) + }; + } + } + (screen, Vec::new()) +} + +/// A checklist row that has not started. +pub(crate) fn pending(step: DevStep) -> StepRow { + StepRow { + step, + state: PhaseState::Pending, + detail: String::new(), + started: None, + elapsed: None, + } +} + +/// The location is known; the plain screen names it and the target. +pub(crate) fn located(mut screen: DevScreen, location: String) -> Printed { + screen.location = location; + if screen.live { + return (screen, Vec::new()); + } + let line: String = format_plain_line( + &format!( + "location {} \u{b7} target {}", + screen.location, screen.header.target + ), + None, + ); + (screen, vec![line]) +} + +/// A startup step ended in `state`; the plain screen reports the control store and the compile. +pub(crate) fn step_finished( + mut screen: DevScreen, + (step, state): (DevStep, PhaseState), + detail: String, + now: Instant, +) -> Printed { + match step { + DevStep::Compile => screen.compile_detail.clone_from(&detail), + DevStep::Api => screen.ui_url = format!("http://{detail}"), + _ => {} + } + if screen.ready { + return (screen, Vec::new()); + } + let mut elapsed: Option = None; + for row in &mut screen.steps { + if row.step == step { + row.elapsed = Some(elapsed_since(row.started, now).unwrap_or_default()); + row.state = state; + row.detail.clone_from(&detail); + elapsed = row.elapsed; + } + } + if screen.live { + return (screen, Vec::new()); + } + let lines: Vec = match step { + DevStep::ControlStore => vec![format_plain_line("control store ready", elapsed)], + DevStep::Compile => vec![format_plain_line( + &format!("discovered {}", screen.compile_detail), + elapsed_since(screen.generation_started, now), + )], + _ => Vec::new(), + }; + (screen, lines) +} + +/// An event from the location child. +pub(crate) fn child_event( + mut screen: DevScreen, + event: DevEvent, + now: Instant, + clock: &str, +) -> Printed { + match event { + DevEvent::Step { + phase, + state, + detail, + } => { + let step: DevStep = match phase { + DevPhase::Discover => DevStep::Discover, + DevPhase::Compile => DevStep::Compile, + DevPhase::Publish => DevStep::Publish, + }; + match state { + DevStepState::Started => step_started(screen, (step, String::new()), now), + DevStepState::Finished => step_finished( + screen, + (step, PhaseState::Ok), + detail.unwrap_or_default(), + now, + ), + } + } + DevEvent::Changed { paths, more } => { + screen.pending_changes += paths.len() + more; + let mut titles: Vec = paths; + if more > 0 { + titles.push(format!("{more} more files")); + } + let lines: Vec = titles + .into_iter() + .flat_map(|title| changed_line(&screen, clock, title)) + .collect(); + (screen, lines) + } + DevEvent::Restarted { reason } => { + let lines: Vec = report_lines( + &screen, + Report { + clock, + mark: EventMark::Restarted, + title: "location restarted".to_owned(), + plain: format!("location restarted: {reason}"), + detail: reason, + elapsed: None, + notes: Vec::new(), + }, + ); + (screen, lines) + } + DevEvent::Published { + generation, + snapshot_id, + .. + } => published(screen, (generation, &snapshot_id), now, clock), + DevEvent::Failed { + generation, + message, + position, + } => failed( + screen, + (generation, &message, position.as_ref()), + now, + clock, + ), + DevEvent::Warning { title, detail } => { + let lines: Vec = report_lines( + &screen, + Report { + clock, + mark: EventMark::Warning, + plain: format!("{title}: {detail}"), + title, + detail: String::new(), + elapsed: None, + notes: vec![vec![Segment { + text: detail, + tone: Tone::Muted, + link: None, + }]], + }, + ); + (screen, lines) + } + } +} + +fn changed_line(screen: &DevScreen, clock: &str, title: String) -> Vec { + report_lines( + screen, + Report { + clock, + mark: EventMark::Changed, + plain: format!("changed {title}"), + title, + detail: String::new(), + elapsed: None, + notes: Vec::new(), + }, + ) +} + +fn published( + mut screen: DevScreen, + (generation, snapshot_id): (i64, &str), + now: Instant, + clock: &str, +) -> Printed { + let short: String = snapshot_id.chars().take(SNAPSHOT_ID_CHARS).collect(); + let elapsed: Option = elapsed_since(screen.generation_started.take(), now); + let changes: usize = std::mem::take(&mut screen.pending_changes); + screen.serving = Some(generation); + let plain: String = format!("published gen {generation} {short}"); + if !screen.ready { + let mut row_elapsed: Option = None; + for row in &mut screen.steps { + if row.step == DevStep::Publish { + row.detail = format!("gen {generation} \u{b7} {short}"); + row_elapsed = row.elapsed; + } + } + if screen.live { + return (screen, Vec::new()); + } + return (screen, vec![format_plain_line(&plain, row_elapsed)]); + } + let detail: String = match changes { + 0 => String::new(), + 1 => "1 file changed".to_owned(), + count => format!("{count} files changed"), + }; + let lines: Vec = report_lines( + &screen, + Report { + clock, + mark: EventMark::Succeeded, + title: format!("gen {generation} published"), + detail, + elapsed, + notes: Vec::new(), + plain, + }, + ); + (screen, lines) +} + +fn failed( + mut screen: DevScreen, + (generation, message, position): (i64, &str, Option<&SourcePosition>), + now: Instant, + clock: &str, +) -> Printed { + screen.generation_started = None; + screen.pending_changes = 0; + for row in &mut screen.steps { + if !screen.ready && matches!(row.state, PhaseState::Running { .. }) { + row.state = PhaseState::Fail; + row.elapsed = elapsed_since(row.started, now); + row.detail = format!("gen {generation} failed"); + } + } + let title: String = match screen.serving { + Some(serving) => format!("gen {generation} failed \u{b7} still serving gen {serving}"), + None => format!("gen {generation} failed"), + }; + let (place, mut first_note): (String, Vec) = match position { + Some(position) => position_note(&screen.project_root, position), + None => (String::new(), Vec::new()), + }; + first_note.push(Segment { + text: message.to_owned(), + tone: Tone::Plain, + link: None, + }); + let lines: Vec = report_lines( + &screen, + Report { + clock, + mark: EventMark::Failed, + plain: format!("{title}: {place}{message}"), + title, + detail: String::new(), + elapsed: None, + notes: vec![ + first_note, + vec![Segment { + text: FAILED_HINT.to_owned(), + tone: Tone::Muted, + link: None, + }], + ], + }, + ); + (screen, lines) +} + +/// `path:line:col ` for the plain line, and the clickable position the live note starts with. +fn position_note(project_root: &str, position: &SourcePosition) -> (String, Vec) { + let place: String = format!("{}:{}:{}", position.path, position.line, position.column); + let note: Vec = vec![ + Segment { + text: place.clone(), + tone: Tone::Link, + link: Some(format!( + "file://{}/{}", + project_root.trim_end_matches('/'), + position.path + )), + }, + Segment { + text: " ".to_owned(), + tone: Tone::Plain, + link: None, + }, + ]; + (format!("{place} "), note) +} + +/// The foreground connection settled: close any framed client block and record the outcome. +pub(crate) fn warehouse_settled( + mut screen: DevScreen, + (adapter, state): (String, WarehouseState), + now: Instant, + clock: &str, +) -> (DevScreen, String, Vec) { + let mut raw: String = String::new(); + if screen.client_output { + if screen.client_partial { + raw.push('\n'); + } + if screen.live { + raw.push_str(&render_frame_rule(None, &screen.facts)); + raw.push_str("\n\n"); + } + screen.client_output = false; + screen.client_partial = false; + screen.drawn = 0; + } + screen.adapter.clone_from(&adapter); + screen.warehouse = Some(state.clone()); + let (row_state, detail, plain): (PhaseState, String, Option) = match &state { + WarehouseState::Connected { detail } => { + (PhaseState::Ok, format!("{adapter} \u{b7} {detail}"), None) + } + WarehouseState::NotConnected { reason } => ( + PhaseState::Warn, + format!("not connected \u{b7} {reason}"), + Some(format!("warehouse not connected: {reason}")), + ), + WarehouseState::Failed { message } => ( + PhaseState::Fail, + message.clone(), + Some(format!("warehouse connection failed: {message}")), + ), + }; + if !screen.ready { + let (screen, _) = step_finished(screen, (DevStep::Warehouse, row_state), detail, now); + let lines: Vec = match (screen.live, plain) { + (false, Some(plain)) => vec![format_plain_line(&plain, None)], + _ => Vec::new(), + }; + return (screen, raw, lines); + } + let (mark, title): (EventMark, &str) = match row_state { + PhaseState::Ok => (EventMark::Succeeded, "warehouse connected"), + _ => (EventMark::Failed, "warehouse not connected"), + }; + let lines: Vec = report_lines( + &screen, + Report { + clock, + mark, + title: title.to_owned(), + plain: plain.unwrap_or_else(|| format!("warehouse connected: {detail}")), + detail, + elapsed: None, + notes: Vec::new(), + }, + ); + (screen, raw, lines) +} + +/// The client's own output, written as it came; the first chunk freezes the checklist above a rule. +pub(crate) fn client_text(mut screen: DevScreen, text: &str, now: Instant) -> (DevScreen, String) { + let mut output: String = String::new(); + if !screen.client_output { + screen.client_output = true; + if screen.live { + let rows: Vec = live_rows(&screen, now); + output.push_str(&redraw_rows(screen.drawn, &rows)); + screen.drawn = 0; + } + } + if !text.is_empty() { + screen.client_partial = !text.ends_with('\n'); + } + output.push_str(text); + (screen, output) +} + +/// A run started or finished. +pub(crate) fn run_reported( + screen: &DevScreen, + (name, detail): (String, String), + finished: Option<(bool, Duration)>, + clock: &str, +) -> Vec { + let (mark, plain, elapsed): (EventMark, String, Option) = match finished { + None => ( + EventMark::Started, + format!("run {name} started: {detail}"), + None, + ), + Some((true, elapsed)) => ( + EventMark::Succeeded, + format!("run {name} succeeded: {detail}"), + Some(elapsed), + ), + Some((false, elapsed)) => ( + EventMark::Failed, + format!("run {name} failed: {detail}"), + Some(elapsed), + ), + }; + report_lines( + screen, + Report { + clock, + mark, + title: format!("run {name}"), + detail, + elapsed, + notes: Vec::new(), + plain, + }, + ) +} + +/// Keep `line` in the ring, and show it when the log is open. +pub(crate) fn logged(mut screen: DevScreen, line: String) -> Printed { + if screen.log.len() == LOG_RING_LINES { + screen.log.pop_front(); + } + let shown: Vec = if screen.live && screen.show_log { + vec![render_log_line(&line, &screen.facts)] + } else { + Vec::new() + }; + screen.log.push_back(line); + (screen, shown) +} + +/// What a key asks for; `l` toggles the log and opens it with the lines kept so far. +pub(crate) fn key_pressed( + mut screen: DevScreen, + key: char, +) -> (DevScreen, Vec, ScreenAction) { + if !screen.live { + return (screen, Vec::new(), ScreenAction::None); + } + let action: ScreenAction = match key { + 'o' => ScreenAction::OpenUi, + 'r' => ScreenAction::Recompile, + 's' => ScreenAction::RetryWarehouse, + 'q' => ScreenAction::Quit, + _ => ScreenAction::None, + }; + if key != 'l' { + return (screen, Vec::new(), action); + } + screen.show_log = !screen.show_log; + let shown: Vec = if screen.show_log { + screen + .log + .iter() + .map(|line| render_log_line(line, &screen.facts)) + .collect() + } else { + Vec::new() + }; + (screen, shown, action) +} diff --git a/crates/sqlbuild-cli/src/dev_screen/constants.rs b/crates/sqlbuild-cli/src/dev_screen/constants.rs new file mode 100644 index 0000000000..408a61a69c --- /dev/null +++ b/crates/sqlbuild-cli/src/dev_screen/constants.rs @@ -0,0 +1,38 @@ +//! Labels, bounds and keys of the `sqb dev` screen. + +use std::time::Duration; + +use crate::dev_screen::models::DevStep; + +/// Detailed log lines kept in memory for `l`; older lines drop off the front. +pub(crate) const LOG_RING_LINES: usize = 500; +/// The keys the hotkey bar shows, in order. +pub(crate) const HOTKEYS: [(char, &str); 5] = [ + ('o', "open UI"), + ('r', "recompile"), + ('s', "retry warehouse"), + ('l', "logs"), + ('q', "quit"), +]; +/// The note under a failed generation's event line. +pub(crate) const FAILED_HINT: &str = "r retry \u{b7} l full log"; +/// The snapshot ID characters a published line shows. +pub(crate) const SNAPSHOT_ID_CHARS: usize = 8; +/// The checklist's steps, in the order they are drawn. +pub(crate) const STEPS: [DevStep; 7] = [ + DevStep::ControlStore, + DevStep::Api, + DevStep::Scheduler, + DevStep::Discover, + DevStep::Compile, + DevStep::Publish, + DevStep::Warehouse, +]; +/// The framed sign-in block's label while the adapter is not yet known. +pub(crate) const WAREHOUSE_FRAME_LABEL: &str = "warehouse"; +/// How often the hotkey thread checks whether it may read stdin. +pub(crate) const KEY_POLL: Duration = Duration::from_millis(50); +/// How often the running step's spinner and timer redraw. +pub(crate) const TICK: Duration = Duration::from_millis(100); +/// Log lines printed when `sqb dev` stops on an error, so the cause stays on screen. +pub(crate) const FAILURE_LOG_LINES: usize = 40; diff --git a/crates/sqlbuild-cli/src/dev_screen/main/mod.rs b/crates/sqlbuild-cli/src/dev_screen/main/mod.rs new file mode 100644 index 0000000000..c96d41d3cd --- /dev/null +++ b/crates/sqlbuild-cli/src/dev_screen/main/mod.rs @@ -0,0 +1 @@ +pub(crate) mod start_dev_screen; diff --git a/crates/sqlbuild-cli/src/dev_screen/main/start_dev_screen.rs b/crates/sqlbuild-cli/src/dev_screen/main/start_dev_screen.rs new file mode 100644 index 0000000000..0be3b4b1f2 --- /dev/null +++ b/crates/sqlbuild-cli/src/dev_screen/main/start_dev_screen.rs @@ -0,0 +1,52 @@ +//! Run the `sqb dev` screen as the only writer to the terminal. + +use std::fs::File; +use std::time::Instant; + +use tokio::sync::mpsc; +use tokio::sync::oneshot; + +use crate::dev_screen::_helpers::capture::capture_stderr; +use crate::dev_screen::_helpers::drawing::draw_screen; +use crate::dev_screen::_helpers::keys::KeyReader; +use crate::dev_screen::_helpers::new_dev_screen::new_dev_screen; +use crate::dev_screen::models::{ + DevScreen, DevScreenHandle, ScreenAction, ScreenMessage, ScreenSetup, +}; + +/// Draw `setup`'s screen from the messages sent to the handle; a live screen also reads hotkeys. +pub(crate) fn start_dev_screen(setup: ScreenSetup) -> DevScreenHandle { + let live: bool = setup.live; + let (messages, inbox) = mpsc::unbounded_channel::(); + let (actions, asked) = mpsc::unbounded_channel::(); + let (finish, finished) = oneshot::channel::(); + let keys: Option = if live { + KeyReader::start(messages.clone()) + } else { + None + }; + let gate = keys.as_ref().map(KeyReader::gate); + let terminal: Option = match capture_stderr(messages.clone()) { + Ok(terminal) => terminal, + Err(error) => { + drop(messages.send(ScreenMessage::Log(format!( + "stderr stays on the terminal: {error}" + )))); + None + } + }; + let screen: DevScreen = new_dev_screen(setup, Instant::now()); + let task = tokio::spawn(draw_screen( + screen, + (inbox, actions), + finished, + (keys, terminal), + )); + DevScreenHandle { + messages, + actions: asked, + keys: gate, + finish, + task, + } +} diff --git a/crates/sqlbuild-cli/src/dev_screen/mod.rs b/crates/sqlbuild-cli/src/dev_screen/mod.rs new file mode 100644 index 0000000000..b825924802 --- /dev/null +++ b/crates/sqlbuild-cli/src/dev_screen/mod.rs @@ -0,0 +1,10 @@ +//! The `sqb dev` screen: a live checklist that collapses into a ready card, then one line per event. + +pub(crate) mod _helpers; +pub(crate) mod constants; +pub(crate) mod main; +pub(crate) mod models; +pub(crate) mod types; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-cli/src/dev_screen/models.rs b/crates/sqlbuild-cli/src/dev_screen/models.rs new file mode 100644 index 0000000000..2ac0e72884 --- /dev/null +++ b/crates/sqlbuild-cli/src/dev_screen/models.rs @@ -0,0 +1,173 @@ +//! The state of the `sqb dev` screen and the messages that change it. + +use std::collections::VecDeque; +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::time::{Duration, Instant}; + +use tokio::sync::mpsc::{UnboundedReceiver, UnboundedSender}; +use tokio::sync::oneshot; +use tokio::task::JoinHandle; + +use sqlbuild_core::screen_text::models::{DevHeader, PhaseState, ScreenFacts, WarehouseState}; +use sqlbuild_orchestration::dev_events::models::DevEvent; + +/// One startup step of the checklist, in the order it is drawn. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum DevStep { + ControlStore, + Api, + Scheduler, + Discover, + Compile, + Publish, + Warehouse, +} + +/// What the screen hears about: its own steps, the location child, the warehouse, logs and keys. +#[derive(Clone, Debug)] +pub(crate) enum ScreenMessage { + /// A step began; `detail` says what it waits on, such as `snowflake \u{b7} waiting for sign-in`. + Started { + step: DevStep, + detail: String, + }, + Finished { + step: DevStep, + detail: String, + }, + Child(DevEvent), + /// The development location's ID, known once the control store is open. + Location(String), + /// The dev target's triggers, such as `41 paused`, read after the first publication. + Triggers(String), + /// The foreground warehouse connection's outcome. + Warehouse { + adapter: String, + state: WarehouseState, + }, + /// A line of the warehouse client's own output, drawn inside the framed block. + ClientOutput(String), + /// A run started, or finished with `ok`; `detail` names its origin or node count. + Run { + name: String, + detail: String, + finished: Option<(bool, Duration)>, + }, + /// A detailed log line, kept for `l`. + Log(String), + Key(char), + /// Redraw the running step's spinner and timer. + Tick, +} + +/// What the screen asks its owner to do after a key. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum ScreenAction { + None, + OpenUi, + Recompile, + RetryWarehouse, + Quit, +} + +/// The text one message writes and the action it asks for. +#[derive(Clone, Debug)] +pub(crate) struct ScreenOutput { + pub(crate) text: String, + pub(crate) action: ScreenAction, +} + +/// One checklist row: its state, detail and when it started. +#[derive(Clone, Debug)] +pub(crate) struct StepRow { + pub(crate) step: DevStep, + pub(crate) state: PhaseState, + pub(crate) detail: String, + pub(crate) started: Option, + pub(crate) elapsed: Option, +} + +/// What the screen knows before the first step starts. +#[derive(Clone, Debug)] +pub(crate) struct ScreenSetup { + pub(crate) live: bool, + pub(crate) facts: ScreenFacts, + pub(crate) header: DevHeader, + pub(crate) watching: String, + pub(crate) project_root: String, +} + +/// Everything the screen draws, and how many live rows sit below the last printed line. +#[derive(Clone, Debug)] +pub(crate) struct DevScreen { + /// Redraws in place with a spinner and hotkeys; otherwise plain stable lines. + pub(crate) live: bool, + pub(crate) facts: ScreenFacts, + pub(crate) header: DevHeader, + pub(crate) ui_url: String, + pub(crate) watching: String, + pub(crate) triggers: String, + pub(crate) started: Instant, + pub(crate) steps: Vec, + pub(crate) ready: bool, + pub(crate) location: String, + pub(crate) adapter: String, + pub(crate) warehouse: Option, + /// When the generation being compiled started, for its event line's time. + pub(crate) generation_started: Option, + /// Changed files since the last generation finished. + pub(crate) pending_changes: usize, + /// The newest published generation, which keeps serving after a failure. + pub(crate) serving: Option, + /// The last compile's summary, such as `412 python nodes, 2280 models`. + pub(crate) compile_detail: String, + /// The project directory, which error positions link into. + pub(crate) project_root: String, + /// The warehouse client is writing inside the framed block; nothing is redrawn until it ends. + pub(crate) client_output: bool, + /// The client's last output ended mid-line, such as at a prompt. + pub(crate) client_partial: bool, + /// The ready card was printed and no line follows it yet; the first one is set apart by a blank row. + pub(crate) event_gap: bool, + pub(crate) log: VecDeque, + pub(crate) show_log: bool, + pub(crate) frame: usize, + /// Rows drawn below the last permanent line, redrawn by the next message. + pub(crate) drawn: usize, +} + +/// Who may read stdin: the hotkey thread, or a foreground child while the gate is closed. +#[derive(Debug, Default)] +pub(crate) struct KeyGate { + pub(crate) closed: AtomicBool, + /// Whether stdin is in raw mode and the thread reads it. + pub(crate) reading: AtomicBool, + pub(crate) stop: AtomicBool, +} + +impl KeyGate { + /// Restore the terminal and stop reading keys; returns once the thread has let go of stdin. + pub(crate) async fn close(&self) { + self.closed.store(true, Ordering::SeqCst); + while self.reading.load(Ordering::SeqCst) { + tokio::time::sleep(crate::dev_screen::constants::KEY_POLL).await; + } + } + + /// Read keys again. + pub(crate) fn open(&self) { + self.closed.store(false, Ordering::SeqCst); + } +} + +/// A running screen: where to send messages, the actions its keys ask for, and how to stop it. +#[derive(Debug)] +pub(crate) struct DevScreenHandle { + pub(crate) messages: UnboundedSender, + pub(crate) actions: UnboundedReceiver, + /// Closed while a foreground child owns the terminal's input; `None` without hotkeys. + pub(crate) keys: Option>, + pub(crate) finish: oneshot::Sender, + pub(crate) task: JoinHandle<()>, +} diff --git a/crates/sqlbuild-cli/src/dev_screen/tests.rs b/crates/sqlbuild-cli/src/dev_screen/tests.rs new file mode 100644 index 0000000000..1797dac3e5 --- /dev/null +++ b/crates/sqlbuild-cli/src/dev_screen/tests.rs @@ -0,0 +1,6 @@ +#[path = "tests/test_apply_screen_message.rs"] +mod apply_screen_message; +#[path = "tests/helpers.rs"] +mod helpers; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-cli/src/dev_screen/tests/helpers.rs b/crates/sqlbuild-cli/src/dev_screen/tests/helpers.rs new file mode 100644 index 0000000000..51666d2ebb --- /dev/null +++ b/crates/sqlbuild-cli/src/dev_screen/tests/helpers.rs @@ -0,0 +1,269 @@ +//! Screens at fixed widths, the messages of a session, and a terminal that replays the redraws. + +use std::time::{Duration, Instant}; + +use sqlbuild_core::cli_text::models::CliStyle; +use sqlbuild_core::screen_text::models::{DevHeader, ScreenFacts, WarehouseState}; +use sqlbuild_orchestration::dev_events::models::{ + DevEvent, DevPhase, DevStepState, SourcePosition, +}; + +use crate::dev_screen::_helpers::apply_screen_message::apply_screen_message; +use crate::dev_screen::_helpers::new_dev_screen::new_dev_screen; +use crate::dev_screen::models::{DevScreen, DevStep, ScreenMessage, ScreenOutput, ScreenSetup}; +use crate::dev_screen::tests::test_types::Transcript; + +const LOCATION_ID: &str = "01a1d7ff-0b3a-7631-ba35-e44d44d072f2"; +const CLEAR_BELOW: &str = "\u{1b}[J"; +const CURSOR_UP: &str = "\u{1b}["; + +/// An uncoloured screen `columns` wide; `None` makes it plain. +pub(super) fn screen(columns: Option, start: Instant) -> DevScreen { + new_dev_screen( + ScreenSetup { + live: columns.is_some(), + facts: ScreenFacts { + columns: columns.unwrap_or(80), + style: CliStyle { use_color: false }, + links: false, + indent: 2, + }, + header: DevHeader { + version: "0.232.0".to_owned(), + project: "shop".to_owned(), + target: "dev".to_owned(), + }, + watching: "project/".to_owned(), + project_root: "/work/shop".to_owned(), + }, + start, + ) +} + +pub(super) fn step(phase: DevPhase, state: DevStepState, detail: Option<&str>) -> ScreenMessage { + ScreenMessage::Child(DevEvent::Step { + phase, + state, + detail: detail.map(str::to_owned), + }) +} + +/// Startup up to the first publication, each message 100 ms after the last. +pub(super) fn startup() -> Vec { + vec![ + ScreenMessage::Started { + step: DevStep::ControlStore, + detail: String::new(), + }, + ScreenMessage::Finished { + step: DevStep::ControlStore, + detail: "sqlite \u{b7} /home/analyst/projects/shop/.sqlbuild/control.sqlite3" + .to_owned(), + }, + ScreenMessage::Location(LOCATION_ID.to_owned()), + ScreenMessage::Started { + step: DevStep::Api, + detail: String::new(), + }, + ScreenMessage::Finished { + step: DevStep::Api, + detail: "127.0.0.1:8000".to_owned(), + }, + ScreenMessage::Started { + step: DevStep::Scheduler, + detail: String::new(), + }, + ScreenMessage::Finished { + step: DevStep::Scheduler, + detail: "41 paused".to_owned(), + }, + step(DevPhase::Discover, DevStepState::Started, None), + step(DevPhase::Discover, DevStepState::Finished, None), + step(DevPhase::Compile, DevStepState::Started, None), + step( + DevPhase::Compile, + DevStepState::Finished, + Some("412 python nodes, 2280 models"), + ), + step(DevPhase::Publish, DevStepState::Started, None), + step(DevPhase::Publish, DevStepState::Finished, None), + ScreenMessage::Child(DevEvent::Published { + generation: 1, + snapshot_id: "b429729b5c1e".to_owned(), + record: serde_json::Value::Null, + }), + ScreenMessage::Triggers("41 paused".to_owned()), + ScreenMessage::Started { + step: DevStep::Warehouse, + detail: "duckdb \u{b7} connecting".to_owned(), + }, + ] +} + +pub(super) fn connected() -> ScreenMessage { + ScreenMessage::Warehouse { + adapter: "duckdb".to_owned(), + state: WarehouseState::Connected { + detail: "main".to_owned(), + }, + } +} + +pub(super) fn not_connected() -> ScreenMessage { + ScreenMessage::Warehouse { + adapter: "snowflake".to_owned(), + state: WarehouseState::NotConnected { + reason: "sign-in skipped".to_owned(), + }, + } +} + +/// `messages` after startup. +pub(super) fn after_startup(messages: Vec) -> Vec { + startup().into_iter().chain(messages).collect() +} + +/// Two edits after ready: one publishes, the next fails at a source position; then a sync stops. +pub(super) fn edits() -> Vec { + vec![ + ScreenMessage::Child(DevEvent::Changed { + paths: vec!["models/orders/int_orders.sql".to_owned()], + more: 0, + }), + step(DevPhase::Discover, DevStepState::Started, None), + ScreenMessage::Child(DevEvent::Published { + generation: 2, + snapshot_id: "c01d2e3f4a".to_owned(), + record: serde_json::Value::Null, + }), + ScreenMessage::Child(DevEvent::Changed { + paths: vec!["models/finance/fct_revenue.sql".to_owned()], + more: 2, + }), + step(DevPhase::Discover, DevStepState::Started, None), + ScreenMessage::Child(DevEvent::Failed { + generation: 3, + message: "unknown column net_amount_usd".to_owned(), + position: Some(SourcePosition { + path: "models/finance/fct_revenue.sql".to_owned(), + line: 42, + column: 7, + }), + }), + ScreenMessage::Child(DevEvent::Warning { + title: "warehouse sync stopped".to_owned(), + detail: "5 failed reads in a row: ProgrammingError: insufficient privileges".to_owned(), + }), + ] +} + +/// A Snowflake-style sign-in prompt, ticks while it waits, a log line, then the connection. +pub(super) fn framed_sign_in() -> Vec { + vec![ + ScreenMessage::ClientOutput( + "Initiating login request with your identity provider...\n".to_owned(), + ), + ScreenMessage::Tick, + ScreenMessage::ClientOutput("Enter the URL the OAuth flow redirected you to: ".to_owned()), + ScreenMessage::Tick, + ScreenMessage::Log("connection check started".to_owned()), + connected(), + ] +} + +/// The ready card with `warehouse` on its warehouse row. +pub(super) fn card(warehouse: &str, ready_in: &str) -> Vec { + vec![ + format!(" sqlbuild dev v0.232.0 ready in {ready_in}"), + String::new(), + " \u{279c} UI http://127.0.0.1:8000".to_owned(), + " \u{279c} Location dev \u{b7} gen 1 \u{b7} 412 python nodes, 2280 models".to_owned(), + format!(" \u{279c} Warehouse {warehouse}"), + " \u{279c} Triggers 41 paused".to_owned(), + " \u{279c} Watching project/".to_owned(), + ] +} + +/// The startup checklist, its detail column `width` wide. +pub(super) fn checklist(control_store: &str, width: usize) -> Vec { + let row = |glyph: &str, label: &str, detail: &str, time: &str| -> String { + format!(" {glyph} {label:<20} {detail:8}") + }; + vec![ + " sqlbuild dev v0.232.0 \u{b7} shop \u{b7} target dev".to_owned(), + String::new(), + row("\u{2713}", "Control store", control_store, "0.1s"), + row("\u{2713}", "API", "127.0.0.1:8000", "0.1s"), + row("\u{2713}", "Scheduler", "41 paused", "0.1s"), + row("\u{2713}", "Discover project", "", "0.1s"), + row( + "\u{2713}", + "Compile snapshot", + "412 python nodes, 2280 models", + "0.1s", + ), + row("\u{2713}", "Publish", "gen 1 \u{b7} b429729b", "0.1s"), + row("\u{280b}", "Warehouse", "duckdb \u{b7} connecting", "0.0s"), + ] +} + +/// Lines in order, each followed by a newline. +pub(super) fn lines_of, S: Into>(lines: I) -> Vec { + lines.into_iter().map(Into::into).collect() +} + +/// Apply `messages` 100 ms apart at clock `09:41:`, then replay what they wrote. +pub(super) fn play(columns: Option, messages: Vec) -> Transcript { + let start: Instant = Instant::now(); + let (_, outputs): (DevScreen, Vec) = messages.into_iter().enumerate().fold( + (screen(columns, start), Vec::new()), + |(screen, mut outputs), (index, message)| { + let now: Instant = start + Duration::from_millis(100 * (index as u64 + 1)); + let (screen, output) = + apply_screen_message(screen, message, now, &format!("09:41:{:02}", index % 60)); + outputs.push(output); + (screen, outputs) + }, + ); + Transcript { + rows: replay(outputs.iter().map(|output| output.text.as_str())), + actions: outputs.iter().map(|output| output.action).collect(), + } +} + +/// The rows a terminal shows after `writes`; `ESC[nF ESC[J` erases the last `n` rows. +fn replay<'a>(writes: impl Iterator) -> Vec { + let text: String = writes + .flat_map(|write| write.split_inclusive(CLEAR_BELOW)) + .fold(String::new(), |shown, piece| { + let clears: bool = piece.ends_with(CLEAR_BELOW); + let piece: &str = piece.trim_end_matches(CLEAR_BELOW); + let (printed, up): (&str, usize) = piece + .rsplit_once(CURSOR_UP) + .filter(|_| clears) + .map_or((piece, 0), |(printed, rows)| { + (printed, rows.trim_end_matches('F').parse().unwrap_or(0)) + }); + let shown: String = shown + printed; + let keep: usize = shown.split_inclusive('\n').count().saturating_sub(up); + shown.split_inclusive('\n').take(keep).collect() + }); + text.split_inclusive('\n') + .map(|row| row.trim_end_matches('\n').to_owned()) + .collect() +} + +/// `count` log lines from `log line `, as the open log shows them. +pub(super) fn log_rows(first: usize, count: usize) -> Vec { + (first..first + count) + .map(|index| format!(" log line {index}")) + .collect() +} + +/// Every row that is a log line. +pub(super) fn shown_log(rows: &[String]) -> Vec { + rows.iter() + .filter(|row| row.starts_with(" log line ")) + .cloned() + .collect() +} diff --git a/crates/sqlbuild-cli/src/dev_screen/tests/test_apply_screen_message.rs b/crates/sqlbuild-cli/src/dev_screen/tests/test_apply_screen_message.rs new file mode 100644 index 0000000000..e7fe423e19 --- /dev/null +++ b/crates/sqlbuild-cli/src/dev_screen/tests/test_apply_screen_message.rs @@ -0,0 +1,212 @@ +//! The `sqb dev` screen: checklist, framed sign-in, ready card, event lines, plain mode and keys. + +use sqlbuild_core::text::main::cell_len::cell_len; + +use crate::dev_screen::constants::LOG_RING_LINES; +use crate::dev_screen::models::{ScreenAction, ScreenMessage}; +use crate::dev_screen::tests::helpers::{ + after_startup, card, checklist, connected, edits, framed_sign_in, lines_of, log_rows, + not_connected, play, shown_log, startup, +}; +use crate::dev_screen::tests::test_types::{KeyTestCase, ScreenTestCase}; + +const HOTKEY_BAR: &str = + " o open UI \u{b7} r recompile \u{b7} s retry warehouse \u{b7} l logs \u{b7} q quit"; + +#[test] +fn given_a_session_when_each_message_is_applied_then_the_terminal_shows_stable_uncut_rows() { + let edited: Vec = [ + card("\u{25cf} duckdb \u{b7} main", "1.7s"), + lines_of([ + "", + " 09:41:17 ~ models/orders/int_orders.sql", + " 09:41:19 \u{2713} gen 2 published 1 file changed 0.1s", + " 09:41:20 ~ models/finance/fct_revenue.sql", + " 09:41:20 ~ 2 more files", + " 09:41:22 \u{2717} gen 3 failed \u{b7} still serving gen 2", + " models/finance/fct_revenue.sql:42:7 unknown column net_amount_usd", + " r retry \u{b7} l full log", + " 09:41:23 \u{26a0} warehouse sync stopped", + " 5 failed reads in a row: ProgrammingError: insufficient privileges", + "", + HOTKEY_BAR, + ]), + ] + .concat(); + let rule: String = "\u{2500}".repeat(78); + let test_cases = [ + ScreenTestCase { + description: "starting at 80 columns cuts the long control store path with an ellipsis", + columns: Some(80), + messages: startup(), + expected_rows: checklist( + "sqlite \u{b7} /home/analyst/projects/shop/.sqlbuil\u{2026}", + 46, + ), + }, + ScreenTestCase { + description: "starting at 120 columns shows the whole path", + columns: Some(120), + messages: startup(), + expected_rows: checklist( + "sqlite \u{b7} /home/analyst/projects/shop/.sqlbuild/control.sqlite3", + 62, + ), + }, + ScreenTestCase { + description: "at 80 columns the checklist collapses into the card and events print above the hotkeys", + columns: Some(80), + messages: after_startup([vec![connected()], edits()].concat()), + expected_rows: edited.clone(), + }, + ScreenTestCase { + description: "at 120 columns event lines keep their columns", + columns: Some(120), + messages: after_startup([vec![connected()], edits()].concat()), + expected_rows: edited, + }, + ScreenTestCase { + description: "an unavailable warehouse says not connected and how to retry", + columns: Some(80), + messages: after_startup(vec![not_connected()]), + expected_rows: [ + card( + "\u{25cb} not connected \u{b7} sign-in skipped \u{b7} press s to retry", + "1.7s", + ), + lines_of(["", HOTKEY_BAR]), + ] + .concat(), + }, + ScreenTestCase { + description: "a client prompting for sign-in is framed verbatim and ticks never redraw over it", + columns: Some(80), + messages: after_startup(framed_sign_in()), + expected_rows: [ + checklist( + "sqlite \u{b7} /home/analyst/projects/shop/.sqlbuil\u{2026}", + 46, + )[..8] + .to_vec(), + lines_of([ + " \u{280b} Warehouse duckdb \u{b7} connecting 0.1s", + "", + ]), + vec![format!( + " \u{2500}\u{2500} warehouse {}", + "\u{2500}".repeat(65) + )], + lines_of([ + "Initiating login request with your identity provider...", + "Enter the URL the OAuth flow redirected you to: ", + ]), + vec![format!(" {rule}"), String::new()], + card("\u{25cf} duckdb \u{b7} main", "2.2s"), + lines_of(["", HOTKEY_BAR]), + ] + .concat(), + }, + ScreenTestCase { + description: "plain output is stable unstyled lines that ignore ticks and keys", + columns: None, + messages: after_startup( + [ + vec![ScreenMessage::Tick, ScreenMessage::Key('q'), not_connected()], + edits(), + ] + .concat(), + ), + expected_rows: lines_of([ + "sqlbuild dev: control store ready (0.1s)", + "sqlbuild dev: location 01a1d7ff-0b3a-7631-ba35-e44d44d072f2 \u{b7} target dev", + "sqlbuild dev: discovered 412 python nodes, 2280 models (0.3s)", + "sqlbuild dev: published gen 1 b429729b (0.1s)", + "sqlbuild dev: warehouse not connected: sign-in skipped", + "sqlbuild dev: ready http://127.0.0.1:8000", + "sqlbuild dev: changed models/orders/int_orders.sql", + "sqlbuild dev: published gen 2 c01d2e3f (0.1s)", + "sqlbuild dev: changed models/finance/fct_revenue.sql", + "sqlbuild dev: changed 2 more files", + "sqlbuild dev: gen 3 failed \u{b7} still serving gen 2: models/finance/fct_revenue.sql:42:7 unknown column net_amount_usd", + "sqlbuild dev: warehouse sync stopped: 5 failed reads in a row: ProgrammingError: insufficient privileges", + ]), + }, + ]; + + for test_case in test_cases { + let transcript = play(test_case.columns, test_case.messages); + + assert_eq!( + transcript.rows, test_case.expected_rows, + "{}", + test_case.description + ); + assert!( + transcript + .rows + .iter() + .all(|row| cell_len(row) <= test_case.columns.unwrap_or(usize::MAX)), + "{}", + test_case.description + ); + } +} + +#[test] +fn given_keys_after_ready_when_pressed_then_they_ask_for_actions_and_l_shows_the_newest_log_lines() +{ + let flood: Vec = (0..=LOG_RING_LINES) + .map(|index| ScreenMessage::Log(format!("log line {index}"))) + .collect(); + let test_cases = [ + KeyTestCase { + description: "l opens the ring's newest lines, x does nothing, l closes, then o r s q act", + columns: Some(80), + messages: after_startup([vec![connected()], flood.clone()].concat()), + keys: "lxlorsq", + expected_actions: vec![ + ScreenAction::None, + ScreenAction::None, + ScreenAction::None, + ScreenAction::OpenUi, + ScreenAction::Recompile, + ScreenAction::RetryWarehouse, + ScreenAction::Quit, + ], + expected_log: log_rows(1, LOG_RING_LINES), + }, + KeyTestCase { + description: "plain output has no hotkeys", + columns: None, + messages: after_startup([vec![connected()], flood].concat()), + keys: "lq", + expected_actions: vec![ScreenAction::None, ScreenAction::None], + expected_log: Vec::new(), + }, + ]; + + for test_case in test_cases { + let keys: usize = test_case.keys.len(); + let transcript = play( + test_case.columns, + [ + test_case.messages, + test_case.keys.chars().map(ScreenMessage::Key).collect(), + ] + .concat(), + ); + + assert_eq!( + transcript.actions[transcript.actions.len() - keys..], + test_case.expected_actions, + "{}", + test_case.description + ); + assert_eq!( + shown_log(&transcript.rows), + test_case.expected_log, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-cli/src/dev_screen/tests/test_types.rs b/crates/sqlbuild-cli/src/dev_screen/tests/test_types.rs new file mode 100644 index 0000000000..818fa620b9 --- /dev/null +++ b/crates/sqlbuild-cli/src/dev_screen/tests/test_types.rs @@ -0,0 +1,27 @@ +//! Table rows for the `sqb dev` screen. + +use crate::dev_screen::models::{ScreenAction, ScreenMessage}; + +/// What a terminal shows after a session, and the action each message asked for. +pub(super) struct Transcript { + pub(super) rows: Vec, + pub(super) actions: Vec, +} + +/// A session at `columns` (`None` for plain output) and every row it leaves on the terminal. +pub(super) struct ScreenTestCase { + pub(super) description: &'static str, + pub(super) columns: Option, + pub(super) messages: Vec, + pub(super) expected_rows: Vec, +} + +/// A session ending in keys: the actions the keys ask for and the log lines left on screen. +pub(super) struct KeyTestCase { + pub(super) description: &'static str, + pub(super) columns: Option, + pub(super) messages: Vec, + pub(super) keys: &'static str, + pub(super) expected_actions: Vec, + pub(super) expected_log: Vec, +} diff --git a/crates/sqlbuild-cli/src/dev_screen/types.rs b/crates/sqlbuild-cli/src/dev_screen/types.rs new file mode 100644 index 0000000000..d9650b1926 --- /dev/null +++ b/crates/sqlbuild-cli/src/dev_screen/types.rs @@ -0,0 +1,14 @@ +//! The parts of a running screen that other roles hold. + +use std::sync::Arc; + +use tokio::sync::mpsc::{UnboundedReceiver, UnboundedSender}; + +use crate::dev_screen::models::{KeyGate, ScreenAction, ScreenMessage}; + +/// Where to send messages, the actions keys ask for, and the hotkey gate. +pub(crate) type DevScreenParts = ( + UnboundedSender, + UnboundedReceiver, + Option>, +); diff --git a/crates/sqlbuild-cli/src/dev_session/_helpers/child.rs b/crates/sqlbuild-cli/src/dev_session/_helpers/child.rs new file mode 100644 index 0000000000..96b33ce753 --- /dev/null +++ b/crates/sqlbuild-cli/src/dev_session/_helpers/child.rs @@ -0,0 +1,44 @@ +//! The location child's pipes: its log lines, its publication records and the control lines sent to it. + +use std::io::{IsTerminal, Write}; + +use sqlbuild_orchestration::dev_events::main::dev_control_line::dev_control_line; +use sqlbuild_orchestration::dev_events::models::DevControl; +use sqlbuild_orchestration::store::errors::ControlError; +use tokio::io::{AsyncBufReadExt, AsyncWriteExt, BufReader}; +use tokio::process::ChildStdin; +use tokio::sync::mpsc::UnboundedSender; + +use crate::dev_screen::models::ScreenMessage; + +/// The child's log lines, kept for `l`. +pub(crate) async fn relay_logs( + stderr: BufReader, + screen: UnboundedSender, +) { + let mut lines = stderr.lines(); + while let Ok(Some(line)) = lines.next_line().await { + drop(screen.send(ScreenMessage::Log(line))); + } +} + +/// Redirected stdout keeps one publication record per line, as `sqb location serve` prints them. +pub(crate) fn publish_record(record: &serde_json::Value) { + if std::io::stdout().is_terminal() { + return; + } + let mut stdout = std::io::stdout().lock(); + drop(writeln!(stdout, "{record}").and_then(|()| stdout.flush())); +} + +/// Send one control line to the location child. +pub(crate) async fn send_control( + stdin: &mut ChildStdin, + control: DevControl, +) -> Result<(), ControlError> { + stdin + .write_all(dev_control_line(control).as_bytes()) + .await?; + stdin.flush().await?; + Ok(()) +} diff --git a/crates/sqlbuild-cli/src/dev_session/_helpers/connection.rs b/crates/sqlbuild-cli/src/dev_session/_helpers/connection.rs new file mode 100644 index 0000000000..32794a331f --- /dev/null +++ b/crates/sqlbuild-cli/src/dev_session/_helpers/connection.rs @@ -0,0 +1,93 @@ +//! The foreground `sqb debug` connection: its own lifecycle rows apart from the client's output, and its verdict. + +use serde_json::Value; +use sqlbuild_core::screen_text::models::WarehouseState; + +use crate::dev_session::constants::{ + ADAPTER_LABEL, CONNECTION_TEST_LABEL, ERROR_STATUS, LIFECYCLE_STATUSES, NO_REPORT, OK_STATUS, + REPORT_ERROR_SECTIONS, +}; + +/// SQLBuild's own progress row, such as `Project discovery OK (0.02s)`, never the client's. +pub(crate) fn is_lifecycle_row(line: &str) -> bool { + let mut columns = line.split(" "); + columns.next().is_some_and(|label| !label.trim().is_empty()) + && columns + .next() + .is_some_and(|status| LIFECYCLE_STATUSES.contains(&status)) +} + +/// `chunk` split into the client's text, kept as written, and SQLBuild's own complete rows. +pub(crate) fn split_debug_output(chunk: &str) -> (String, Vec) { + let mut client: String = String::new(); + let mut rows: Vec = Vec::new(); + for line in chunk.split_inclusive('\n') { + let row: &str = line.trim_end_matches(['\n', '\r']); + if line.ends_with('\n') && is_lifecycle_row(row) { + rows.push(row.to_owned()); + } else { + client.push_str(line); + } + } + (client, rows) +} + +/// The rows of one report section. +fn section(report: &Value, name: &str) -> Vec { + report[name].as_array().cloned().unwrap_or_default() +} + +/// One text field of a report row, empty when absent. +fn field(row: &Value, name: &str) -> String { + row[name].as_str().unwrap_or_default().to_owned() +} + +/// The adapter and connection state a `sqb debug --json` report gives, or a failure without one. +pub(crate) fn connection_outcome(report: &[u8]) -> (String, WarehouseState) { + let report: Value = match serde_json::from_slice(report) { + Ok(report) => report, + Err(_) => { + return ( + String::new(), + WarehouseState::Failed { + message: NO_REPORT.to_owned(), + }, + ); + } + }; + let adapter: String = section(&report, "configuration") + .iter() + .find(|row| row["label"] == ADAPTER_LABEL) + .map(|row| field(row, "message")) + .unwrap_or_default(); + let connection: Vec = section(&report, "connection"); + let test: Option = connection + .iter() + .find(|row| row["label"] == CONNECTION_TEST_LABEL) + .cloned(); + if test.as_ref().is_some_and(|row| row["status"] == OK_STATUS) { + let detail: Vec = connection + .iter() + .filter(|row| row["status"].is_null()) + .map(|row| field(row, "message")) + .filter(|message| !message.is_empty()) + .collect(); + return ( + adapter, + WarehouseState::Connected { + detail: detail.join(" \u{b7} "), + }, + ); + } + let mut candidates: Vec = test.into_iter().collect(); + for name in REPORT_ERROR_SECTIONS { + candidates.extend(section(&report, name)); + } + let message: String = candidates + .iter() + .find(|row| row["status"] == ERROR_STATUS) + .map(|row| field(row, "status_message")) + .filter(|message| !message.is_empty()) + .unwrap_or_else(|| NO_REPORT.to_owned()); + (adapter, WarehouseState::Failed { message }) +} diff --git a/crates/sqlbuild-cli/src/dev_session/_helpers/foreground.rs b/crates/sqlbuild-cli/src/dev_session/_helpers/foreground.rs new file mode 100644 index 0000000000..471cbaf0ee --- /dev/null +++ b/crates/sqlbuild-cli/src/dev_session/_helpers/foreground.rs @@ -0,0 +1,96 @@ +//! Open the dev target's first connection in the foreground, attached to the terminal. + +use std::process::Stdio; + +use sqlbuild_core::screen_text::models::WarehouseState; +use sqlbuild_orchestration::dev_events::constants::DEV_EVENTS_VARIABLE; +use tokio::io::AsyncReadExt; +use tokio::process::Command; +use tokio::sync::mpsc::UnboundedSender; + +use crate::dev_screen::models::{DevStep, KeyGate, ScreenMessage}; +use crate::dev_session::_helpers::connection::{connection_outcome, split_debug_output}; +use crate::dev_session::constants::{CONNECTING, DEBUG_DRAIN, NO_REPORT}; +use crate::dev_session::models::DevProject; + +/// Run `sqb debug --json` with the terminal's input, framing the client's own output; true when it connected. +pub(crate) async fn connect_in_foreground( + screen: &UnboundedSender, + keys: Option<&KeyGate>, + dev: &DevProject, +) -> bool { + drop(screen.send(ScreenMessage::Started { + step: DevStep::Warehouse, + detail: CONNECTING.to_owned(), + })); + if let Some(keys) = keys { + keys.close().await; + } + let (adapter, state): (String, WarehouseState) = match run_debug(screen, dev).await { + Ok(report) => connection_outcome(&report), + Err(error) => ( + String::new(), + WarehouseState::Failed { + message: format!("{NO_REPORT}: {error}"), + }, + ), + }; + if let Some(keys) = keys { + keys.open(); + } + let connected: bool = matches!(state, WarehouseState::Connected { .. }); + drop(screen.send(ScreenMessage::Warehouse { adapter, state })); + connected +} + +/// The report on stdout and stderr streamed to the screen, until `sqb debug` exits even if a browser it opened keeps the pipes. +async fn run_debug( + screen: &UnboundedSender, + dev: &DevProject, +) -> std::io::Result> { + let mut child = Command::new(&dev.executable) + .arg("--project-dir") + .arg(&dev.project) + .args(["debug", "--target", &dev.target, "--json"]) + .env_remove(DEV_EVENTS_VARIABLE) + .stdin(Stdio::inherit()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .kill_on_drop(true) + .spawn()?; + let (Some(mut stdout), Some(mut stderr)) = (child.stdout.take(), child.stderr.take()) else { + return Err(std::io::Error::other("sqb debug output is unavailable")); + }; + let mut report: Vec = Vec::new(); + let relay = async { + let mut chunk: [u8; 4096] = [0; 4096]; + loop { + let read: usize = stderr.read(&mut chunk).await?; + if read == 0 { + return Ok::<(), std::io::Error>(()); + } + let (client, rows) = split_debug_output(&String::from_utf8_lossy(&chunk[..read])); + if !client.is_empty() { + drop(screen.send(ScreenMessage::ClientOutput(client))); + } + for row in rows { + drop(screen.send(ScreenMessage::Log(row))); + } + } + }; + { + let reading = async { tokio::try_join!(relay, stdout.read_to_end(&mut report)) }; + tokio::pin!(reading); + tokio::select! { + read = &mut reading => { read?; } + exited = child.wait() => { + exited?; + if let Ok(read) = tokio::time::timeout(DEBUG_DRAIN, &mut reading).await { + read?; + } + } + } + } + child.wait().await?; + Ok(report) +} diff --git a/crates/sqlbuild-cli/src/dev_session/_helpers/mod.rs b/crates/sqlbuild-cli/src/dev_session/_helpers/mod.rs new file mode 100644 index 0000000000..db0d5d467c --- /dev/null +++ b/crates/sqlbuild-cli/src/dev_session/_helpers/mod.rs @@ -0,0 +1,5 @@ +pub(crate) mod child; +pub(crate) mod connection; +pub(crate) mod foreground; +pub(crate) mod runs; +pub(crate) mod triggers; diff --git a/crates/sqlbuild-cli/src/dev_session/_helpers/runs.rs b/crates/sqlbuild-cli/src/dev_session/_helpers/runs.rs new file mode 100644 index 0000000000..2b4d02b731 --- /dev/null +++ b/crates/sqlbuild-cli/src/dev_session/_helpers/runs.rs @@ -0,0 +1,59 @@ +//! Event lines for runs: one when a run starts, one when it ends. + +use std::collections::HashMap; +use std::time::Duration; + +use sqlbuild_orchestration::runs::models::{RunRecord, RunStatus}; +use uuid::Uuid; + +use crate::dev_screen::models::ScreenMessage; + +/// The runs seen so far and their last status. +pub(crate) type SeenRuns = HashMap; + +/// Messages for runs that started or ended since `seen`, and the runs now seen; the first read only records. +pub(crate) fn run_messages( + seen: Option, + runs: &[RunRecord], +) -> (Vec, SeenRuns) { + let now: SeenRuns = runs.iter().map(|run| (run.run_id, run.status)).collect(); + let Some(seen) = seen else { + return (Vec::new(), now); + }; + let mut messages: Vec = Vec::new(); + for run in runs.iter().rev() { + let before: Option = seen.get(&run.run_id).copied(); + if before.is_none() { + messages.push(ScreenMessage::Run { + name: run_name(run), + detail: format!("started from {}", run.origin.as_str()), + finished: None, + }); + } + if run.status.is_terminal() && before.is_none_or(|status| !status.is_terminal()) { + messages.push(ScreenMessage::Run { + name: run_name(run), + detail: format!("{} nodes", run.node_counts_json.values().sum::()), + finished: Some(( + matches!(run.status, RunStatus::Success | RunStatus::Warn), + run_elapsed(run), + )), + }); + } + } + (messages, now) +} + +fn run_name(run: &RunRecord) -> String { + run.job_key + .clone() + .or_else(|| run.trigger_name.clone()) + .unwrap_or_else(|| run.kind.as_str().to_owned()) +} + +fn run_elapsed(run: &RunRecord) -> Duration { + match (run.started_at, run.ended_at) { + (Some(started), Some(ended)) => (ended - started).to_std().unwrap_or_default(), + _ => Duration::ZERO, + } +} diff --git a/crates/sqlbuild-cli/src/dev_session/_helpers/triggers.rs b/crates/sqlbuild-cli/src/dev_session/_helpers/triggers.rs new file mode 100644 index 0000000000..40e60245e9 --- /dev/null +++ b/crates/sqlbuild-cli/src/dev_session/_helpers/triggers.rs @@ -0,0 +1,18 @@ +//! The ready card's trigger summary. + +use sqlbuild_orchestration::declarations::models::{TriggerRecord, TriggerStatus}; + +/// `41 paused`, `3 running · 38 paused` or `none`. +pub(crate) fn trigger_summary(triggers: &[TriggerRecord]) -> String { + let paused: usize = triggers + .iter() + .filter(|trigger| trigger.status == TriggerStatus::Paused) + .count(); + let running: usize = triggers.len() - paused; + match (running, paused) { + (0, 0) => "none".to_owned(), + (0, paused) => format!("{paused} paused"), + (running, 0) => format!("{running} running"), + (running, paused) => format!("{running} running \u{b7} {paused} paused"), + } +} diff --git a/crates/sqlbuild-cli/src/dev_session/constants.rs b/crates/sqlbuild-cli/src/dev_session/constants.rs new file mode 100644 index 0000000000..33d092f88a --- /dev/null +++ b/crates/sqlbuild-cli/src/dev_session/constants.rs @@ -0,0 +1,26 @@ +//! The foreground warehouse connection, the location child's pipes and the run feed of `sqb dev`. + +use std::time::Duration; + +/// The statuses of SQLBuild's own progress rows, which the framed block leaves out. +pub(crate) const LIFECYCLE_STATUSES: [&str; 3] = ["START", "OK", "FAIL"]; +/// The `sqb debug` row that says whether the warehouse answered. +pub(crate) const CONNECTION_TEST_LABEL: &str = "connection test"; +/// The warehouse state when `sqb debug` printed no report. +pub(crate) const NO_REPORT: &str = "sqb debug did not report a connection; see l for its log"; +/// The `sqb debug` configuration row that names the adapter. +pub(crate) const ADAPTER_LABEL: &str = "adapter"; +/// A `sqb debug` row that passed. +pub(crate) const OK_STATUS: &str = "OK"; +/// A `sqb debug` row that failed. +pub(crate) const ERROR_STATUS: &str = "ERROR"; +/// Report sections searched for the first error when the connection test did not pass. +pub(crate) const REPORT_ERROR_SECTIONS: [&str; 2] = ["configuration", "providers"]; +/// How often finished and newly started runs are read for event lines. +pub(crate) const RUN_POLL: Duration = Duration::from_secs(2); +/// The newest runs read per poll. +pub(crate) const RUN_PAGE: u32 = 20; +/// The warehouse row's detail while the foreground connection runs. +pub(crate) const CONNECTING: &str = "connecting"; +/// How long the foreground `sqb debug`'s pipes are drained after it exits. +pub(crate) const DEBUG_DRAIN: Duration = Duration::from_secs(2); diff --git a/crates/sqlbuild-cli/src/dev_session/main/mod.rs b/crates/sqlbuild-cli/src/dev_session/main/mod.rs new file mode 100644 index 0000000000..44c0d4bad9 --- /dev/null +++ b/crates/sqlbuild-cli/src/dev_session/main/mod.rs @@ -0,0 +1 @@ +pub(crate) mod run_dev_session; diff --git a/crates/sqlbuild-cli/src/dev_session/main/run_dev_session.rs b/crates/sqlbuild-cli/src/dev_session/main/run_dev_session.rs new file mode 100644 index 0000000000..e75cddea98 --- /dev/null +++ b/crates/sqlbuild-cli/src/dev_session/main/run_dev_session.rs @@ -0,0 +1,93 @@ +//! Feed the `sqb dev` screen from the location child, open the warehouse in the foreground and act on hotkeys. + +use sqlbuild_orchestration::declarations::main::list_triggers::list_triggers; +use sqlbuild_orchestration::dev_events::main::parse_dev_event::parse_dev_event; +use sqlbuild_orchestration::dev_events::models::{DevControl, DevEvent}; +use sqlbuild_orchestration::runs::main::list_runs::list_runs; +use sqlbuild_orchestration::store::errors::ControlError; +use sqlbuild_orchestration::store::models::ControlStore; +use tokio::io::{AsyncBufReadExt, BufReader}; +use tokio::sync::watch; +use uuid::Uuid; + +use crate::dev_screen::models::{ScreenAction, ScreenMessage}; +use crate::dev_screen::types::DevScreenParts; +use crate::dev_session::_helpers::child::{publish_record, relay_logs, send_control}; +use crate::dev_session::_helpers::foreground::connect_in_foreground; +use crate::dev_session::_helpers::runs::{SeenRuns, run_messages}; +use crate::dev_session::_helpers::triggers::trigger_summary; +use crate::dev_session::constants::{RUN_PAGE, RUN_POLL}; +use crate::dev_session::models::{DevProject, LocationPipes}; +use crate::person_login::main::open_browser::open_browser; + +/// Run until `q`, shutdown or the location child closing its output. +pub(crate) async fn run_dev_session( + (screen, mut actions, keys): DevScreenParts, + (pipes, dev): (LocationPipes, DevProject), + (store, location_id, url): (ControlStore, Uuid, String), + mut shutdown: watch::Receiver, +) -> Result<(), ControlError> { + let LocationPipes { + mut stdin, + stdout, + stderr, + } = pipes; + tokio::spawn(relay_logs(BufReader::new(stderr), screen.clone())); + let mut events = BufReader::new(stdout).lines(); + let mut runs: Option = None; + let mut connected: Option = None; + let mut poll = tokio::time::interval(RUN_POLL); + loop { + tokio::select! { + changed = shutdown.changed() => { + if changed.is_err() || *shutdown.borrow() { + return Ok(()); + } + } + line = events.next_line() => { + let Some(line) = line? else { + return Ok(()); + }; + let Some(event) = parse_dev_event(&line) else { + drop(screen.send(ScreenMessage::Log(line))); + continue; + }; + if let DevEvent::Published { record, .. } = &event { + publish_record(record); + drop(screen.send(ScreenMessage::Triggers( + trigger_summary(&list_triggers(&store, location_id, &dev.target).await?), + ))); + } + let first: bool = matches!(event, DevEvent::Published { .. }) && connected.is_none(); + drop(screen.send(ScreenMessage::Child(event))); + if first { + let verdict: bool = connect_in_foreground(&screen, keys.as_deref(), &dev).await; + connected = Some(verdict); + send_control(&mut stdin, DevControl::Warehouse { connected: verdict }).await?; + } + } + action = actions.recv() => match action { + Some(ScreenAction::Quit) | None => return Ok(()), + Some(ScreenAction::OpenUi) => drop(open_browser(&url)), + Some(ScreenAction::Recompile) => send_control(&mut stdin, DevControl::Recompile).await?, + Some(ScreenAction::RetryWarehouse) => { + let verdict: bool = connect_in_foreground(&screen, keys.as_deref(), &dev).await; + connected = Some(verdict); + send_control(&mut stdin, DevControl::Warehouse { connected: verdict }).await?; + } + Some(ScreenAction::None) => {} + }, + _ = poll.tick(), if connected.is_some() => { + let records = list_runs(&store, (location_id, &dev.target), RUN_PAGE).await?; + let messages; + (messages, runs) = { + let (messages, seen) = run_messages(runs, &records); + (messages, Some(seen)) + }; + for message in messages { + drop(screen.send(message)); + } + } + } + } +} diff --git a/crates/sqlbuild-cli/src/dev_session/mod.rs b/crates/sqlbuild-cli/src/dev_session/mod.rs new file mode 100644 index 0000000000..d031795eef --- /dev/null +++ b/crates/sqlbuild-cli/src/dev_session/mod.rs @@ -0,0 +1,9 @@ +//! `sqb dev`'s foreground: the screen fed by the location child, the warehouse sign-in, runs and hotkeys. + +pub(crate) mod _helpers; +pub(crate) mod constants; +pub(crate) mod main; +pub(crate) mod models; + +#[cfg(test)] +mod tests; diff --git a/crates/sqlbuild-cli/src/dev_session/models.rs b/crates/sqlbuild-cli/src/dev_session/models.rs new file mode 100644 index 0000000000..e74b33fbf0 --- /dev/null +++ b/crates/sqlbuild-cli/src/dev_session/models.rs @@ -0,0 +1,21 @@ +//! What a `sqb dev` session owns: the location child's pipes and how to run `sqb` for the project. + +use std::path::PathBuf; + +use tokio::process::{ChildStderr, ChildStdin, ChildStdout}; + +/// The location child's control input, event output and log output. +#[derive(Debug)] +pub(crate) struct LocationPipes { + pub(crate) stdin: ChildStdin, + pub(crate) stdout: ChildStdout, + pub(crate) stderr: ChildStderr, +} + +/// The `sqb` executable, project and target the foreground connection check runs with. +#[derive(Clone, Debug)] +pub(crate) struct DevProject { + pub(crate) executable: PathBuf, + pub(crate) project: PathBuf, + pub(crate) target: String, +} diff --git a/crates/sqlbuild-cli/src/dev_session/tests.rs b/crates/sqlbuild-cli/src/dev_session/tests.rs new file mode 100644 index 0000000000..473cecc95d --- /dev/null +++ b/crates/sqlbuild-cli/src/dev_session/tests.rs @@ -0,0 +1,8 @@ +#[path = "tests/test_connection.rs"] +mod connection; +#[path = "tests/helpers.rs"] +mod helpers; +#[path = "tests/test_run_messages.rs"] +mod run_messages; +#[path = "tests/test_types.rs"] +mod test_types; diff --git a/crates/sqlbuild-cli/src/dev_session/tests/helpers.rs b/crates/sqlbuild-cli/src/dev_session/tests/helpers.rs new file mode 100644 index 0000000000..0adebb6187 --- /dev/null +++ b/crates/sqlbuild-cli/src/dev_session/tests/helpers.rs @@ -0,0 +1,103 @@ +//! Builders for `sqb debug` reports and stored runs, and plain text for what the session sends. + +use serde_json::{Value, json}; +use sqlbuild_core::screen_text::models::WarehouseState; +use sqlbuild_orchestration::runs::models::{RunRecord, RunStatus}; +use uuid::Uuid; + +use crate::dev_screen::models::ScreenMessage; +use crate::dev_session::_helpers::runs::SeenRuns; + +/// A `sqb debug --json` report with these configuration and connection rows. +pub(super) fn report(configuration: Value, connection: Value) -> String { + json!({ + "configuration": configuration, + "connection": connection, + "providers": [{"label": "providers", "message": "none discovered", "status": "OK"}], + }) + .to_string() +} + +/// One report row as `sqb debug --json` writes it; `status` is null for detail rows. +pub(super) fn row(label: &str, status: Option<&str>, message: &str, status_message: &str) -> Value { + json!({"label": label, "status": status, "message": message, "status_message": status_message}) +} + +/// The state as `Connected { detail: "..." }`, `Failed { message: "..." }` and so on. +pub(super) fn state_text(state: &WarehouseState) -> String { + format!("{state:?}") +} + +/// `name | detail | running`, or `name | detail | ok 90s` / `failed 90s` once finished. +pub(super) fn message_text(message: &ScreenMessage) -> String { + let ScreenMessage::Run { + name, + detail, + finished, + } = message + else { + return format!("{message:?}"); + }; + let outcome: String = finished.map_or_else( + || "running".to_owned(), + |(ok, elapsed)| { + format!( + "{} {}s", + ["failed", "ok"][usize::from(ok)], + elapsed.as_secs() + ) + }, + ); + format!("{name} | {detail} | {outcome}") +} + +/// The run with number `id`, named by `job_key` or `trigger_name`, that ran `seconds` with `nodes` nodes. +pub(super) fn run( + id: u128, + (status, origin): (&str, &str), + (job_key, trigger_name): (Option<&str>, Option<&str>), + (seconds, nodes): (i64, i64), +) -> RunRecord { + let started = chrono::DateTime::parse_from_rfc3339("2026-11-15T09:40:00Z") + .expect("start time") + .to_utc(); + serde_json::from_value(json!({ + "run_id": Uuid::from_u128(id), + "location_id": Uuid::from_u128(900), + "target": "dev", + "snapshot_id": Uuid::from_u128(901), + "job_key": job_key, + "kind": "adhoc", + "origin": origin, + "parent_run_id": null, + "root_run_id": Uuid::from_u128(id), + "trigger_id": null, + "trigger_name": trigger_name, + "tick_id": null, + "rerun_of_run_id": null, + "requested_by": "system:scheduler", + "requester_name": null, + "priority": 0, + "rewrite_history": false, + "status": status, + "status_reason": null, + "queued_at": started, + "launched_at": started, + "started_at": started, + "ended_at": started + chrono::Duration::seconds(seconds), + "cancel_requested_at": null, + "selection_json": [], + "node_counts_json": {"success": nodes - 1, "skipped": 1}, + "last_event_storage_id": 0, + })) + .expect("run record") +} + +/// Runs already seen, by number and status. +pub(super) fn seen(runs: &[(u128, RunStatus)]) -> Option { + Some( + runs.iter() + .map(|(id, status)| (Uuid::from_u128(*id), *status)) + .collect(), + ) +} diff --git a/crates/sqlbuild-cli/src/dev_session/tests/test_connection.rs b/crates/sqlbuild-cli/src/dev_session/tests/test_connection.rs new file mode 100644 index 0000000000..d1ee910ff8 --- /dev/null +++ b/crates/sqlbuild-cli/src/dev_session/tests/test_connection.rs @@ -0,0 +1,123 @@ +//! The foreground `sqb debug` connection: its verdict from the report, and its stderr split from the client's. + +use serde_json::json; + +use crate::dev_session::_helpers::connection::{connection_outcome, split_debug_output}; +use crate::dev_session::constants::NO_REPORT; +use crate::dev_session::tests::helpers::{report, row, state_text}; +use crate::dev_session::tests::test_types::{OutcomeTestCase, SplitTestCase}; + +#[test] +fn given_a_debug_report_when_read_then_the_adapter_and_warehouse_state_follow_its_rows() { + let configured = json!([ + row("project", Some("OK"), "shop", "loaded"), + row("adapter", Some("OK"), "duckdb", "found"), + ]); + let test_cases = [ + OutcomeTestCase { + description: "a passing connection test is connected, detailed by the status-less rows", + report: report( + configured.clone(), + json!([ + row("database", None, "/work/shop/warehouse.duckdb", ""), + row("connection test", Some("OK"), "", "connected"), + row("query test", Some("OK"), "", "SELECT 1"), + ]), + ), + expected_adapter: "duckdb", + expected_state: r#"Connected { detail: "/work/shop/warehouse.duckdb" }"#.to_owned(), + }, + OutcomeTestCase { + description: "a failing connection test reports its own message, not the skipped query test", + report: report( + configured, + json!([ + row("database", None, "/missing/warehouse.duckdb", ""), + row( + "connection test", + Some("ERROR"), + "", + "IO Error: Cannot open file" + ), + row("query test", Some("SKIP"), "", "connection failed"), + ]), + ), + expected_adapter: "duckdb", + expected_state: r#"Failed { message: "IO Error: Cannot open file" }"#.to_owned(), + }, + OutcomeTestCase { + description: "with no connection rows, the first configuration error explains the failure", + report: report( + json!([ + row("project", Some("OK"), "shop", "loaded"), + row( + "adapter", + Some("ERROR"), + "", + "adapter package is not installed" + ), + ]), + json!([]), + ), + expected_adapter: "", + expected_state: r#"Failed { message: "adapter package is not installed" }"#.to_owned(), + }, + OutcomeTestCase { + description: "output that is not a report, such as an unknown target, fails without one", + report: String::new(), + expected_adapter: "", + expected_state: format!("Failed {{ message: {NO_REPORT:?} }}"), + }, + ]; + + for test_case in test_cases { + let (adapter, state) = connection_outcome(test_case.report.as_bytes()); + + assert_eq!( + adapter, test_case.expected_adapter, + "{}", + test_case.description + ); + assert_eq!( + state_text(&state), + test_case.expected_state, + "{}", + test_case.description + ); + } +} + +#[test] +fn given_debug_stderr_when_split_then_only_complete_sqlbuild_rows_leave_the_client_text() { + let test_cases = [ + SplitTestCase { + description: "a prompt without a newline stays with the client, after SQLBuild's own row", + chunk: "Project discovery OK (0.02s)\nEnter passcode: ", + expected_client: "Enter passcode: ", + expected_rows: vec!["Project discovery OK (0.02s)"], + }, + SplitTestCase { + description: "client lines that only mention a status, or start blank, are kept as written", + chunk: "Opening the browser for OK sign-in\n OK indented\r\nConnection test FAIL (1.2s)\n", + expected_client: "Opening the browser for OK sign-in\n OK indented\r\n", + expected_rows: vec!["Connection test FAIL (1.2s)"], + }, + SplitTestCase { + description: "a line cut off by the end of the chunk passes through as client text", + chunk: "Connection test START", + expected_client: "Connection test START", + expected_rows: Vec::new(), + }, + ]; + + for test_case in test_cases { + let (client, rows) = split_debug_output(test_case.chunk); + + assert_eq!( + client, test_case.expected_client, + "{}", + test_case.description + ); + assert_eq!(rows, test_case.expected_rows, "{}", test_case.description); + } +} diff --git a/crates/sqlbuild-cli/src/dev_session/tests/test_run_messages.rs b/crates/sqlbuild-cli/src/dev_session/tests/test_run_messages.rs new file mode 100644 index 0000000000..ebb23fb906 --- /dev/null +++ b/crates/sqlbuild-cli/src/dev_session/tests/test_run_messages.rs @@ -0,0 +1,81 @@ +//! Run event lines: one when a run starts, one when it ends, oldest first, never repeated. + +use sqlbuild_orchestration::runs::models::RunStatus; + +use crate::dev_session::_helpers::runs::run_messages; +use crate::dev_session::tests::helpers::{message_text, run, seen}; +use crate::dev_session::tests::test_types::RunPollTestCase; + +#[test] +fn given_runs_since_the_last_poll_when_read_then_each_start_and_end_is_reported_once() { + let test_cases = [ + RunPollTestCase { + description: "the first read only records the runs that already exist", + seen: None, + runs: vec![ + run( + 2, + ("running", "schedule"), + (Some("orders_daily"), None), + (5, 3), + ), + run(1, ("success", "manual"), (Some("customers"), None), (40, 2)), + ], + expected_messages: Vec::new(), + }, + RunPollTestCase { + description: "new runs start oldest first; a run that started and ended between polls reports both", + seen: seen(&[]), + runs: vec![ + run( + 3, + ("running", "schedule"), + (Some("orders_daily"), None), + (5, 3), + ), + run( + 2, + ("failed", "sensor"), + (None, Some("new_orders_file")), + (12, 4), + ), + ], + expected_messages: vec![ + "new_orders_file | started from sensor | running".to_owned(), + "new_orders_file | 4 nodes | failed 12s".to_owned(), + "orders_daily | started from schedule | running".to_owned(), + ], + }, + RunPollTestCase { + description: "a seen run reports only its end, named by its kind when it has no job or trigger", + seen: seen(&[(3, RunStatus::Running), (4, RunStatus::Running)]), + runs: vec![ + run(4, ("warn", "manual"), (None, None), (90, 5)), + run( + 3, + ("running", "schedule"), + (Some("orders_daily"), None), + (5, 3), + ), + ], + expected_messages: vec!["adhoc | 5 nodes | ok 90s".to_owned()], + }, + RunPollTestCase { + description: "a run already seen ending is not reported again", + seen: seen(&[(4, RunStatus::Warn)]), + runs: vec![run(4, ("warn", "manual"), (None, None), (90, 5))], + expected_messages: Vec::new(), + }, + ]; + + for test_case in test_cases { + let (messages, _) = run_messages(test_case.seen, &test_case.runs); + + assert_eq!( + messages.iter().map(message_text).collect::>(), + test_case.expected_messages, + "{}", + test_case.description + ); + } +} diff --git a/crates/sqlbuild-cli/src/dev_session/tests/test_types.rs b/crates/sqlbuild-cli/src/dev_session/tests/test_types.rs new file mode 100644 index 0000000000..5aa27342f8 --- /dev/null +++ b/crates/sqlbuild-cli/src/dev_session/tests/test_types.rs @@ -0,0 +1,29 @@ +//! Table rows for the `sqb dev` foreground session. + +use sqlbuild_orchestration::runs::models::RunRecord; + +use crate::dev_session::_helpers::runs::SeenRuns; + +/// A `sqb debug --json` report and the adapter and warehouse state read from it. +pub(super) struct OutcomeTestCase { + pub(super) description: &'static str, + pub(super) report: String, + pub(super) expected_adapter: &'static str, + pub(super) expected_state: String, +} + +/// A stderr chunk from `sqb debug` and how it splits into client text and SQLBuild rows. +pub(super) struct SplitTestCase { + pub(super) description: &'static str, + pub(super) chunk: &'static str, + pub(super) expected_client: &'static str, + pub(super) expected_rows: Vec<&'static str>, +} + +/// One run poll: what was seen before, the runs read newest first, and the event lines it yields. +pub(super) struct RunPollTestCase { + pub(super) description: &'static str, + pub(super) seen: Option, + pub(super) runs: Vec, + pub(super) expected_messages: Vec, +} diff --git a/crates/sqlbuild-cli/src/lib.rs b/crates/sqlbuild-cli/src/lib.rs index e8494fd932..781d162aa3 100644 --- a/crates/sqlbuild-cli/src/lib.rs +++ b/crates/sqlbuild-cli/src/lib.rs @@ -6,6 +6,8 @@ pub mod adhoc; pub mod argv; pub mod compute_logs; pub mod cost_store; +pub(crate) mod dev_screen; +pub(crate) mod dev_session; pub mod diff_report; pub mod dispatch; pub mod launcher; diff --git a/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs b/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs index 77ff315cda..556186ab90 100644 --- a/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs +++ b/crates/sqlbuild-cli/src/native_commands/_helpers/execution.rs @@ -1,11 +1,25 @@ //! Running a Python-free command: its checks, then its output with the observed services. -use std::collections::HashMap; +use crate::dev_screen::models::ScreenSetup; +use crate::native_commands::_helpers::paths::exclude_location_databases; +use crate::native_commands::_helpers::project_facts::{project_directory_name, project_name}; +use crate::native_commands::_helpers::terminal::{live_screen, stderr_columns}; +use crate::native_commands::constants::SCREEN_INDENT; +use crate::native_commands::types::{CompileOutcome, NextGeneration, Settled}; +use sqlbuild_core::cli_text::models::CliStyle; +use sqlbuild_core::screen_text::models::{DevHeader, ScreenFacts}; +use sqlbuild_orchestration::credentials::main::identities::development_person::development_person; +use sqlbuild_orchestration::credentials::main::locations::development_location::development_location; +use sqlbuild_orchestration::credentials::models::AuditContext; +use sqlbuild_orchestration::declarations::main::pause_development_triggers::pause_development_triggers; +use std::collections::{BTreeMap, HashMap}; use std::future::Future; use std::io; +use std::net::SocketAddr; use std::path::{Path, PathBuf}; use std::process::Stdio; use std::time::{Duration, SystemTime}; +use tokio::sync::mpsc::UnboundedSender; use chrono::Utc; use serde_json::json; @@ -14,6 +28,7 @@ use sqlbuild_orchestration::api::errors::ClientError; use sqlbuild_orchestration::api::main::serve_api::serve_api; use crate::adhoc::main::adhoc_planner::adhoc_planner; +use sqlbuild_cache::project_snapshot::models::PathStamp; use sqlbuild_config::project::main::load_control_config::load_control_config; use sqlbuild_config::project::main::project_config_paths::project_config_paths; use sqlbuild_config::project::main::sync_interval_seconds::sync_interval_seconds; @@ -21,6 +36,13 @@ use sqlbuild_config::project::models::ControlConfig; use sqlbuild_orchestration::api::models::{ ApiAuthentication, DevelopmentScope, HumanLogin, ServerFacts, }; +use sqlbuild_orchestration::dev_events::constants::{DEV_EVENTS_ON, DEV_EVENTS_VARIABLE}; +use sqlbuild_orchestration::dev_events::main::dev_event_line::dev_event_line; +use sqlbuild_orchestration::dev_events::main::dev_events_enabled::dev_events_enabled; +use sqlbuild_orchestration::dev_events::main::emit_dev_event::emit_dev_event; +use sqlbuild_orchestration::dev_events::main::parse_dev_control::parse_dev_control; +use sqlbuild_orchestration::dev_events::main::parse_dev_event::parse_dev_event; +use sqlbuild_orchestration::dev_events::models::{DevControl, DevEvent}; use sqlbuild_orchestration::platform::constants::PLATFORM_WEBHOOK_VARIABLE; use sqlbuild_orchestration::platform::main::report_remote_location_health::report_remote_location_health; use sqlbuild_orchestration::platform::main::run_maintenance::run_maintenance; @@ -39,8 +61,9 @@ use sqlbuild_orchestration::store::main::set_release_version::set_release_versio use sqlbuild_orchestration::store::models::ControlStore; use sqlbuild_orchestration::warehouse_metadata::constants::MAX_HISTORY_RECORDS; use sqlbuild_orchestration::warehouse_metadata::models::{ - MetadataBatch, MetadataScan, StatisticsDueRead, StatisticsRead, WarehouseHistoryKind, - WarehouseHistoryPage, WarehouseStateLocation, WarehouseSyncDueRead, WarehouseWatermarkRead, + MetadataBatch, MetadataScan, StatisticsDueRead, StatisticsRead, WarehouseHistoryFailure, + WarehouseHistoryKind, WarehouseHistoryPage, WarehouseStateLocation, WarehouseSyncDueRead, + WarehouseWatermarkRead, }; use sqlbuild_orchestration::webhooks::constants::SECURITY_AUDIT_WEBHOOK_VARIABLE; use sqlbuild_orchestration::webhooks::main::{ @@ -61,8 +84,12 @@ use crate::compute_logs::main::finish_capture::finish_capture; use crate::compute_logs::main::open_log_root::open_log_root; use crate::compute_logs::main::start_capture::start_capture; use crate::compute_logs::models::{CaptureEnd, CaptureStart, OpenCapture}; +use crate::dev_screen::models::{DevStep, ScreenMessage}; +use crate::dev_screen::types::DevScreenParts; +use crate::dev_session::main::run_dev_session::run_dev_session; +use crate::dev_session::models::{DevProject, LocationPipes}; use crate::native_commands::_helpers::decision::{ - control_database, control_database_variable, text, + control_address, control_database, control_database_variable, flag, text, }; use crate::native_commands::_helpers::init::run_init; use crate::native_commands::_helpers::package::installed_package; @@ -74,17 +101,18 @@ use crate::native_commands::_helpers::terminal::{ finish_lost_stdout, terminal_columns, write_output, }; use crate::native_commands::constants::{ - ADMIN_COMMAND, COLUMNS_VARIABLE, FALLBACK_COLUMNS, METADATA_OVERLAP_SECONDS, - METADATA_POLL_SECONDS, METADATA_READER_CLOSE_SECONDS, METADATA_RECONCILE_SECONDS, - METADATA_REPLY_BYTES, METADATA_SAFETY_SYNC_SECONDS, METADATA_SYNC_CHECK_SECONDS, - SAFETY_SYNC_ENV, SCHEDULER_COMMAND, UNCAUGHT_EXCEPTION_STATUS, UNKNOWN_COMMAND, + ADMIN_COMMAND, CHANGED_PATHS_SHOWN, COLUMNS_VARIABLE, FALLBACK_COLUMNS, METADATA_ERROR_CHARS, + METADATA_FAILURE_LIMIT, METADATA_OVERLAP_SECONDS, METADATA_POLL_SECONDS, + METADATA_READER_CLOSE_SECONDS, METADATA_RECONCILE_SECONDS, METADATA_REPLY_BYTES, + METADATA_SAFETY_SYNC_SECONDS, METADATA_SYNC_CHECK_SECONDS, SAFETY_SYNC_ENV, SCHEDULER_COMMAND, + UNCAUGHT_EXCEPTION_STATUS, UNKNOWN_COMMAND, }; use crate::native_commands::errors::PythonNeeded; use crate::native_commands::main::_run_admin::run_admin; use crate::native_commands::main::_run_control::run_control; use crate::native_commands::main::_run_cost::run_cost; use crate::native_commands::models::{ - CommandOutput, InstalledPackage, LocationCompileRequest, LocationCompiler, + CommandOutput, DevControlState, InstalledPackage, LocationCompileRequest, LocationCompiler, LocationMetadataPage, LocationMetadataSync, LocationServe, LocationWatch, LostStdout, MetadataReply, NativeCommand, NativeInvocation, ProjectLocation, ScanToken, }; @@ -96,18 +124,17 @@ use crate::workers::main::open_workers::open_workers; use crate::workers::main::terminate_child::terminate_child; use crate::workers::models::{Launchers, Workers}; -/// Own the API, scheduler and outbound location as one development process tree. +/// Own the API, scheduler and outbound location as one development process tree, drawn by `screen`. pub(crate) async fn run_development_roles( (store, scope): (ControlStore, DevelopmentScope), (project, executable): (PathBuf, PathBuf), listener: tokio::net::TcpListener, - scheduler: bool, + (scheduler, (messages, actions, keys)): (bool, DevScreenParts), ) -> Result<(), ControlError> { let address = listener.local_addr()?; let url = format!("http://{address}"); let location_id = scope.location_id; let target = scope.target.clone(); - let triggers = scope.triggers; let workers = open_development_workers( (location_id, target.clone()), project.clone(), @@ -116,36 +143,62 @@ pub(crate) async fn run_development_roles( )?; let (stop, shutdown) = watch::channel(false); let mut tasks = JoinSet::new(); + drop(messages.send(ScreenMessage::Started { + step: DevStep::Api, + detail: String::new(), + })); + let api_ready = messages.clone(); tasks.spawn(serve_api( store.clone(), listener, - (ApiAuthentication { - login: HumanLogin::Development(scope), - ..ApiAuthentication::default() - }, (Vec::new(), adhoc_planner()), ServerFacts { - version: env!("CARGO_PKG_VERSION").into(), - scheduler, - ..ServerFacts::default() - }), - (move || { - drop(write_output(&[(Stream::Stderr, &format!( - "Development API ready: {address}\nDevelopment location: {location_id}\nAuthentication disabled on loopback; triggers {}\n", - if triggers { "follow code defaults and operator overrides" } else { "start paused" } - ))])); - }, development_stopped(shutdown.clone())), + ( + ApiAuthentication { + login: HumanLogin::Development(scope), + ..ApiAuthentication::default() + }, + (Vec::new(), adhoc_planner()), + ServerFacts { + version: env!("CARGO_PKG_VERSION").into(), + scheduler, + ..ServerFacts::default() + }, + ), + ( + move || { + drop(api_ready.send(ScreenMessage::Finished { + step: DevStep::Api, + detail: address.to_string(), + })); + }, + development_stopped(shutdown.clone()), + ), )); start_platform_roles(&mut tasks, (&store, &shutdown), Default::default())?; - tasks.spawn(development_location_role( + let (location, pipes) = development_location_role( (&project, &executable), (location_id, &target), (&url, &store), shutdown.clone(), - )?); + )?; + tasks.spawn(location); + tasks.spawn(run_dev_session( + (messages.clone(), actions, keys), + ( + pipes, + DevProject { + executable: executable.clone(), + project: project.clone(), + target: target.clone(), + }, + ), + (store.clone(), location_id, url), + shutdown.clone(), + )); + drop(messages.send(ScreenMessage::Started { + step: DevStep::Scheduler, + detail: String::new(), + })); if scheduler { - drop(write_output(&[( - Stream::Stderr, - "Starting development scheduler...\n", - )])); let scheduler_store = store.clone(); tasks.spawn(async move { run_scheduler( @@ -156,11 +209,11 @@ pub(crate) async fn run_development_roles( ) .await }); - drop(write_output(&[( - Stream::Stderr, - "Development scheduler started\n", - )])); } + drop(messages.send(ScreenMessage::Finished { + step: DevStep::Scheduler, + detail: if scheduler { "started" } else { "off" }.to_owned(), + })); supervise_control_roles(tasks, stop).await } @@ -178,7 +231,13 @@ fn development_location_role( (location_id, target): (Uuid, &str), (url, store): (&str, &ControlStore), shutdown: watch::Receiver, -) -> Result> + Send + 'static, ControlError> { +) -> Result< + ( + impl Future> + Send + 'static, + LocationPipes, + ), + ControlError, +> { let mut command = Command::new(executable); command .args(["--project-dir"]) @@ -199,7 +258,10 @@ fn development_location_role( .env_remove(SECURITY_AUDIT_WEBHOOK_VARIABLE) .env_remove("SQLBUILD_RUN_TOKEN") .env_remove("SQLBUILD_DEVELOPMENT_CONTROL_FILE") - .stdin(Stdio::null()) + .env(DEV_EVENTS_VARIABLE, DEV_EVENTS_ON) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) .kill_on_drop(true); if let ControlStore::Sqlite(path) = store { command.env( @@ -208,31 +270,46 @@ fn development_location_role( ); } let mut child = command.spawn()?; - Ok(async move { - tokio::select! { - result = child.wait() => { - result?; - Err(ControlError::Invalid("development code location stopped unexpectedly".into())) - }, - () = development_stopped(shutdown) => { - terminate_child(&mut child)?; - match tokio::time::timeout(Duration::from_secs(10), child.wait()).await { - Ok(result) => { result?; }, - Err(_) => child.kill().await?, - } - Ok(()) - }, - } - }) + let (Some(stdin), Some(stdout), Some(stderr)) = + (child.stdin.take(), child.stdout.take(), child.stderr.take()) + else { + return Err(ControlError::Invalid( + "development code location pipes are unavailable".into(), + )); + }; + let pipes = LocationPipes { + stdin, + stdout, + stderr, + }; + Ok(( + async move { + tokio::select! { + result = child.wait() => { + result?; + Err(ControlError::Invalid("development code location stopped unexpectedly".into())) + }, + () = development_stopped(shutdown) => { + terminate_child(&mut child)?; + match tokio::time::timeout(Duration::from_secs(10), child.wait()).await { + Ok(result) => { result?; }, + Err(_) => child.kill().await?, + } + Ok(()) + }, + } + }, + pipes, + )) } /// Request one publication without transferring interpreter ownership to a cancellable future. pub(crate) async fn compile_generation( - requests: &mpsc::Sender, + (requests, dev_events): (&mpsc::Sender, bool), generation: i64, sources: &[sqlbuild_cache::project_snapshot::models::PathStamp], stopped: impl Future>, -) -> Result, Vec)>, ControlError> { +) -> Result, ControlError> { let (reply, publication) = oneshot::channel(); requests .send(LocationCompileRequest { @@ -260,6 +337,16 @@ pub(crate) async fn compile_generation( "Location compiler failed; generation {generation} may already be published: {error}\n" ), )])); + if dev_events { + drop(write_output(&[( + Stream::Stdout, + &dev_event_line(&DevEvent::Failed { + generation, + position: None, + message: error, + }), + )])); + } None } }; @@ -270,9 +357,20 @@ pub(crate) async fn compile_generation( refresh_at = refresh .map(|deadline| Instant::now() + (deadline - Utc::now()).to_std().unwrap_or_default()); locations = state_locations; - let output = serde_json::to_string(&record) - .map_err(|error| ControlError::Invalid(error.to_string()))?; - if write_output(&[(Stream::Stdout, &format!("{output}\n"))]).is_some() { + let output: String = if dev_events { + dev_event_line(&DevEvent::Published { + generation, + snapshot_id: record.snapshot_id.to_string(), + record: serde_json::to_value(&record) + .map_err(|error| ControlError::Invalid(error.to_string()))?, + }) + } else { + let mut output: String = serde_json::to_string(&record) + .map_err(|error| ControlError::Invalid(error.to_string()))?; + output.push('\n'); + output + }; + if write_output(&[(Stream::Stdout, &output)]).is_some() { return Err(ControlError::Invalid(format!( "Snapshot {} was published, but result delivery failed", record.snapshot_id @@ -293,10 +391,17 @@ pub(crate) async fn exchange_location_generation( .await?; compiler.stdin.flush().await?; let mut line = String::new(); - if compiler.stdout.read_line(&mut line).await? == 0 { - return Err(ControlError::Invalid( - "location interpreter exited before replying".into(), - )); + loop { + line.clear(); + if compiler.stdout.read_line(&mut line).await? == 0 { + return Err(ControlError::Invalid( + "location interpreter exited before replying".into(), + )); + } + if parse_dev_event(&line).is_none() { + break; + } + drop(write_output(&[(Stream::Stdout, &line)])); } serde_json::from_str(&line).map_err(|_| { ControlError::Invalid("location interpreter returned an invalid publication record".into()) @@ -361,7 +466,8 @@ pub(crate) fn metadata_sync( cycle_started_at: None, safety_seconds, safety_at: Instant::now() + Duration::from_secs(safety_seconds), - busy: false, + failed_reads: 0, + halted: false, change_tokens: HashMap::new(), scan_token: None, scan_delivered: false, @@ -465,6 +571,10 @@ pub(crate) fn freeze_history_page( Stream::Stderr, "Warehouse history row exceeds the request bound; synchronization incomplete, no stale rows removed.\n", )])); + location_warning( + "warehouse sync incomplete", + "a history row exceeds the request bound".to_owned(), + ); } } else { state.pending = Some(LocationMetadataPage { @@ -559,7 +669,7 @@ pub(crate) async fn load_watermark( Err(error) => { tokio::select! { result = &mut stopped => { result?; return Ok(None); }, - () = metadata_health(&state, (client, poll), (started, Some(WarehouseSyncFailure::Watermark))) => {}, + () = metadata_health(&state, (client, poll), (started, Some(WarehouseSyncFailure::Watermark)), None) => {}, } return Ok(Some((state, Some(error)))); } @@ -599,11 +709,34 @@ pub(crate) fn next_scan(mut state: LocationMetadataSync) -> LocationMetadataSync state } +/// Under `sqb dev`, a warning the location logs also becomes a screen line, so plain output shows it. +pub(crate) fn location_warning(title: &str, detail: String) { + drop(emit_dev_event(&DevEvent::Warning { + title: title.to_owned(), + detail, + })); +} + +/// A sync stopped by failed reads makes one attempt when its safety sync is due, and says so. +pub(crate) fn retry_stopped_sync(state: &mut LocationMetadataSync) { + if state.resume_after_safety_wait() { + drop(write_output(&[( + Stream::Stderr, + "Retrying the stopped warehouse history synchronization at its safety sync.\n", + )])); + location_warning( + "warehouse sync retrying", + "one attempt after the safety interval".to_owned(), + ); + } +} + /// Operational cost is elapsed IO time, not an estimate of warehouse billing. pub(crate) async fn metadata_health( state: &LocationMetadataSync, (client, poll): (&LocationWorkClient, &LocationWorkPoll), (started, failure): (Instant, Option), + error: Option<&str>, ) { let report = LocationHealthReport { event_id: Uuid::now_v7(), @@ -619,6 +752,7 @@ pub(crate) async fn metadata_health( .as_ref() .map_or(0, |page| page.batch.records.len()) as u64, elapsed_millis: started.elapsed().as_millis() as u64, + error: error.map(|error| error.chars().take(METADATA_ERROR_CHARS).collect()), }, }; if let Err(error) = client.report_health(&report).await { @@ -731,6 +865,7 @@ pub(crate) fn location_options(namespace: &Namespace) -> Result if let Some(image) = &options.image_id { command.args(["--image-id", image]); } + if options.dev_events { + command.env(DEV_EVENTS_VARIABLE, DEV_EVENTS_ON); + } command.stdin(Stdio::piped()).stdout(Stdio::piped()).spawn() } pub(crate) async fn check_published_location( - options: &LocationServe, - script: &Path, + (options, script, connect): (&LocationServe, &Path, bool), (bearer, session_id, generation): (&str, Uuid, i64), stopped: impl Future>, ) -> Result>, ControlError> { @@ -813,7 +950,7 @@ pub(crate) async fn check_published_location( )])); tokio::pin!(stopped); let Some((connected, missing, files)) = - check_location_connection(options, script, stopped.as_mut()).await? + check_location_connection((options, script, connect), stopped.as_mut()).await? else { return Ok(None); }; @@ -832,6 +969,10 @@ pub(crate) async fn check_published_location( "{missing} provider secret(s) are unset in this location's environment; runs that use them fail. Run sqb debug for the list.\n" ), )])); + location_warning( + "provider secrets unset", + format!("{missing} unset; runs that use them fail; sqb debug lists them"), + ); } drop(write_output(&[( Stream::Stderr, @@ -844,16 +985,17 @@ pub(crate) async fn check_published_location( Ok(Some(files)) } -/// Capture the ordinary debug check's exact local file paths and missing provider secret count without contaminating publication stdout. +/// Capture the debug check's local file paths and missing secret count; under `sqb dev` it never connects, so nobody signs in twice. pub(crate) async fn check_location_connection( - options: &LocationServe, - script: &Path, + (options, script, connect): (&LocationServe, &Path, bool), stopped: impl Future>, ) -> Result)>, ControlError> { - let mut child = location_child(options, script) - .args(["debug", "--target", &options.target, "--json"]) - .stdout(Stdio::piped()) - .spawn()?; + let mut command = location_child(options, script); + command.args(["debug", "--target", &options.target, "--json"]); + if !connect || options.dev_events { + command.arg("--no-connection"); + } + let mut child = command.stdout(Stdio::piped()).spawn()?; let mut stdout = child .stdout .take() @@ -887,7 +1029,7 @@ pub(crate) async fn check_location_connection( } }; Ok(Some(( - connected, + connect && connected, missing, location_database_files(&options.project_dir, files), ))) @@ -900,6 +1042,7 @@ pub(crate) fn location_child(options: &LocationServe, script: &Path) -> Command command.env(COLUMNS_VARIABLE, FALLBACK_COLUMNS.to_string()); } command + .env_remove(DEV_EVENTS_VARIABLE) .stdin(Stdio::null()) .stderr(Stdio::inherit()) .kill_on_drop(true); @@ -1209,13 +1352,12 @@ pub(crate) async fn stop_location_runtime( Ok(()) } -/// An outer absence means shutdown; an inner absence is a failed read, never an empty table. +/// An absence means shutdown; a failed read never stands for an empty table. pub(crate) async fn read_metadata_page( (options, script): (&LocationServe, &Path), mut state: LocationMetadataSync, stopped: impl Future>, -) -> Result)>, ControlError> { - state.busy = false; +) -> Result, ControlError> { if let Some(statistics) = &state.statistics && statistics.names.is_empty() { @@ -1225,7 +1367,7 @@ pub(crate) async fn read_metadata_page( change_token: None, unchanged: false, }; - return Ok(Some((state, Some(page)))); + return Ok(Some((state, MetadataReply::Page(page)))); } let request = history_request(&state)?; let reader = match state.reader.take() { @@ -1233,52 +1375,111 @@ pub(crate) async fn read_metadata_page( None => match open_location_runtime(options, script, Vec::new()) { Ok(reader) => reader, Err(error) => { - drop(write_output(&[( - Stream::Stderr, - &format!("Warehouse metadata reader did not start: {error}\n"), - )])); - return Ok(Some((state, None))); + let reason = format!("Warehouse metadata reader did not start: {error}"); + drop(write_output(&[(Stream::Stderr, &format!("{reason}\n"))])); + let failure = WarehouseHistoryFailure { + reason, + page_size: false, + }; + return Ok(Some((state, MetadataReply::Failed(failure)))); } }, }; tokio::pin!(stopped); - tokio::select! { + let reply = tokio::select! { result = &mut stopped => { result?; - Ok(None) + return Ok(None); }, result = tokio::time::timeout( Duration::from_secs(METADATA_POLL_SECONDS), exchange_metadata_read(reader, request), ) => match result { - Ok((reader, Ok(MetadataReply::Page(page)))) => { - state.reader = Some(reader); - Ok(Some((state, page))) - } - Ok((reader, Ok(MetadataReply::Busy))) => { - state.busy = true; + Ok((reader, Ok(reply))) => { state.reader = Some(reader); - Ok(Some((state, None))) + reply } - Ok((reader, Err(error))) => { + Ok((reader, Err(failure))) => { drop(write_output(&[( Stream::Stderr, - &format!("Warehouse metadata reader failed: {error}\n"), + &format!("Warehouse metadata reader failed: {}\n", failure.reason), )])); stop_location_runtime(Some(reader)).await?; - Ok(Some((state, None))) + MetadataReply::Failed(failure) } Err(_) => { - drop(write_output(&[( - Stream::Stderr, - &format!( - "Warehouse metadata read did not answer within {METADATA_POLL_SECONDS}s.\n" - ), - )])); - Ok(Some((state, None))) + let reason = + format!("Warehouse metadata read did not answer within {METADATA_POLL_SECONDS}s."); + drop(write_output(&[(Stream::Stderr, &format!("{reason}\n"))])); + MetadataReply::Failed(WarehouseHistoryFailure { + reason, + page_size: true, + }) } }, + }; + if matches!(reply, MetadataReply::Page(_)) { + state.failed_reads = 0; } + Ok(Some((state, reply))) +} + +/// A warehouse file held by another process, such as a DuckDB build, waits a poll interval. +pub(crate) async fn wait_for_file( + mut state: LocationMetadataSync, + (client, poll): (&LocationWorkClient, &LocationWorkPoll), + started: Instant, + stopped: impl Future>, +) -> Result, ControlError> { + let failure = Some(WarehouseSyncFailure::Busy); + tokio::select! { + result = stopped => { result?; return Ok(None); }, + () = metadata_health(&state, (client, poll), (started, failure), None) => {}, + } + state.next_at = Instant::now() + Duration::from_secs(METADATA_POLL_SECONDS); + Ok(Some(state)) +} + +/// Report the failed read with its error, then retry it later or stop after failures in a row. +pub(crate) async fn failed_read( + mut state: LocationMetadataSync, + (client, poll): (&LocationWorkClient, &LocationWorkPoll), + (started, failure): (Instant, WarehouseHistoryFailure), + stopped: impl Future>, +) -> Result, ControlError> { + let read = Some(WarehouseSyncFailure::Read); + tokio::select! { + result = stopped => { result?; return Ok(None); }, + () = metadata_health(&state, (client, poll), (started, read), Some(&failure.reason)) => {}, + } + let message = match state.fail_read(failure.page_size) { + Some(wait) => { + location_warning( + "warehouse read failed", + format!("retrying in {}s: {}", wait.as_secs(), failure.reason), + ); + format!( + "Warehouse history read failed; retaining mirror and retrying in {}s.\n", + wait.as_secs() + ) + } + None => { + state = close_metadata_reader(state).await?; + location_warning( + "warehouse sync stopped", + format!( + "{METADATA_FAILURE_LIMIT} failed reads in a row: {}; it retries at the next safety sync, on request or after a publish", + failure.reason + ), + ); + format!( + "Warehouse history synchronization stopped after {METADATA_FAILURE_LIMIT} failed reads in a row: {}. It retries at the next safety sync, on request or after a publish.\n", + failure.reason + ) + } + }; + drop(write_output(&[(Stream::Stderr, &message)])); + Ok(Some(state)) } /// A completed synchronization closes its reader, so no warehouse session outlives the sync. @@ -1313,38 +1514,54 @@ pub(crate) async fn close_metadata_reader( Ok(state) } -/// A reply of `null` means the reader's read failed and it dropped its warehouse session. +/// A reply too long, or a reader gone and likely killed for memory, may fit a smaller page. async fn exchange_metadata_read( mut reader: LocationCompiler, request: Vec, -) -> (LocationCompiler, Result) { +) -> ( + LocationCompiler, + Result, +) { + let failed = |reason: &str, page_size| WarehouseHistoryFailure { + reason: reason.into(), + page_size, + }; let result = async { - reader.stdin.write_all(&request).await?; - reader.stdin.write_all(b"\n").await?; - reader.stdin.flush().await?; + let written: std::io::Result<()> = async { + reader.stdin.write_all(&request).await?; + reader.stdin.write_all(b"\n").await?; + reader.stdin.flush().await + } + .await; + written.map_err(|error| failed(&error.to_string(), false))?; let mut line = String::new(); let read = (&mut reader.stdout) .take(METADATA_REPLY_BYTES + 1) .read_line(&mut line) - .await?; + .await + .map_err(|error| failed(&error.to_string(), false))?; if read == 0 { - return Err(ControlError::Invalid( - "location metadata reader exited before replying".into(), + return Err(failed( + "location metadata reader exited before replying", + true, )); } if read as u64 > METADATA_REPLY_BYTES || !line.ends_with('\n') { - return Err(ControlError::Invalid( - "location reply exceeds its byte limit".into(), - )); + return Err(failed("location reply exceeds its byte limit", true)); } - let reply: serde_json::Value = serde_json::from_str(&line) - .map_err(|_| ControlError::Invalid("invalid location reply".into()))?; + let reply: serde_json::Value = + serde_json::from_str(&line).map_err(|_| failed("invalid location reply", false))?; if reply.get("busy") == Some(&serde_json::Value::Bool(true)) { return Ok(MetadataReply::Busy); } + if let Some(failure) = reply.get("failed") { + return serde_json::from_value(failure.clone()) + .map(MetadataReply::Failed) + .map_err(|_| failed("invalid location reply", false)); + } serde_json::from_value(reply) .map(MetadataReply::Page) - .map_err(|_| ControlError::Invalid("invalid location reply".into())) + .map_err(|_| failed("invalid location reply", false)) } .await; (reader, result) @@ -1368,9 +1585,304 @@ pub(crate) async fn next_turn( } } +/// `sqb dev`'s control lines from stdin; outside `sqb dev`, a fixed connected state. +pub(crate) fn dev_controls(options: &LocationServe) -> watch::Receiver { + let (sender, receiver) = watch::channel(DevControlState { + warehouse: (!options.dev_events).then_some(true), + recompiles: 0, + }); + if options.dev_events { + tokio::task::spawn_blocking(move || { + for line in std::io::stdin().lines().map_while(Result::ok) { + match parse_dev_control(&line) { + Some(DevControl::Warehouse { connected }) => { + sender.send_modify(|state| state.warehouse = Some(connected)); + } + Some(DevControl::Recompile) => { + sender.send_modify(|state| state.recompiles += 1); + } + None => {} + } + } + }); + } + receiver +} + +/// Tell `sqb dev` which inputs changed since the last compile: the first few paths and how many more. +pub(crate) fn report_changed_inputs(before: &[PathStamp], after: &[PathStamp]) { + let before: BTreeMap<&str, &PathStamp> = before + .iter() + .map(|stamp| (stamp.relative_path.as_str(), stamp)) + .collect(); + let after: BTreeMap<&str, &PathStamp> = after + .iter() + .map(|stamp| (stamp.relative_path.as_str(), stamp)) + .collect(); + let changed: Vec<&str> = after + .iter() + .filter(|(path, stamp)| before.get(*path) != Some(*stamp)) + .map(|(path, _)| *path) + .chain( + before + .keys() + .copied() + .filter(|path| !after.contains_key(path)), + ) + .collect(); + if changed.is_empty() { + return; + } + let shown: usize = changed.len().min(CHANGED_PATHS_SHOWN); + drop(write_output(&[( + Stream::Stdout, + &dev_event_line(&DevEvent::Changed { + paths: changed[..shown] + .iter() + .map(|path| (*path).to_owned()) + .collect(), + more: changed.len() - shown, + }), + )])); +} + /// The location's 250 ms turn; a slow turn skips missed ticks instead of bursting. pub(crate) fn location_clock() -> Interval { let mut clock = tokio::time::interval(Duration::from_millis(250)); clock.set_missed_tick_behavior(MissedTickBehavior::Skip); clock } + +/// The inputs this compile saw; under `sqb dev`, each one changed since the last compile is reported first. +pub(crate) fn note_compiled_inputs( + options: &LocationServe, + before: Option>, + observed: &[PathStamp], +) -> Option> { + if !options.dev_events { + return None; + } + if let Some(before) = &before { + report_changed_inputs(before, observed); + } + Some(observed.to_vec()) +} + +/// Sync a published generation's warehouse state, or defer it until the warehouse connects, then check the location. +pub(crate) async fn settle_publication( + (options, script, (bearer, session_id)): (&LocationServe, &Path, (&str, Uuid)), + (locations, generation, connect): (Vec, i64, bool), + (watch, metadata): (LocationWatch, Option), + stopped: impl Future>, +) -> Result, ControlError> { + let (metadata, deferred) = if connect { + let sync: LocationMetadataSync = metadata_sync( + (locations, generation), + ( + options.metadata_reconcile_seconds, + options.safety_sync_seconds, + ), + ); + (Some(sync), None) + } else { + (metadata, Some((locations, generation))) + }; + let Some(files) = check_published_location( + (options, script, connect), + (bearer, session_id, generation), + stopped, + ) + .await? + else { + return Ok(None); + }; + let watch: LocationWatch = exclude_location_databases(&options.project_dir, files, watch); + Ok(Some((watch, metadata, deferred))) +} + +/// Act on what `sqb dev` last asked: a new recompile retries now, and a connected warehouse settles the deferred sync. +pub(crate) async fn apply_dev_controls( + (options, script, session): (&LocationServe, &Path, (&str, Uuid)), + (control, recompiles): (DevControlState, u64), + (mut watch, metadata, deferred): Settled, + stopped: impl Future>, +) -> Result, ControlError> { + if control.recompiles != recompiles { + watch.retry_at = Some(Instant::now()); + } + let (Some(true), Some((locations, generation))) = (control.warehouse, deferred.clone()) else { + return Ok(Some((control.recompiles, (watch, metadata, deferred)))); + }; + let settled: Option = settle_publication( + (options, script, session), + (locations, generation, true), + (watch, metadata), + stopped, + ) + .await?; + Ok(settled.map(|settled| (control.recompiles, settled))) +} + +/// The loopback address `sqb dev` listens on. +pub(crate) fn loopback_address(namespace: &Namespace) -> Result { + let address = control_address(namespace, false, &ServerSection::default())? + .ok_or_else(|| ControlError::Invalid("development address is required".into()))?; + if !address.ip().is_loopback() { + return Err(ControlError::Invalid( + "sqb dev only listens on loopback".into(), + )); + } + Ok(address) +} + +/// Open the development store, with its checklist row and the location it names on the screen. +pub(crate) async fn announced_store( + namespace: &Namespace, + (project, target): (&Path, &str), + messages: &UnboundedSender, +) -> Result<(ControlStore, DevelopmentScope), ControlError> { + drop(messages.send(ScreenMessage::Started { + step: DevStep::ControlStore, + detail: String::new(), + })); + let database: String = text(namespace, "development_store_url").unwrap_or_else(|| { + project + .join(".sqlbuild/control.sqlite3") + .to_string_lossy() + .into_owned() + }); + let (store, scope) = development_store(namespace, (project, target), &database).await?; + drop(messages.send(ScreenMessage::Finished { + step: DevStep::ControlStore, + detail: database, + })); + drop(messages.send(ScreenMessage::Location(scope.location_id.to_string()))); + Ok((store, scope)) +} + +/// The header, width and paths the screen starts with. +pub(crate) fn screen_setup(project: &Path, target: &str) -> ScreenSetup { + let live: bool = live_screen(); + ScreenSetup { + live, + facts: ScreenFacts { + columns: stderr_columns(), + style: CliStyle { use_color: live }, + links: live, + indent: SCREEN_INDENT, + }, + header: DevHeader { + version: env!("CARGO_PKG_VERSION").to_owned(), + project: project_name(project), + target: target.to_owned(), + }, + watching: format!("{}/", project_directory_name(project)), + project_root: project.to_string_lossy().into_owned(), + } +} + +/// Complete retained identity and startup pauses before starting any scheduling roles. +pub(crate) async fn development_store( + namespace: &Namespace, + (project, target): (&Path, &str), + database: &str, +) -> Result<(ControlStore, DevelopmentScope), ControlError> { + set_release_version(env!("CARGO_PKG_VERSION")); + let store = open_store(database).await?; + migrate_store(&store).await?; + let context = AuditContext { + actor: "local-development".into(), + granting_role: "control-store-owner".into(), + request_id: Uuid::now_v7(), + ip_address: None, + }; + let location_id = + development_location(&store, &project.to_string_lossy(), target, &context).await?; + let scope = DevelopmentScope { + location_id, + target: target.into(), + triggers: flag(namespace, "development_triggers"), + principal_id: development_person(&store, &context).await?, + }; + if !scope.triggers { + pause_development_triggers(&store, (location_id, target)).await?; + } + Ok((store, scope)) +} + +/// After a compile: a publication waits for the `sqb dev` warehouse verdict and settles; a failure keeps the last good snapshot. +pub(crate) async fn finish_compile( + (options, script, session): (&LocationServe, &Path, (&str, Uuid)), + (compiled, generation): (CompileOutcome, i64), + (mut watch, metadata, deferred): Settled, + (controls, stopped): ( + &mut watch::Receiver, + impl Future>, + ), +) -> Result, ControlError> { + tokio::pin!(stopped); + let (published, refresh_at, locations) = compiled; + if !published { + watch.retry_at = Some(Instant::now() + Duration::from_secs(30)); + tokio::select! { + result = &mut stopped => { result?; return Ok(None); }, + () = report_location_health(options, session.0, session.1, LocationHealth::CompileFailed { generation }) => {}, + } + drop(write_output(&[( + Stream::Stderr, + "Location reload failed; retaining last good snapshot. Retrying on change or in 30 seconds.\n", + )])); + return Ok(Some((watch, metadata, deferred))); + } + watch.retry_at = refresh_at; + let connect: bool = tokio::select! { + result = &mut stopped => { result?; return Ok(None); }, + verdict = controls.wait_for(|control| control.warehouse.is_some()) => { + verdict.is_ok_and(|control| control.warehouse == Some(true)) + }, + }; + settle_publication( + (options, script, session), + (locations, generation, connect), + (watch, metadata), + stopped.as_mut(), + ) + .await +} + +/// Close the metadata reader, take the next generation number and compile it; `None` inside when no number was issued. +pub(crate) async fn compile_next_generation( + (options, bearer, requests): (&LocationServe, &str, &mpsc::Sender), + (metadata, compiled_inputs, observed): ( + Option, + Option>, + &[PathStamp], + ), + stopped: impl Future>, +) -> Result, ControlError> { + tokio::pin!(stopped); + let metadata: Option = match metadata { + Some(state) => Some(close_metadata_reader(state).await?), + None => None, + }; + let generation: Option = tokio::select! { + result = &mut stopped => { result?; return Ok(None); }, + generation = location_generation(options, bearer) => generation?, + }; + let Some(generation) = generation else { + return Ok(Some((None, metadata, compiled_inputs))); + }; + drop(write_output(&[( + Stream::Stderr, + &format!("Compiling location generation {generation} in an isolated interpreter...\n"), + )])); + let compiled_inputs = note_compiled_inputs(options, compiled_inputs, observed); + let compiled: Option = compile_generation( + (requests, options.dev_events), + generation, + observed, + stopped.as_mut(), + ) + .await?; + Ok(compiled.map(|compiled| (Some((compiled, generation)), metadata, compiled_inputs))) +} diff --git a/crates/sqlbuild-cli/src/native_commands/_helpers/project_facts.rs b/crates/sqlbuild-cli/src/native_commands/_helpers/project_facts.rs index 6d5d7546bd..6daecaa49d 100644 --- a/crates/sqlbuild-cli/src/native_commands/_helpers/project_facts.rs +++ b/crates/sqlbuild-cli/src/native_commands/_helpers/project_facts.rs @@ -2,6 +2,7 @@ use std::path::Path; +use sqlbuild_config::models::ConfigValue; use sqlbuild_config::project::main::load_project_config::load_project_config; use sqlbuild_config::project::main::project_config_paths::project_config_paths; use sqlbuild_config::project::models::{Built, ConfigPaths}; @@ -68,3 +69,24 @@ pub(crate) fn development_target( ControlError::Invalid("sqb dev requires --target or a configured default target".into()) }) } + +/// The project's configured name, or its directory's name when the configuration has none. +pub(crate) fn project_name(project_dir: &Path) -> String { + let paths = project_config_paths(project_dir); + let configured: Option = match load_project_config(&paths) { + Ok(config) => match object_field(&config, "name") { + Some(Built::Value(ConfigValue::String(name))) => Some(name.clone()), + _ => None, + }, + Err(_) => None, + }; + configured.unwrap_or_else(|| project_directory_name(project_dir)) +} + +/// The last component of the project directory. +pub(crate) fn project_directory_name(project_dir: &Path) -> String { + project_dir + .file_name() + .map(|name| name.to_string_lossy().into_owned()) + .unwrap_or_default() +} diff --git a/crates/sqlbuild-cli/src/native_commands/_helpers/terminal.rs b/crates/sqlbuild-cli/src/native_commands/_helpers/terminal.rs index 5476f82e40..b951cbc2cf 100644 --- a/crates/sqlbuild-cli/src/native_commands/_helpers/terminal.rs +++ b/crates/sqlbuild-cli/src/native_commands/_helpers/terminal.rs @@ -7,9 +7,9 @@ use sqlbuild_orchestration::store::errors::ControlError; use crate::argv::models::Stream; use crate::native_commands::_helpers::argv_host::python_int; use crate::native_commands::constants::{ - BROKEN_PIPE_EXCEPTION, COLUMNS_VARIABLE, EXIT_FLUSH_FAILURE, EXIT_FLUSH_FAILURE_STATUS, - FALLBACK_COLUMNS, NO_COLOR_VARIABLE, PASSWORD_INPUT_BYTES, PYTHON_STDOUT_BUFFER_SIZE, - UNCAUGHT_EXCEPTION_STATUS, + BROKEN_PIPE_EXCEPTION, CI_VARIABLE, COLUMNS_VARIABLE, EXIT_FLUSH_FAILURE, + EXIT_FLUSH_FAILURE_STATUS, FALLBACK_COLUMNS, NO_COLOR_VARIABLE, NO_PROGRESS_VARIABLE, + PASSWORD_INPUT_BYTES, PYTHON_STDOUT_BUFFER_SIZE, UNCAUGHT_EXCEPTION_STATUS, }; use crate::native_commands::errors::PythonNeeded; use crate::native_commands::models::LostStdout; @@ -197,3 +197,27 @@ pub(crate) fn finish_lost_stdout(lost: Option<&LostStdout>, exit_code: i32) -> i } } } + +/// Whether the `sqb dev` screen can redraw in place: on Unix, stderr is a terminal and nothing asks for plain lines. +pub(crate) fn live_screen() -> bool { + cfg!(unix) + && io::stderr().is_terminal() + && [NO_COLOR_VARIABLE, CI_VARIABLE, NO_PROGRESS_VARIABLE] + .iter() + .all(|variable| std::env::var_os(variable).is_none()) +} + +/// The width of the terminal on stderr, or the fallback when stderr is not one. +#[cfg(unix)] +pub(crate) fn stderr_columns() -> usize { + match rustix::termios::tcgetwinsize(io::stderr()) { + Ok(size) if size.ws_col > 0 => usize::from(size.ws_col), + _ => usize::try_from(FALLBACK_COLUMNS).unwrap_or(80), + } +} + +/// Without the console API, the fallback width. +#[cfg(not(unix))] +pub(crate) fn stderr_columns() -> usize { + usize::try_from(FALLBACK_COLUMNS).unwrap_or(80) +} diff --git a/crates/sqlbuild-cli/src/native_commands/constants.rs b/crates/sqlbuild-cli/src/native_commands/constants.rs index 6ca3f7a7b5..4362a47bc5 100644 --- a/crates/sqlbuild-cli/src/native_commands/constants.rs +++ b/crates/sqlbuild-cli/src/native_commands/constants.rs @@ -6,6 +6,10 @@ pub(crate) const POSITIVE_INTEGER_CONVERTER: &str = "_positive_integer"; pub(crate) const POSITIVE_INTEGER_MESSAGE: &str = "value must be a positive integer"; pub(crate) const NO_COLOR_OPTION: &str = "--no-color"; pub(crate) const NO_COLOR_VARIABLE: &str = "NO_COLOR"; +/// Set by CI systems; `sqb dev` then prints plain lines. +pub(crate) const CI_VARIABLE: &str = "CI"; +/// Turns live progress off; `sqb dev` then prints plain lines. +pub(crate) const NO_PROGRESS_VARIABLE: &str = "SQLBUILD_NO_PROGRESS"; pub(crate) const COLUMNS_VARIABLE: &str = "COLUMNS"; /// `shutil.get_terminal_size`'s fallback width. pub(crate) const FALLBACK_COLUMNS: i64 = 80; @@ -92,6 +96,10 @@ pub(crate) const SENSOR_EVALUATION_SECONDS: u64 = 30; pub(crate) const SENSOR_RESULT_BYTES: u64 = 1_048_576; /// Read timeout, and the delay before retrying a failed page of an unfinished synchronization. pub(crate) const METADATA_POLL_SECONDS: u64 = 30; +/// Failed reads in a row, waiting twice as long after each, that stop a sync until one is requested. +pub(crate) const METADATA_FAILURE_LIMIT: u32 = 5; +/// Characters of a failed read's error that its health report carries. +pub(crate) const METADATA_ERROR_CHARS: usize = 500; /// A metadata reader that ignores end of input this long after a synchronization is killed. pub(crate) const METADATA_READER_CLOSE_SECONDS: u64 = 5; /// How often a location asks git whether `HEAD` moved. @@ -116,3 +124,7 @@ pub(crate) const PLAN_COMMAND: &str = "plan"; /// How much `sqb plan --json` output a location reads; model diffs over the server's cap are dropped. pub(crate) const PLAN_OUTPUT_BYTES: usize = 8 * sqlbuild_orchestration::platform::constants::PLAN_DOCUMENT_BYTES; +/// `sqb dev` names at most this many changed inputs per change event. +pub(crate) const CHANGED_PATHS_SHOWN: usize = 3; +/// The spaces every `sqb dev` screen row starts with. +pub(crate) const SCREEN_INDENT: usize = 2; diff --git a/crates/sqlbuild-cli/src/native_commands/main/_location_compiler.rs b/crates/sqlbuild-cli/src/native_commands/main/_location_compiler.rs index e3035ec0e9..6a675a7bb2 100644 --- a/crates/sqlbuild-cli/src/native_commands/main/_location_compiler.rs +++ b/crates/sqlbuild-cli/src/native_commands/main/_location_compiler.rs @@ -2,11 +2,15 @@ use std::path::Path; +use sqlbuild_orchestration::dev_events::main::dev_event_line::dev_event_line; +use sqlbuild_orchestration::dev_events::models::DevEvent; use sqlbuild_orchestration::store::errors::ControlError; use tokio::sync::mpsc; +use crate::argv::models::Stream; use crate::native_commands::_helpers::execution::exchange_location_generation; use crate::native_commands::_helpers::execution::{open_location_runtime, stop_location_runtime}; +use crate::native_commands::_helpers::terminal::write_output; use crate::native_commands::models::{LocationCompileRequest, LocationCompiler, LocationServe}; pub(crate) async fn run_location_compiler( @@ -22,6 +26,14 @@ pub(crate) async fn run_location_compiler( .is_some_and(|compiler| compiler.python_sources != request.python_sources) { stop_location_runtime(compiler.take()).await?; + if options.dev_events { + drop(write_output(&[( + Stream::Stdout, + &dev_event_line(&DevEvent::Restarted { + reason: "Python changed".into(), + }), + )])); + } } if compiler.is_none() { compiler = Some(open_location_runtime( diff --git a/crates/sqlbuild-cli/src/native_commands/main/_location_metadata.rs b/crates/sqlbuild-cli/src/native_commands/main/_location_metadata.rs index 790d880b25..2494859dc4 100644 --- a/crates/sqlbuild-cli/src/native_commands/main/_location_metadata.rs +++ b/crates/sqlbuild-cli/src/native_commands/main/_location_metadata.rs @@ -17,11 +17,16 @@ use crate::native_commands::_helpers::execution::{ acknowledge_metadata_page, begin_after_run, freeze_history_page, load_watermark, metadata_health, next_scan, statistics_page, }; -use crate::native_commands::_helpers::execution::{finish_metadata_page, read_metadata_page}; +use crate::native_commands::_helpers::execution::{ + failed_read, finish_metadata_page, location_warning, read_metadata_page, retry_stopped_sync, + wait_for_file, +}; use crate::native_commands::_helpers::paths::location_files; use crate::native_commands::_helpers::terminal::write_output; use crate::native_commands::constants::METADATA_POLL_SECONDS; -use crate::native_commands::models::{LocationMetadataSync, LocationServe, LocationWatch}; +use crate::native_commands::models::{ + LocationMetadataSync, LocationServe, LocationWatch, MetadataReply, +}; /// One bounded source or mirror page leaves named work polling a turn between pages. pub(super) async fn sync_location_metadata( @@ -30,7 +35,9 @@ pub(super) async fn sync_location_metadata( (client, poll): (&LocationWorkClient, &LocationWorkPoll), stopped: impl Future>, ) -> Result, ControlError> { - if Instant::now() < state.next_at + retry_stopped_sync(&mut state); + if state.halted + || Instant::now() < state.next_at || state.locations.is_empty() || location_files(&options.project_dir, &watch.rules, &watch.local_files) != watch.observed.0 @@ -72,14 +79,21 @@ pub(super) async fn sync_location_metadata( return retry_metadata(state, error); } } - let Some((read, page)) = + let Some((read, reply)) = read_metadata_page((options, script), state, stopped.as_mut()).await? else { return Ok(None); }; state = read; - let Some(page) = page else { - return failed_read(state, (client, poll), started, stopped.as_mut()).await; + let page = match reply { + MetadataReply::Page(page) => page, + MetadataReply::Busy => { + return wait_for_file(state, (client, poll), started, stopped.as_mut()).await; + } + MetadataReply::Failed(failure) => { + return failed_read(state, (client, poll), (started, failure), stopped.as_mut()) + .await; + } }; if location_files(&options.project_dir, &watch.rules, &watch.local_files) != watch.observed.0 @@ -89,7 +103,7 @@ pub(super) async fn sync_location_metadata( if page.unchanged { tokio::select! { result = &mut stopped => { result?; return Ok(None); }, - () = metadata_health(&state, (client, poll), (started, None)) => {}, + () = metadata_health(&state, (client, poll), (started, None), None) => {}, } return finish_metadata_page(next_scan(state)).await; } @@ -105,45 +119,17 @@ pub(super) async fn sync_location_metadata( if let Err(error) = response { tokio::select! { result = &mut stopped => { result?; return Ok(None); }, - () = metadata_health(&state, (client, poll), (started, Some(WarehouseSyncFailure::Delivery))) => {}, + () = metadata_health(&state, (client, poll), (started, Some(WarehouseSyncFailure::Delivery)), None) => {}, } return retry_metadata(state, error); } tokio::select! { result = &mut stopped => { result?; return Ok(None); }, - () = metadata_health(&state, (client, poll), (started, None)) => {}, + () = metadata_health(&state, (client, poll), (started, None), None) => {}, } finish_metadata_page(acknowledge_metadata_page(state)).await } -/// A busy warehouse file waits a poll interval; any other failed read retries a smaller page. -async fn failed_read( - mut state: LocationMetadataSync, - (client, poll): (&LocationWorkClient, &LocationWorkPoll), - started: Instant, - stopped: impl Future>, -) -> Result, ControlError> { - let failure = if state.busy { - WarehouseSyncFailure::Busy - } else { - WarehouseSyncFailure::Read - }; - tokio::select! { - result = stopped => { result?; return Ok(None); }, - () = metadata_health(&state, (client, poll), (started, Some(failure))) => {}, - } - if state.busy { - state.next_at = Instant::now() + Duration::from_secs(METADATA_POLL_SECONDS); - return Ok(Some(state)); - } - state.retry_read(); - drop(write_output(&[( - Stream::Stderr, - "Warehouse history read failed; retaining mirror and retrying a bounded page.\n", - )])); - Ok(Some(state)) -} - fn retry_metadata( mut state: LocationMetadataSync, error: ClientError, @@ -155,6 +141,10 @@ fn retry_metadata( } state.next_at = Instant::now() + Duration::from_secs(METADATA_POLL_SECONDS); state.reader = None; + location_warning( + "warehouse sync failed", + format!("retrying in {METADATA_POLL_SECONDS}s: {error}"), + ); drop(write_output(&[( Stream::Stderr, &format!( diff --git a/crates/sqlbuild-cli/src/native_commands/main/_location_work.rs b/crates/sqlbuild-cli/src/native_commands/main/_location_work.rs index 67ebe7e7d5..abd916e9ab 100644 --- a/crates/sqlbuild-cli/src/native_commands/main/_location_work.rs +++ b/crates/sqlbuild-cli/src/native_commands/main/_location_work.rs @@ -23,7 +23,7 @@ use crate::native_commands::models::{LocationServe, LocationWatch}; /// Retry the exact completed result without rerunning it; `true` asks the metadata loop to sync. pub(super) async fn run_location_work( - (options, script): (&LocationServe, &Path), + (options, script, connected): (&LocationServe, &Path, bool), (client, poll, watch): (&LocationWorkClient, &LocationWorkPoll, &LocationWatch), generation: Option, stopped: impl Future>, @@ -59,7 +59,13 @@ pub(super) async fn run_location_work( } }; let copy = (client, poll.session_id, claim.generation); - let checked = perform_work((options, script), claim.request.work, copy, expires).await?; + let checked = perform_work( + (options, script, connected), + claim.request.work, + copy, + expires, + ) + .await?; let Some((outcome, local_files)) = checked else { drop(write_output(&[( Stream::Stderr, @@ -121,9 +127,9 @@ pub(super) async fn run_location_work( Ok(Some((files, sync))) } -/// One named operation; `None` means it stopped at shutdown or its lease ended. +/// One named operation; `None` means it stopped at shutdown or its lease ended; `sqb dev` never signs in here. async fn perform_work( - (options, script): (&LocationServe, &Path), + (options, script, connected): (&LocationServe, &Path, bool), work: LocationWork, copy: (&LocationWorkClient, Uuid, i64), expires: impl Future>, @@ -137,9 +143,11 @@ async fn perform_work( drop(write_output(&[(Stream::Stderr, message)])); Ok(match work { LocationWork::SyncWarehouse {} => Some((LocationWorkOutcome::SyncStarted {}, Vec::new())), - LocationWork::CheckConnection {} => check_location_connection(options, script, expires) - .await? - .map(|(healthy, _, files)| (LocationWorkOutcome::Connection { healthy }, files)), + LocationWork::CheckConnection {} => { + check_location_connection((options, script, connected), expires) + .await? + .map(|(healthy, _, files)| (LocationWorkOutcome::Connection { healthy }, files)) + } LocationWork::EvaluateSensor { evaluation } => { evaluate_location_sensor(options, script, &evaluation, expires) .await? diff --git a/crates/sqlbuild-cli/src/native_commands/main/_run_dev.rs b/crates/sqlbuild-cli/src/native_commands/main/_run_dev.rs index 2b3c162bba..99ff6c2c9a 100644 --- a/crates/sqlbuild-cli/src/native_commands/main/_run_dev.rs +++ b/crates/sqlbuild-cli/src/native_commands/main/_run_dev.rs @@ -1,87 +1,48 @@ //! Collapse the production API, scheduler and outbound location into one local process tree. -use std::path::Path; +use std::net::SocketAddr; -use sqlbuild_orchestration::api::models::DevelopmentScope; -use sqlbuild_orchestration::credentials::main::identities::development_person::development_person; -use sqlbuild_orchestration::credentials::main::locations::development_location::development_location; -use sqlbuild_orchestration::credentials::models::AuditContext; -use sqlbuild_orchestration::declarations::main::pause_development_triggers::pause_development_triggers; use sqlbuild_orchestration::store::errors::ControlError; -use sqlbuild_orchestration::store::main::set_release_version::set_release_version; -use sqlbuild_orchestration::store::main::{migrate_store::migrate_store, open_store::open_store}; -use sqlbuild_orchestration::store::models::ControlStore; -use uuid::Uuid; -use crate::argv::models::{Namespace, Stream}; -use crate::native_commands::_helpers::decision::{control_address, flag, text}; +use crate::argv::models::Namespace; +use crate::dev_screen::main::start_dev_screen::start_dev_screen; +use crate::dev_screen::models::DevScreenHandle; +use crate::native_commands::_helpers::decision::{flag, text}; use crate::native_commands::_helpers::execution::run_development_roles; +use crate::native_commands::_helpers::execution::{ + announced_store, loopback_address, screen_setup, +}; use crate::native_commands::_helpers::project_facts::development_target; -use crate::native_commands::_helpers::terminal::write_output; -use crate::server_config::models::ServerSection; pub(super) async fn run_dev(namespace: &Namespace) -> Result<(), ControlError> { let project = std::fs::canonicalize(text(namespace, "project_dir").unwrap_or_else(|| ".".into()))?; let target = development_target(&project, text(namespace, "target"))?; - let address = control_address(namespace, false, &ServerSection::default())? - .ok_or_else(|| ControlError::Invalid("development address is required".into()))?; - if !address.ip().is_loopback() { - return Err(ControlError::Invalid( - "sqb dev only listens on loopback".into(), - )); - } + let address: SocketAddr = loopback_address(namespace)?; let executable = std::env::current_exe()?; - drop(write_output(&[( - Stream::Stderr, - "Starting development control store...\n", - )])); - let (store, scope) = development_store(namespace, &project, &target).await?; - drop(write_output(&[( - Stream::Stderr, - "Development control store ready\nStarting development API...\n", - )])); - let listener = tokio::net::TcpListener::bind(address).await?; - run_development_roles( - (store, scope), - (project, executable), - listener, - !flag(namespace, "control_no_scheduler"), - ) - .await -} - -/// Complete retained identity and startup pauses before starting any scheduling roles. -async fn development_store( - namespace: &Namespace, - project: &Path, - target: &str, -) -> Result<(ControlStore, DevelopmentScope), ControlError> { - let database = text(namespace, "development_store_url").unwrap_or_else(|| { - project - .join(".sqlbuild/control.sqlite3") - .to_string_lossy() - .into_owned() - }); - set_release_version(env!("CARGO_PKG_VERSION")); - let store = open_store(&database).await?; - migrate_store(&store).await?; - let context = AuditContext { - actor: "local-development".into(), - granting_role: "control-store-owner".into(), - request_id: Uuid::now_v7(), - ip_address: None, - }; - let location_id = - development_location(&store, &project.to_string_lossy(), target, &context).await?; - let scope = DevelopmentScope { - location_id, - target: target.into(), - triggers: flag(namespace, "development_triggers"), - principal_id: development_person(&store, &context).await?, - }; - if !scope.triggers { - pause_development_triggers(&store, (location_id, target)).await?; + let DevScreenHandle { + messages, + actions, + keys, + finish, + task, + } = start_dev_screen(screen_setup(&project, &target)); + let result: Result<(), ControlError> = async { + let (store, scope) = announced_store(namespace, (&project, &target), &messages).await?; + let listener = tokio::net::TcpListener::bind(address).await?; + run_development_roles( + (store, scope), + (project.clone(), executable), + listener, + ( + !flag(namespace, "control_no_scheduler"), + (messages.clone(), actions, keys), + ), + ) + .await } - Ok((store, scope)) + .await; + let _ = finish.send(result.is_err()); + drop(task.await); + result } diff --git a/crates/sqlbuild-cli/src/native_commands/main/_run_location.rs b/crates/sqlbuild-cli/src/native_commands/main/_run_location.rs index c95603715d..096361a952 100644 --- a/crates/sqlbuild-cli/src/native_commands/main/_run_location.rs +++ b/crates/sqlbuild-cli/src/native_commands/main/_run_location.rs @@ -4,20 +4,21 @@ use std::future::Future; use std::path::Path; use std::time::Duration; +use sqlbuild_cache::project_snapshot::models::PathStamp; use sqlbuild_orchestration::platform::main::connect_location_work::connect_location_work; -use sqlbuild_orchestration::platform::models::{ - LocationHealth, LocationWorkClient, LocationWorkPoll, -}; +use sqlbuild_orchestration::platform::models::{LocationWorkClient, LocationWorkPoll}; use sqlbuild_orchestration::store::errors::ControlError; -use tokio::sync::mpsc; +use sqlbuild_orchestration::warehouse_metadata::models::WarehouseStateLocation; +use tokio::sync::{mpsc, watch}; use tokio::time::Instant; use uuid::Uuid; use crate::argv::models::Stream; use crate::native_commands::_helpers::execution::{ - check_published_location, close_metadata_reader, compile_generation, location_clock, - location_generation, location_heartbeats, metadata_sync, next_turn, report_location_health, - shutdown_signal, + apply_dev_controls, compile_next_generation, finish_compile, +}; +use crate::native_commands::_helpers::execution::{ + dev_controls, location_clock, location_heartbeats, next_turn, shutdown_signal, }; use crate::native_commands::_helpers::paths::{ exclude_location_databases, location_watch, observe_location_inputs, @@ -27,8 +28,9 @@ use crate::native_commands::main::_location_compiler::run_location_compiler; use crate::native_commands::main::_location_metadata::sync_location_metadata; use crate::native_commands::main::_location_work::run_location_work; use crate::native_commands::models::{ - LocationCompileRequest, LocationMetadataSync, LocationServe, LocationWatch, + DevControlState, LocationCompileRequest, LocationMetadataSync, LocationServe, LocationWatch, }; +use crate::native_commands::types::CompileOutcome; pub(super) async fn run_location( options: LocationServe, @@ -43,9 +45,10 @@ pub(super) async fn run_location( target: options.target.clone(), }; let (requests, receiver) = mpsc::channel(1); + let controls = dev_controls(&options); let serving = async { tokio::select! { - result = watch_location(&options, script, (bearer, &client, &poll), requests) => result, + result = watch_location(&options, script, (bearer, &client, &poll), (requests, controls)) => result, result = location_heartbeats(&options, bearer, session_id) => result, } }; @@ -58,9 +61,15 @@ async fn watch_location( options: &LocationServe, script: &Path, (bearer, client, poll): (&str, &LocationWorkClient, &LocationWorkPoll), - requests: mpsc::Sender, + (requests, mut controls): ( + mpsc::Sender, + watch::Receiver, + ), ) -> Result<(), ControlError> { let mut watch = location_watch(&options.project_dir); + let mut recompiles: u64 = 0; + let mut compiled_inputs: Option> = None; + let mut deferred_sync: Option<(Vec, i64)> = None; let mut work_at = Instant::now() + Duration::from_secs(2); let mut published_generation = None; let mut metadata = None; @@ -81,11 +90,23 @@ async fn watch_location( { return result; } + let Some(applied) = apply_dev_controls( + (options, script, (bearer, poll.session_id)), + (*controls.borrow_and_update(), recompiles), + (watch, metadata, deferred_sync), + stopped.as_mut(), + ) + .await? + else { + return Ok(()); + }; + (recompiles, (watch, metadata, deferred_sync)) = applied; let ready; (watch, ready) = observe_location_inputs(&options.project_dir, watch); if !ready { + let connected: bool = controls.borrow().warehouse == Some(true); let Some(idle) = poll_location( - (options, script), + (options, script, connected), (client, poll, published_generation), (watch, work_at, metadata), stopped.as_mut(), @@ -98,67 +119,39 @@ async fn watch_location( continue; } published_generation = None; - if let Some(state) = metadata.take() { - metadata = Some(close_metadata_reader(state).await?); - } - let generation = tokio::select! { - result = &mut stopped => return result, - generation = location_generation(options, bearer) => generation?, + let Some(next) = compile_next_generation( + (options, bearer, &requests), + (metadata, compiled_inputs, &watch.observed.0), + stopped.as_mut(), + ) + .await? + else { + return Ok(()); }; - let generation = match generation { - Some(generation) => generation, - None => { - watch.retry_at = Some(Instant::now() + Duration::from_secs(5)); - continue; - } + let next_compile: Option<(CompileOutcome, i64)>; + (next_compile, metadata, compiled_inputs) = next; + let Some((compiled, generation)) = next_compile else { + watch.retry_at = Some(Instant::now() + Duration::from_secs(5)); + continue; }; - drop(write_output(&[( - Stream::Stderr, - &format!("Compiling location generation {generation} in an isolated interpreter...\n"), - )])); - let Some((published, refresh_at, locations)) = - compile_generation(&requests, generation, &watch.observed.0, stopped.as_mut()).await? + published_generation = compiled.0.then_some(generation); + let Some(settled) = finish_compile( + (options, script, (bearer, poll.session_id)), + (compiled, generation), + (watch, metadata, deferred_sync), + (&mut controls, stopped.as_mut()), + ) + .await? else { return Ok(()); }; - if published { - watch.retry_at = refresh_at; - published_generation = Some(generation); - metadata = Some(metadata_sync( - (locations, generation), - ( - options.metadata_reconcile_seconds, - options.safety_sync_seconds, - ), - )); - let Some(files) = check_published_location( - options, - script, - (bearer, poll.session_id, generation), - stopped.as_mut(), - ) - .await? - else { - return Ok(()); - }; - watch = exclude_location_databases(&options.project_dir, files, watch); - } else { - watch.retry_at = Some(Instant::now() + Duration::from_secs(30)); - tokio::select! { - result = &mut stopped => return result, - () = report_location_health(options, bearer, poll.session_id, LocationHealth::CompileFailed { generation }) => {}, - } - drop(write_output(&[( - Stream::Stderr, - "Location reload failed; retaining last good snapshot. Retrying on change or in 30 seconds.\n", - )])); - } + (watch, metadata, deferred_sync) = settled; } } /// Named work and metadata share one idle turn without transferring interpreter ownership. async fn poll_location( - (options, script): (&LocationServe, &Path), + (options, script, connected): (&LocationServe, &Path, bool), (client, poll, generation): (&LocationWorkClient, &LocationWorkPoll, Option), (mut watch, mut work_at, mut metadata): (LocationWatch, Instant, Option), stopped: impl Future>, @@ -167,7 +160,7 @@ async fn poll_location( if generation.is_some() && watch.attempted.as_ref() == Some(&watch.observed) { if Instant::now() >= work_at { let Some((files, sync)) = run_location_work( - (options, script), + (options, script, connected), (client, poll, &watch), generation, stopped.as_mut(), diff --git a/crates/sqlbuild-cli/src/native_commands/models.rs b/crates/sqlbuild-cli/src/native_commands/models.rs index 9a762fcf63..86862f40df 100644 --- a/crates/sqlbuild-cli/src/native_commands/models.rs +++ b/crates/sqlbuild-cli/src/native_commands/models.rs @@ -10,12 +10,13 @@ use sqlbuild_orchestration::declarations::models::TriggerChange; use sqlbuild_orchestration::platform::models::LocationWorkRequest; use sqlbuild_orchestration::runs::models::{JobRunRequest, RerunRequest}; use sqlbuild_orchestration::warehouse_metadata::models::{ - MetadataBatch, StatisticsRead, WarehouseHistoryKind, WarehouseHistoryPage, - WarehouseStateLocation, + MetadataBatch, StatisticsRead, WarehouseHistoryFailure, WarehouseHistoryKind, + WarehouseHistoryPage, WarehouseStateLocation, }; use uuid::Uuid; use crate::argv::models::Namespace; +use crate::native_commands::constants::{METADATA_FAILURE_LIMIT, METADATA_POLL_SECONDS}; use crate::native_commands::types::LocationPublication; /// Privileged local operations require direct ownership of the control database. @@ -152,6 +153,16 @@ pub(crate) struct LocationServe { pub(crate) metadata_reconcile_seconds: u32, /// Seconds between safety syncs of an idle location; `0` turns them off. pub(crate) safety_sync_seconds: u64, + /// `sqb dev` reads events on stdout and sends control lines on stdin. + pub(crate) dev_events: bool, +} + +/// What `sqb dev` last asked its location: the foreground warehouse verdict and how many recompiles. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub(crate) struct DevControlState { + /// `None` until `sqb dev` reports its foreground connection; outside `sqb dev`, always connected. + pub(crate) warehouse: Option, + pub(crate) recompiles: u64, } /// Retained source observations survive compilation; warehouse exclusions never replace them. @@ -218,8 +229,10 @@ pub(crate) struct LocationMetadataSync { pub(crate) safety_seconds: u64, /// When an idle location syncs even though no run ended and nobody asked. pub(crate) safety_at: tokio::time::Instant, - /// The last read found the warehouse file held by another process, such as a DuckDB build. - pub(crate) busy: bool, + /// Failed reads in a row that did not shrink the page; a delivered page resets it. + pub(crate) failed_reads: u32, + /// Failed reads in a row stopped the synchronization until a sync request or a new publication. + pub(crate) halted: bool, /// Per scan index: the table's change token after its last complete read. pub(crate) change_tokens: HashMap, /// The token read before the current scan's first page, recorded once the scan completes. @@ -235,8 +248,9 @@ pub(crate) struct LocationMetadataSync { /// The metadata reader's answer to one read. #[derive(Debug)] pub(crate) enum MetadataReply { - /// A page, or `None` when the read failed. - Page(Option), + Page(WarehouseHistoryPage), + /// The read failed and delivered nothing; never an empty table. + Failed(WarehouseHistoryFailure), /// Another process holds the warehouse file; nothing was read. Busy, } @@ -255,7 +269,8 @@ impl LocationMetadataSync { /// A person asked for a sync: start one now when idle, or one more after the current one. pub(crate) fn request_sync(&mut self) { self.cycle_started_at = None; - if self.idle { + let backing_off = std::mem::take(&mut self.failed_reads) > 0; + if std::mem::take(&mut self.halted) || backing_off || self.idle { self.next_at = tokio::time::Instant::now(); } } @@ -277,26 +292,55 @@ impl LocationMetadataSync { self.next_at = tokio::time::Instant::now(); } - /// Failed samples revisit catalog before retrying; no frozen delivery is discarded. - pub(crate) fn retry_read(&mut self) { + /// Statistics revisit catalog; a page-size failure halves the page, any other backs off; `None` once stopped. + pub(crate) fn fail_read(&mut self, page_size: bool) -> Option { self.statistics = None; - if self.kind() == WarehouseHistoryKind::Statistics { - self.statistics_limit = (self.statistics_limit / 2).max(1); + let statistics = self.kind() == WarehouseHistoryKind::Statistics; + if statistics { self.index -= 1; self.source_after = None; self.scan_id = Uuid::now_v7(); self.observed_since = None; self.watermark_loaded = false; + } + let limit = if statistics { + &mut self.statistics_limit + } else { + &mut self.limit + }; + let seconds = if page_size && *limit > 1 { + *limit /= 2; + METADATA_POLL_SECONDS } else { - self.limit = (self.limit / 2).max(1); + self.failed_reads += 1; + if self.failed_reads >= METADATA_FAILURE_LIMIT { + self.halted = true; + self.safety_at = tokio::time::Instant::now() + + std::time::Duration::from_secs(self.safety_seconds); + return None; + } + METADATA_POLL_SECONDS << (self.failed_reads - 1) + }; + let wait = std::time::Duration::from_secs(seconds); + self.next_at = tokio::time::Instant::now() + wait; + Some(wait) + } + + /// A stopped sync tries once more when its safety sync is due; one more failure stops it again. + pub(crate) fn resume_after_safety_wait(&mut self) -> bool { + let now = tokio::time::Instant::now(); + let due: bool = self.halted && self.safety_seconds > 0 && now >= self.safety_at; + if due { + self.halted = false; + self.failed_reads = METADATA_FAILURE_LIMIT - 1; + self.next_at = now; } - self.next_at = tokio::time::Instant::now() - + std::time::Duration::from_secs(super::constants::METADATA_POLL_SECONDS); + due } /// A synchronization is mid-flight and its next page is due, so the loop need not wait a tick. pub(crate) fn paging(&self) -> bool { - !self.idle && tokio::time::Instant::now() >= self.next_at + !self.idle && !self.halted && tokio::time::Instant::now() >= self.next_at } pub(crate) fn reconciling(&self) -> bool { diff --git a/crates/sqlbuild-cli/src/native_commands/tests/test_location_metadata.rs b/crates/sqlbuild-cli/src/native_commands/tests/test_location_metadata.rs index 83282fde2c..1596889df5 100644 --- a/crates/sqlbuild-cli/src/native_commands/tests/test_location_metadata.rs +++ b/crates/sqlbuild-cli/src/native_commands/tests/test_location_metadata.rs @@ -11,8 +11,8 @@ use tokio::time::Instant; use crate::native_commands::_helpers::execution::{metadata_sync, safety_sync_seconds}; use crate::native_commands::tests::helpers::{acknowledge_complete_pages, one_location}; use crate::native_commands::tests::test_types::{ - MetadataRetryTestCase, SafetySyncIdleTestCase, SafetySyncSettingTestCase, - SyncCoalescingTestCase, UnchangedScanTestCase, + MetadataBackoffTestCase, MetadataRetryTestCase, SafetyResumeTestCase, SafetySyncIdleTestCase, + SafetySyncSettingTestCase, SyncCoalescingTestCase, UnchangedScanTestCase, }; #[test] @@ -20,22 +20,32 @@ fn given_failed_metadata_read_when_back_at_statistics_then_page_shrinks_until_sc let test_cases = [ MetadataRetryTestCase { description: "a timed-out statistics page is retried smaller after catalog is revisited", + page_size: true, failed_kind: WarehouseHistoryKind::Statistics, acknowledged_pages: 1, expected_statistics_limit: MAX_HISTORY_RECORDS / 2, }, MetadataRetryTestCase { description: "a failed catalog page leaves the statistics page size alone", + page_size: true, failed_kind: WarehouseHistoryKind::Catalog, acknowledged_pages: 1, expected_statistics_limit: MAX_HISTORY_RECORDS, }, MetadataRetryTestCase { description: "a completed statistics scan restores the full statistics page", + page_size: true, failed_kind: WarehouseHistoryKind::Statistics, acknowledged_pages: 1 + WarehouseHistoryKind::ALL.len(), expected_statistics_limit: MAX_HISTORY_RECORDS, }, + MetadataRetryTestCase { + description: "a relation dropped while sampling revisits catalog without shrinking the page", + page_size: false, + failed_kind: WarehouseHistoryKind::Statistics, + acknowledged_pages: 1, + expected_statistics_limit: MAX_HISTORY_RECORDS, + }, ]; for test_case in test_cases { let location = WarehouseStateLocation { @@ -47,7 +57,8 @@ fn given_failed_metadata_read_when_back_at_statistics_then_page_shrinks_until_sc .iter() .position(|kind| *kind == test_case.failed_kind) .expect("kind is listed"); - state.retry_read(); + let wait = state.fail_read(test_case.page_size); + assert!(wait.is_some(), "{}", test_case.description); let state = acknowledge_complete_pages(state, test_case.acknowledged_pages); assert_eq!( (state.kind(), state.statistics_limit, state.limit), @@ -62,6 +73,132 @@ fn given_failed_metadata_read_when_back_at_statistics_then_page_shrinks_until_sc } } +#[test] +fn given_repeated_failed_reads_when_retrying_then_only_page_size_errors_shrink_and_failures_in_a_row_stop() + { + let test_cases = [ + MetadataBackoffTestCase { + description: "a permission or missing-table error never shrinks the page, and a request ends its wait", + failures: vec![false, false, false], + failures_after_request: vec![], + expected_waits: vec![Some(30), Some(60), Some(120)], + expected_limit: MAX_HISTORY_RECORDS, + expected_halted: false, + expected_paging: (true, true), + }, + MetadataBackoffTestCase { + description: "the fifth failure in a row stops the sync, which then never pages", + failures: vec![], + failures_after_request: vec![false, false, false, false, false], + expected_waits: vec![Some(30), Some(60), Some(120), Some(240), None], + expected_limit: MAX_HISTORY_RECORDS, + expected_halted: true, + expected_paging: (true, false), + }, + MetadataBackoffTestCase { + description: "a page-size error halves the page and retries after one poll", + failures: vec![true, true], + failures_after_request: vec![], + expected_waits: vec![Some(30), Some(30)], + expected_limit: MAX_HISTORY_RECORDS / 4, + expected_halted: false, + expected_paging: (false, false), + }, + MetadataBackoffTestCase { + description: "a sync request resumes a stopped sync with a fresh count", + failures: vec![false, false, false, false, false], + failures_after_request: vec![false], + expected_waits: vec![Some(30), Some(60), Some(120), Some(240), None, Some(30)], + expected_limit: MAX_HISTORY_RECORDS, + expected_halted: false, + expected_paging: (true, false), + }, + ]; + for test_case in test_cases { + let mut state = metadata_sync((one_location(), 1), (3600, 0)); + state.index = WarehouseHistoryKind::ALL + .iter() + .position(|kind| *kind == WarehouseHistoryKind::Fingerprints) + .expect("kind is listed"); + state.begin_sync(Utc::now()); + let mut waits: Vec> = test_case + .failures + .into_iter() + .map(|page_size| state.fail_read(page_size).map(|wait| wait.as_secs())) + .collect(); + state.request_sync(); + let paging_after_request = state.paging(); + waits.extend( + test_case + .failures_after_request + .into_iter() + .map(|page_size| state.fail_read(page_size).map(|wait| wait.as_secs())), + ); + assert_eq!( + ( + waits, + state.limit, + state.halted, + (paging_after_request, state.paging()) + ), + ( + test_case.expected_waits, + test_case.expected_limit, + test_case.expected_halted, + test_case.expected_paging, + ), + "{}", + test_case.description + ); + } +} + +#[test] +fn given_a_stopped_sync_when_its_safety_sync_is_due_then_it_makes_one_attempt() { + let test_cases = [ + SafetyResumeTestCase { + description: "before the safety sync is due the sync stays stopped", + safety_seconds: 1800, + due_in: Duration::from_secs(600), + expected_outcome: (false, false, None, true), + }, + SafetyResumeTestCase { + description: "a due safety sync resumes it, and one more failure stops it for another interval", + safety_seconds: 1800, + due_in: Duration::ZERO, + expected_outcome: (true, true, None, true), + }, + SafetyResumeTestCase { + description: "with safety syncs off only a request or a publication resumes it", + safety_seconds: 0, + due_in: Duration::ZERO, + expected_outcome: (false, false, None, true), + }, + ]; + for test_case in test_cases { + let mut state = metadata_sync((one_location(), 1), (3600, test_case.safety_seconds)); + state.begin_sync(Utc::now()); + let stopped: Vec> = (0..5).map(|_| state.fail_read(false)).collect(); + state.safety_at = Instant::now() + test_case.due_in; + let resumed: bool = state.resume_after_safety_wait(); + let paging: bool = state.paging(); + let wait: Option = state.fail_read(false); + assert_eq!( + ( + stopped[4], + (resumed, paging, wait, state.halted), + (state.safety_at.saturating_duration_since(Instant::now()) + + Duration::from_secs(1)) + .as_secs() + >= test_case.safety_seconds, + ), + (None, test_case.expected_outcome, true), + "{}", + test_case.description + ); + } +} + #[test] fn given_recorded_change_token_when_scanning_again_then_only_proven_unchanged_reads_skip() { let test_cases = [ diff --git a/crates/sqlbuild-cli/src/native_commands/tests/test_types.rs b/crates/sqlbuild-cli/src/native_commands/tests/test_types.rs index 8282a11eaf..15fba03e7a 100644 --- a/crates/sqlbuild-cli/src/native_commands/tests/test_types.rs +++ b/crates/sqlbuild-cli/src/native_commands/tests/test_types.rs @@ -92,14 +92,36 @@ pub(super) struct PlanDocumentTestCase { pub(super) expected_first_model: serde_json::Value, } -/// One failed metadata read and the statistics page size the sync uses afterwards. +/// One failed metadata read, whether it was a page-size failure, and the statistics page size afterwards. pub(super) struct MetadataRetryTestCase { pub(super) description: &'static str, + pub(super) page_size: bool, pub(super) failed_kind: WarehouseHistoryKind, pub(super) acknowledged_pages: usize, pub(super) expected_statistics_limit: usize, } +/// Failed reads of one table (true when page-size) around a sync request, and what follows them. +pub(super) struct MetadataBackoffTestCase { + pub(super) description: &'static str, + pub(super) failures: Vec, + pub(super) failures_after_request: Vec, + pub(super) expected_waits: Vec>, + pub(super) expected_limit: usize, + pub(super) expected_halted: bool, + /// Whether the sync pages right after the request, and after the last failure. + pub(super) expected_paging: (bool, bool), +} + +/// A sync stopped by failed reads, its safety interval and how soon the safety sync is due. +pub(super) struct SafetyResumeTestCase { + pub(super) description: &'static str, + pub(super) safety_seconds: u64, + pub(super) due_in: std::time::Duration, + /// Whether it resumed and pages, the next failure's wait, and whether that stopped it again. + pub(super) expected_outcome: (bool, bool, Option, bool), +} + /// A completed state-table read, the later scan of the same table, and whether that scan may skip. pub(super) struct UnchangedScanTestCase { pub(super) description: &'static str, diff --git a/crates/sqlbuild-cli/src/native_commands/types.rs b/crates/sqlbuild-cli/src/native_commands/types.rs index 862c4aa10d..8562deea67 100644 --- a/crates/sqlbuild-cli/src/native_commands/types.rs +++ b/crates/sqlbuild-cli/src/native_commands/types.rs @@ -4,9 +4,32 @@ use chrono::{DateTime, Utc}; use sqlbuild_orchestration::snapshots::models::SnapshotRecord; use sqlbuild_orchestration::warehouse_metadata::models::WarehouseStateLocation; +use crate::native_commands::models::{LocationMetadataSync, LocationWatch}; + /// Publication metadata and the earliest calendar refresh deadline. pub(crate) type LocationPublication = ( SnapshotRecord, Option>, Vec, ); + +/// A settled publication: the watch without its local databases, its metadata sync, and the sync deferred until the warehouse connects. +pub(crate) type Settled = ( + LocationWatch, + Option, + Option<(Vec, i64)>, +); + +/// A compile's outcome: whether it published, its next calendar refresh, and the warehouse state it reads. +pub(crate) type CompileOutcome = ( + bool, + Option, + Vec, +); + +/// The next compile, if a generation number was issued, with the metadata sync and the inputs it saw. +pub(crate) type NextGeneration = ( + Option<(CompileOutcome, i64)>, + Option, + Option>, +); diff --git a/crates/sqlbuild-cli/src/person_login/_helpers/flow.rs b/crates/sqlbuild-cli/src/person_login/_helpers/flow.rs index 3c14e27cb9..2bc05bedc4 100644 --- a/crates/sqlbuild-cli/src/person_login/_helpers/flow.rs +++ b/crates/sqlbuild-cli/src/person_login/_helpers/flow.rs @@ -8,6 +8,7 @@ use serde_json::json; use sqlbuild_recall::records::main::sha256_hex::sha256_hex; use crate::person_login::constants::{LOGIN_SECONDS, POLL_SECONDS, SECRET_BYTES}; +use crate::person_login::main::open_browser::open_browser; use crate::person_login::models::{LoginFlow, PersonClient, PersonCredential}; pub(crate) async fn login( @@ -88,30 +89,3 @@ pub(crate) async fn login( "CLI sign-in expired; start sqb login again", )) } - -fn open_browser(url: &str) -> bool { - #[cfg(target_os = "macos")] - let mut command = std::process::Command::new("open"); - #[cfg(all(unix, not(target_os = "macos")))] - let mut command = std::process::Command::new("xdg-open"); - #[cfg(windows)] - let mut command = { - let mut command = std::process::Command::new("rundll32"); - command.arg("url.dll,FileProtocolHandler"); - command - }; - command - .arg(url) - .stdin(std::process::Stdio::null()) - .stdout(std::process::Stdio::null()) - .stderr(std::process::Stdio::null()); - match command.spawn() { - Ok(mut child) => { - std::thread::spawn(move || { - let _ = child.wait(); - }); - true - } - Err(_) => false, - } -} diff --git a/crates/sqlbuild-cli/src/person_login/main/mod.rs b/crates/sqlbuild-cli/src/person_login/main/mod.rs index ce45b29160..62d35f1ef5 100644 --- a/crates/sqlbuild-cli/src/person_login/main/mod.rs +++ b/crates/sqlbuild-cli/src/person_login/main/mod.rs @@ -1,3 +1,4 @@ +pub(crate) mod open_browser; pub(crate) mod read_node_sql; pub(crate) mod request_warehouse_sync; pub(crate) mod run_login; diff --git a/crates/sqlbuild-cli/src/person_login/main/open_browser.rs b/crates/sqlbuild-cli/src/person_login/main/open_browser.rs new file mode 100644 index 0000000000..ff3a6032a6 --- /dev/null +++ b/crates/sqlbuild-cli/src/person_login/main/open_browser.rs @@ -0,0 +1,29 @@ +//! Open a URL in the person's default browser. + +/// Open `url` in the default browser without waiting for it; false when no opener could start. +pub(crate) fn open_browser(url: &str) -> bool { + #[cfg(target_os = "macos")] + let mut command = std::process::Command::new("open"); + #[cfg(all(unix, not(target_os = "macos")))] + let mut command = std::process::Command::new("xdg-open"); + #[cfg(windows)] + let mut command = { + let mut command = std::process::Command::new("rundll32"); + command.arg("url.dll,FileProtocolHandler"); + command + }; + command + .arg(url) + .stdin(std::process::Stdio::null()) + .stdout(std::process::Stdio::null()) + .stderr(std::process::Stdio::null()); + match command.spawn() { + Ok(mut child) => { + std::thread::spawn(move || { + let _ = child.wait(); + }); + true + } + Err(_) => false, + } +} diff --git a/crates/sqlbuild-core/src/cli_text/constants.rs b/crates/sqlbuild-core/src/cli_text/constants.rs index 483b68b65d..554d2fe9a0 100644 --- a/crates/sqlbuild-core/src/cli_text/constants.rs +++ b/crates/sqlbuild-core/src/cli_text/constants.rs @@ -6,6 +6,7 @@ pub(crate) const RED: &str = "\x1b[38;5;167m"; pub(crate) const GREEN: &str = "\x1b[32m"; pub(crate) const YELLOW: &str = "\x1b[33m"; pub(crate) const BLUE: &str = "\x1b[34m"; +pub(crate) const CYAN: &str = "\x1b[36m"; pub(crate) const ORANGE: &str = "\x1b[38;5;208m"; pub(crate) const RESET: &str = "\x1b[0m"; pub(crate) const SUCCESS_STATUS_LABELS: [&str; 3] = ["OK", "PASS", "SUCCESS"]; diff --git a/crates/sqlbuild-core/src/cli_text/models.rs b/crates/sqlbuild-core/src/cli_text/models.rs index 572efa79f1..2dbb54cc38 100644 --- a/crates/sqlbuild-core/src/cli_text/models.rs +++ b/crates/sqlbuild-core/src/cli_text/models.rs @@ -1,7 +1,7 @@ //! Plain data of the human compile report, and the semantic styling it is rendered with. use crate::cli_text::constants::{ - BLUE, BOLD, DIM, ERROR_STATUS_LABELS, GREEN, ORANGE, RED, RESET, SKIPPED_STATUS_LABELS, + BLUE, BOLD, CYAN, DIM, ERROR_STATUS_LABELS, GREEN, ORANGE, RED, RESET, SKIPPED_STATUS_LABELS, SUCCESS_STATUS_LABELS, WARNING_STATUS_LABELS, YELLOW, }; @@ -50,6 +50,12 @@ impl CliStyle { self.apply(BLUE, text) } + /// Cyan: URLs and started runs on the live screens. + #[must_use] + pub fn link(self, text: &str) -> String { + self.apply(CYAN, text) + } + #[must_use] pub fn accent(self, text: &str) -> String { self.apply(BLUE, text) diff --git a/crates/sqlbuild-core/src/lib.rs b/crates/sqlbuild-core/src/lib.rs index 0b6f781eea..318a55720a 100644 --- a/crates/sqlbuild-core/src/lib.rs +++ b/crates/sqlbuild-core/src/lib.rs @@ -52,6 +52,7 @@ pub mod progress; pub mod query_command; pub mod redaction; pub mod scenario_command; +pub mod screen_text; pub mod seed_command; pub mod shell_text; pub mod terminal_events; diff --git a/crates/sqlbuild-core/src/screen_text/_helpers/mod.rs b/crates/sqlbuild-core/src/screen_text/_helpers/mod.rs new file mode 100644 index 0000000000..0e38b6c271 --- /dev/null +++ b/crates/sqlbuild-core/src/screen_text/_helpers/mod.rs @@ -0,0 +1,2 @@ +pub(crate) mod phase_tones; +pub(crate) mod rows; diff --git a/crates/sqlbuild-core/src/screen_text/_helpers/phase_tones.rs b/crates/sqlbuild-core/src/screen_text/_helpers/phase_tones.rs new file mode 100644 index 0000000000..710a9db56c --- /dev/null +++ b/crates/sqlbuild-core/src/screen_text/_helpers/phase_tones.rs @@ -0,0 +1,47 @@ +//! The glyph and colours of each phase state. + +use crate::screen_text::constants::{ + FAIL_GLYPH, OK_GLYPH, PENDING_GLYPH, SPINNER_FRAMES, WARN_GLYPH, +}; +use crate::screen_text::models::{PhaseState, Tone}; + +/// How one phase line paints its glyph, label, detail and time. +pub(crate) struct PhaseTones { + pub(crate) glyph: &'static str, + pub(crate) glyph_tone: Tone, + pub(crate) label: Tone, + pub(crate) detail: Tone, + pub(crate) time: Tone, +} + +/// Dim while pending, a yellow spinner and timer while running, then green, yellow or red. +pub(crate) fn phase_tones(state: PhaseState) -> PhaseTones { + let (glyph, glyph_tone): (&'static str, Tone) = match state { + PhaseState::Pending => (PENDING_GLYPH, Tone::Muted), + PhaseState::Running { frame } => { + (SPINNER_FRAMES[frame % SPINNER_FRAMES.len()], Tone::Warning) + } + PhaseState::Ok => (OK_GLYPH, Tone::Success), + PhaseState::Warn => (WARN_GLYPH, Tone::Warning), + PhaseState::Fail => (FAIL_GLYPH, Tone::Error), + }; + PhaseTones { + glyph, + glyph_tone, + label: if state == PhaseState::Pending { + Tone::Muted + } else { + Tone::Plain + }, + detail: match state { + PhaseState::Warn => Tone::Warning, + PhaseState::Fail => Tone::Error, + _ => Tone::Muted, + }, + time: if matches!(state, PhaseState::Running { .. }) { + Tone::Warning + } else { + Tone::Muted + }, + } +} diff --git a/crates/sqlbuild-core/src/screen_text/_helpers/rows.rs b/crates/sqlbuild-core/src/screen_text/_helpers/rows.rs new file mode 100644 index 0000000000..640a310506 --- /dev/null +++ b/crates/sqlbuild-core/src/screen_text/_helpers/rows.rs @@ -0,0 +1,148 @@ +//! Cut, pad and style the spans of one screen row so it never wraps. + +use std::time::Duration; + +use crate::screen_text::constants::{ + CUT_MARK, OSC8_END, OSC8_START, SECONDS_PER_HOUR, SECONDS_PER_MINUTE, TENTHS_PER_MINUTE, +}; +use crate::screen_text::models::{ScreenFacts, Segment, Tone}; +use crate::text::main::cell_len::cell_len; +use crate::text::main::char_cells::char_cells; + +/// A span of `text` in `tone`. +pub(crate) fn span(text: impl Into, tone: Tone) -> Segment { + Segment { + text: text.into(), + tone, + link: None, + } +} + +/// A span of `text` in `tone` that links to `link`. +pub(crate) fn linked(text: impl Into, tone: Tone, link: Option) -> Segment { + Segment { + text: text.into(), + tone, + link, + } +} + +/// The leading characters of `text` that fit in `cells`. +pub(crate) fn take_cells(text: &str, cells: usize) -> String { + let mut used: usize = 0; + let mut kept: String = String::new(); + for character in text.chars() { + let width: usize = char_cells(character); + if used + width > cells { + break; + } + used += width; + kept.push(character); + } + kept +} + +/// `text` cut to `cells` with a trailing `…` when it is wider. +pub(crate) fn cut(text: &str, cells: usize) -> String { + if cell_len(text) <= cells { + return text.to_owned(); + } + if cells == 0 { + return String::new(); + } + format!("{}{CUT_MARK}", take_cells(text, cells - 1)) +} + +/// `text` cut to `cells`, then padded with spaces to exactly `cells`. +pub(crate) fn column(text: &str, cells: usize) -> String { + let kept: String = cut(text, cells); + let padding: usize = cells.saturating_sub(cell_len(&kept)); + format!("{kept}{}", " ".repeat(padding)) +} + +/// `text` right-aligned in `cells`. +pub(crate) fn right(text: &str, cells: usize) -> String { + format!("{}{text}", " ".repeat(cells.saturating_sub(cell_len(text)))) +} + +/// An elapsed time as `0.1s`, `16m 43s` or `2h 5m`, truncated rather than rounded. +pub(crate) fn elapsed_text(elapsed: Duration) -> String { + let tenths: u128 = elapsed.as_millis() / 100; + if tenths < TENTHS_PER_MINUTE { + return format!("{}.{}s", tenths / 10, tenths % 10); + } + let seconds: u64 = elapsed.as_secs(); + if seconds < SECONDS_PER_HOUR { + return format!( + "{}m {}s", + seconds / SECONDS_PER_MINUTE, + seconds % SECONDS_PER_MINUTE + ); + } + format!( + "{}h {}m", + seconds / SECONDS_PER_HOUR, + (seconds % SECONDS_PER_HOUR) / SECONDS_PER_MINUTE + ) +} + +/// The indented row of `segments`, without trailing spaces, cut with `…` to the screen width. +pub(crate) fn render_row(segments: Vec, facts: &ScreenFacts) -> String { + let segments: Vec = without_trailing_space(segments); + let budget: usize = facts.columns.saturating_sub(facts.indent); + let total: usize = segments.iter().map(|segment| cell_len(&segment.text)).sum(); + let mut row: String = " ".repeat(facts.indent); + if total <= budget { + for segment in &segments { + row.push_str(&styled(&segment.text, segment, facts)); + } + return row; + } + let keep: usize = budget.saturating_sub(1); + let mut used: usize = 0; + for segment in &segments { + if used >= keep { + break; + } + let text: String = take_cells(&segment.text, keep - used); + used += cell_len(&text); + row.push_str(&styled(&text, segment, facts)); + } + if budget > 0 { + row.push_str(&facts.style.muted(CUT_MARK)); + } + row +} + +fn without_trailing_space(mut segments: Vec) -> Vec { + while segments + .last() + .is_some_and(|segment| segment.text.trim_end().is_empty()) + { + segments.pop(); + } + if let Some(last) = segments.last_mut() { + last.text.truncate(last.text.trim_end().len()); + } + segments +} + +fn styled(text: &str, segment: &Segment, facts: &ScreenFacts) -> String { + let style = facts.style; + let painted: String = match segment.tone { + Tone::Plain => text.to_owned(), + Tone::Muted => style.muted(text), + Tone::Strong => style.section(text), + Tone::Success => style.success(text), + Tone::Warning => style.warning(text), + Tone::Error => style.error(text), + Tone::Change => style.value(text), + Tone::Link => style.link(text), + }; + match &segment.link { + Some(target) if facts.links && !text.is_empty() => { + format!("{OSC8_START}{target}{OSC8_END}{painted}{OSC8_START}{OSC8_END}") + } + _ => painted, + } +} diff --git a/crates/sqlbuild-core/src/screen_text/constants.rs b/crates/sqlbuild-core/src/screen_text/constants.rs new file mode 100644 index 0000000000..5fedecfaa5 --- /dev/null +++ b/crates/sqlbuild-core/src/screen_text/constants.rs @@ -0,0 +1,44 @@ +//! The glyphs, widths and escapes of live terminal screens. + +pub(crate) const OK_GLYPH: &str = "\u{2713}"; +pub(crate) const WARN_GLYPH: &str = "\u{26a0}"; +pub(crate) const FAIL_GLYPH: &str = "\u{2717}"; +pub(crate) const PENDING_GLYPH: &str = "\u{b7}"; +pub(crate) const CHANGED_GLYPH: &str = "~"; +pub(crate) const RESTARTED_GLYPH: &str = "\u{21bb}"; +pub(crate) const STARTED_GLYPH: &str = "\u{25b6}"; +pub(crate) const CARD_GLYPH: &str = "\u{279c}"; +pub(crate) const CONNECTED_GLYPH: &str = "\u{25cf}"; +pub(crate) const DISCONNECTED_GLYPH: &str = "\u{25cb}"; +pub(crate) const SPINNER_FRAMES: [&str; 10] = [ + "\u{280b}", "\u{2819}", "\u{2839}", "\u{2838}", "\u{283c}", "\u{2834}", "\u{2826}", "\u{2827}", + "\u{2807}", "\u{280f}", +]; +/// What replaces the cut tail of a row. +pub(crate) const CUT_MARK: &str = "\u{2026}"; +pub(crate) const SEPARATOR: &str = " \u{b7} "; +pub(crate) const RULE_CHAR: char = '\u{2500}'; +pub(crate) const TREE_BRANCH: &str = "\u{251c}\u{2500}\u{2500} "; +pub(crate) const TREE_LAST: &str = "\u{2514}\u{2500}\u{2500} "; +pub(crate) const TREE_BAR: &str = "\u{2502} "; +pub(crate) const TREE_SPACE: &str = " "; +pub(crate) const PRODUCT: &str = "sqlbuild dev"; +pub(crate) const PLAIN_PREFIX: &str = "sqlbuild dev: "; +/// The label column of a phase line. +pub(crate) const PHASE_LABEL_WIDTH: usize = 20; +/// The right-aligned time column of phase and event lines. +pub(crate) const TIME_WIDTH: usize = 8; +pub(crate) const EVENT_CLOCK_WIDTH: usize = 8; +pub(crate) const EVENT_TITLE_WIDTH: usize = 32; +pub(crate) const EVENT_DETAIL_WIDTH: usize = 18; +pub(crate) const CARD_LABEL_WIDTH: usize = 12; +/// Times under a minute show tenths of a second. +pub(crate) const TENTHS_PER_MINUTE: u128 = 600; +pub(crate) const SECONDS_PER_MINUTE: u64 = 60; +pub(crate) const SECONDS_PER_HOUR: u64 = 3600; +pub(crate) const OSC8_START: &str = "\u{1b}]8;;"; +pub(crate) const OSC8_END: &str = "\u{1b}\\"; +/// Moves the cursor up `n` rows to column one, then clears to the end of the screen. +pub(crate) const CURSOR_UP: &str = "\u{1b}["; +pub(crate) const CURSOR_UP_END: &str = "F"; +pub(crate) const CLEAR_BELOW: &str = "\u{1b}[J"; diff --git a/crates/sqlbuild-core/src/screen_text/main/format_plain_line.rs b/crates/sqlbuild-core/src/screen_text/main/format_plain_line.rs new file mode 100644 index 0000000000..4e8ea54dee --- /dev/null +++ b/crates/sqlbuild-core/src/screen_text/main/format_plain_line.rs @@ -0,0 +1,15 @@ +//! One stable line of the plain `sqb dev` output. + +use std::time::Duration; + +use crate::screen_text::_helpers::rows::elapsed_text; +use crate::screen_text::constants::PLAIN_PREFIX; + +/// `sqlbuild dev: `, followed by ` ()` when timed. +#[must_use] +pub fn format_plain_line(message: &str, elapsed: Option) -> String { + match elapsed { + Some(elapsed) => format!("{PLAIN_PREFIX}{message} ({})", elapsed_text(elapsed)), + None => format!("{PLAIN_PREFIX}{message}"), + } +} diff --git a/crates/sqlbuild-core/src/screen_text/main/mod.rs b/crates/sqlbuild-core/src/screen_text/main/mod.rs new file mode 100644 index 0000000000..b69430ad34 --- /dev/null +++ b/crates/sqlbuild-core/src/screen_text/main/mod.rs @@ -0,0 +1,10 @@ +pub mod format_plain_line; +pub mod redraw_rows; +pub mod render_dev_header; +pub mod render_event_line; +pub mod render_frame_rule; +pub mod render_hotkey_bar; +pub mod render_log_line; +pub mod render_phase_lines; +pub mod render_ready_card; +pub mod render_tree_group; diff --git a/crates/sqlbuild-core/src/screen_text/main/redraw_rows.rs b/crates/sqlbuild-core/src/screen_text/main/redraw_rows.rs new file mode 100644 index 0000000000..76fd5632b6 --- /dev/null +++ b/crates/sqlbuild-core/src/screen_text/main/redraw_rows.rs @@ -0,0 +1,20 @@ +//! Redraw a block of rows in place. + +use std::fmt::Write; + +use crate::screen_text::constants::{CLEAR_BELOW, CURSOR_UP, CURSOR_UP_END}; + +/// Move up over the `previous` rows, clear them, and draw `rows`, each already fitting one line. +#[must_use] +pub fn redraw_rows(previous: usize, rows: &[String]) -> String { + let mut text: String = String::new(); + if previous > 0 { + let _ = write!(text, "{CURSOR_UP}{previous}{CURSOR_UP_END}"); + } + text.push_str(CLEAR_BELOW); + for row in rows { + text.push_str(row); + text.push('\n'); + } + text +} diff --git a/crates/sqlbuild-core/src/screen_text/main/render_dev_header.rs b/crates/sqlbuild-core/src/screen_text/main/render_dev_header.rs new file mode 100644 index 0000000000..cd6f86734c --- /dev/null +++ b/crates/sqlbuild-core/src/screen_text/main/render_dev_header.rs @@ -0,0 +1,23 @@ +//! The first line of the `sqb dev` screen. + +use crate::screen_text::_helpers::rows::{render_row, span}; +use crate::screen_text::constants::{PRODUCT, SEPARATOR}; +use crate::screen_text::models::{DevHeader, ScreenFacts, Tone}; + +/// `sqlbuild dev v0.1.0 · shop · target dev`. +#[must_use] +pub fn render_dev_header(header: &DevHeader, facts: &ScreenFacts) -> String { + render_row( + vec![ + span(PRODUCT, Tone::Strong), + span( + format!( + " v{}{SEPARATOR}{}{SEPARATOR}target {}", + header.version, header.project, header.target + ), + Tone::Muted, + ), + ], + facts, + ) +} diff --git a/crates/sqlbuild-core/src/screen_text/main/render_event_line.rs b/crates/sqlbuild-core/src/screen_text/main/render_event_line.rs new file mode 100644 index 0000000000..10f89448d3 --- /dev/null +++ b/crates/sqlbuild-core/src/screen_text/main/render_event_line.rs @@ -0,0 +1,55 @@ +//! One timestamped event line of the `sqb dev` screen. + +use crate::screen_text::_helpers::rows::{column, elapsed_text, render_row, right, span}; +use crate::screen_text::constants::{ + CHANGED_GLYPH, EVENT_CLOCK_WIDTH, EVENT_DETAIL_WIDTH, EVENT_TITLE_WIDTH, FAIL_GLYPH, OK_GLYPH, + RESTARTED_GLYPH, STARTED_GLYPH, TIME_WIDTH, WARN_GLYPH, +}; +use crate::screen_text::models::{EventLine, EventMark, ScreenFacts, Segment, Tone}; + +/// `09:41:03 ✓ title detail 1.4s`, then each note indented under the title. +#[must_use] +pub fn render_event_line(event: &EventLine, facts: &ScreenFacts) -> Vec { + let (glyph, tone): (&str, Tone) = match event.mark { + EventMark::Changed => (CHANGED_GLYPH, Tone::Change), + EventMark::Succeeded => (OK_GLYPH, Tone::Success), + EventMark::Failed => (FAIL_GLYPH, Tone::Error), + EventMark::Restarted => (RESTARTED_GLYPH, Tone::Warning), + EventMark::Started => (STARTED_GLYPH, Tone::Link), + EventMark::Warning => (WARN_GLYPH, Tone::Warning), + }; + let title_tone: Tone = if event.mark == EventMark::Failed { + Tone::Error + } else { + Tone::Plain + }; + let mut segments: Vec = vec![ + span(format!("{} ", event.clock), Tone::Muted), + span(format!("{glyph} "), tone), + ]; + if event.detail.is_empty() && event.elapsed.is_none() { + segments.push(span(event.title.as_str(), title_tone)); + } else { + segments.push(span( + column(&event.title, EVENT_TITLE_WIDTH - 1), + title_tone, + )); + segments.push(span( + format!(" {}", column(&event.detail, EVENT_DETAIL_WIDTH - 1)), + Tone::Muted, + )); + let time: String = event.elapsed.map(elapsed_text).unwrap_or_default(); + segments.push(span( + format!(" {}", right(&time, TIME_WIDTH - 1)), + Tone::Muted, + )); + } + let mut rows: Vec = vec![render_row(segments, facts)]; + for note in &event.notes { + let mut note_segments: Vec = + vec![span(" ".repeat(EVENT_CLOCK_WIDTH + 2), Tone::Plain)]; + note_segments.extend(note.iter().cloned()); + rows.push(render_row(note_segments, facts)); + } + rows +} diff --git a/crates/sqlbuild-core/src/screen_text/main/render_frame_rule.rs b/crates/sqlbuild-core/src/screen_text/main/render_frame_rule.rs new file mode 100644 index 0000000000..aad507a9d7 --- /dev/null +++ b/crates/sqlbuild-core/src/screen_text/main/render_frame_rule.rs @@ -0,0 +1,20 @@ +//! The dim rule that frames a client's own terminal output. + +use crate::screen_text::_helpers::rows::{render_row, span}; +use crate::screen_text::constants::RULE_CHAR; +use crate::screen_text::models::{ScreenFacts, Tone}; +use crate::text::main::cell_len::cell_len; + +/// `──