diff --git a/README.md b/README.md index e465751..910da43 100644 --- a/README.md +++ b/README.md @@ -80,6 +80,11 @@ lest runs list - **CI:** `lest affected --base origin/main` lists the flows a change touches; webhook notifications (JSON or Slack-compatible) after each run. +- **Optional AI:** off by default. With a provider of your choice, failed + steps get a plain-words explanation next to their deterministic headline, + and Open in agent hands the failure to your own agent CLI. Nothing changes + in the layout either way. + ## Examples [`examples/`](examples) is a project with a small sample web app (Sprout, a @@ -100,6 +105,8 @@ lest run everything - [Browser steps](docs/browser.md) - [Demos](docs/demos.md) - [Running in CI](docs/ci.md) +- [UI](docs/ui.md) +- [AI (optional)](docs/ai.md) - [Design](docs/design.md): why Lest works the way it does - [Development](docs/development.md) diff --git a/crates/lest-core/src/ai.rs b/crates/lest-core/src/ai.rs new file mode 100644 index 0000000..3bfa453 --- /dev/null +++ b/crates/lest-core/src/ai.rs @@ -0,0 +1,682 @@ +//! Optional AI. Lest never needs it: every place a model can help already +//! has deterministic content, and a model only adds to it. +//! +//! - [`explain`]: a short explanation of a failed step. Without a provider, +//! or when the provider fails, the deterministic headline is the answer. +//! - [`handoff`]: the context and command to continue in the user's own +//! agent CLI, which has tools, permissions and memory. Lest builds no chat. +//! +//! A provider is one headless call: prompt on stdin, text on stdout, no +//! tools, an empty working directory, and a hard timeout. + +use std::io::Write; +use std::path::{Path, PathBuf}; +use std::time::Duration; + +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; + +use crate::report::{RunReport, StepReport}; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Serialize, Deserialize, JsonSchema)] +#[serde(rename_all = "lowercase")] +pub enum ProviderKind { + /// No AI (the default). + #[default] + None, + /// The first agent CLI found on PATH (`claude`, then `codex`). + Auto, + Claude, + Codex, + /// Any program that reads a prompt on stdin and prints text. + Command, +} + +#[derive(Debug, Clone, Default, Serialize, Deserialize, JsonSchema)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct AiConfig { + #[serde(default)] + pub provider: ProviderKind, + /// Model name passed to the provider (default: its small model). + #[serde(default, skip_serializing_if = "Option::is_none")] + pub model: Option, + /// For `provider: command`: the program and arguments (run with `sh -c`). + #[serde(default, skip_serializing_if = "Option::is_none")] + pub command: Option, + /// The interactive agent to hand off to; `{prompt}` is replaced with the + /// quoted prompt (default: the provider's CLI). + #[serde(default, skip_serializing_if = "Option::is_none")] + pub agent: Option, + /// Model calls allowed per day (default 50). + #[serde(default, skip_serializing_if = "Option::is_none")] + pub daily_limit: Option, +} + +/// The provider a config resolves to on this machine. +#[derive(Debug, Clone, PartialEq, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct Resolved { + pub kind: ProviderKind, + /// A short label for settings: `claude`, `codex`, `command`. + pub label: String, + pub model: Option, +} + +/// What the settings screen shows: the configured provider, what it +/// resolves to, and which agent CLIs exist on this machine. +#[derive(Debug, Clone, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct Status { + pub configured: ProviderKind, + pub resolved: Option, + pub available: Vec, + pub calls_today: u32, + pub daily_limit: u32, +} + +fn on_path(bin: &str) -> bool { + which::which(bin).is_ok() +} + +pub fn resolve(cfg: &AiConfig) -> Option { + let make = |kind, label: &str| Some(Resolved { kind, label: label.to_string(), model: cfg.model.clone() }); + match cfg.provider { + ProviderKind::None => None, + ProviderKind::Auto => { + if on_path("claude") { + make(ProviderKind::Claude, "claude") + } else if on_path("codex") { + make(ProviderKind::Codex, "codex") + } else { + None + } + } + ProviderKind::Claude => on_path("claude").then(|| make(ProviderKind::Claude, "claude")).flatten(), + ProviderKind::Codex => on_path("codex").then(|| make(ProviderKind::Codex, "codex")).flatten(), + // A custom command is labeled by its program's name. + ProviderKind::Command => cfg.command.as_ref().and_then(|c| { + let program = c.split_whitespace().next().unwrap_or("command"); + let name = std::path::Path::new(program).file_name().map(|n| n.to_string_lossy().into_owned()); + make(ProviderKind::Command, name.as_deref().unwrap_or("command")) + }), + } +} + +pub fn status(cfg: &AiConfig, data_dir: &Path) -> Status { + Status { + configured: cfg.provider, + resolved: resolve(cfg), + available: ["claude", "codex"].into_iter().filter(|b| on_path(b)).map(str::to_string).collect(), + calls_today: Ledger::new(data_dir).calls_today(), + daily_limit: cfg.daily_limit.unwrap_or(50), + } +} + +/// Most a prompt may hold; the context is cut to fit. +pub const MAX_PROMPT: usize = 48 * 1024; + +/// Variables that make an agent CLI believe it runs inside another agent's +/// session. Only these are removed; configuration and credentials stay, so +/// any way the user signs in to their CLI keeps working. +const SESSION_MARKERS: &[&str] = &[ + "CLAUDECODE", + "CLAUDE_CODE_ENTRYPOINT", + "CLAUDE_CODE_SSE_PORT", + "CODEX_SANDBOX", + "CODEX_SANDBOX_NETWORK_DISABLED", +]; + +/// One headless call: the prompt on stdin, the answer on stdout. `remove` +/// names environment variables the provider must not see (the env-backend +/// secrets). The whole call, writing the prompt included, is bounded by +/// `timeout`, and a timeout kills the provider's process group. +pub fn ask( + cfg: &AiConfig, + resolved: &Resolved, + system: &str, + input: &str, + timeout: Duration, + remove: &[String], +) -> Result { + let dir = tempdir()?; + let out_file = dir.join("answer.txt"); + let mut cmd = match resolved.kind { + ProviderKind::Claude => { + let mut c = std::process::Command::new("claude"); + c.args([ + "-p", + "--output-format", + "text", + "--no-session-persistence", + // No tools, no MCP servers, no user customizations: one + // answer from the prompt alone. + "--tools", + "", + "--strict-mcp-config", + "--safe-mode", + "--disable-slash-commands", + "--system-prompt", + system, + "--model", + cfg.model.as_deref().unwrap_or("haiku"), + ]); + c + } + ProviderKind::Codex => { + let mut c = std::process::Command::new("codex"); + // A read-only sandbox without the user's config, rules or MCP + // servers. Codex can still read files to answer; docs/ai.md says + // so. + c.args([ + "exec", + "--ephemeral", + "--skip-git-repo-check", + "--ignore-user-config", + "--ignore-rules", + "-s", + "read-only", + "-o", + ]) + .arg(&out_file) + .args(cfg.model.as_deref().map(|m| vec!["-m", m]).unwrap_or_default()) + // Codex has no system-prompt flag; the instruction leads the prompt. + .arg(format!("{system}\n\nThe input follows on stdin.")); + c + } + ProviderKind::Command => { + let mut c = std::process::Command::new("sh"); + c.arg("-c").arg(cfg.command.as_deref().unwrap_or("false")); + c + } + _ => return Err("no provider".into()), + }; + for k in SESSION_MARKERS.iter().map(|s| s.to_string()).chain(remove.iter().cloned()) { + cmd.env_remove(k); + } + let mut stdin_text = + if resolved.kind == ProviderKind::Command { format!("{system}\n\n{input}") } else { input.to_string() }; + if stdin_text.len() > MAX_PROMPT { + let mut cut = MAX_PROMPT; + while !stdin_text.is_char_boundary(cut) { + cut -= 1; + } + stdin_text.truncate(cut); + stdin_text.push_str("\n[cut to fit]\n"); + } + cmd.current_dir(&dir) + .stdin(std::process::Stdio::piped()) + .stdout(std::process::Stdio::piped()) + .stderr(std::process::Stdio::piped()); + #[cfg(unix)] + { + use std::os::unix::process::CommandExt; + cmd.process_group(0); + } + let mut child = cmd.spawn().map_err(|e| format!("cannot start {}: {e}", resolved.label))?; + // The prompt is written on its own thread: a provider that never reads + // stdin must not stall us past the timeout. + if let Some(mut stdin) = child.stdin.take() { + std::thread::spawn(move || { + let _ = stdin.write_all(stdin_text.as_bytes()); + }); + } + let pid = child.id(); + let (tx, rx) = std::sync::mpsc::channel(); + std::thread::spawn(move || { + let _ = tx.send(child.wait_with_output()); + }); + let output = match rx.recv_timeout(timeout) { + Ok(r) => r.map_err(|e| e.to_string())?, + Err(_) => { + #[cfg(unix)] + // SAFETY: signals the process group of the provider we spawned + // (process_group(0) made it the leader). + unsafe { + libc::kill(-(pid as i32), libc::SIGKILL); + } + let _ = std::fs::remove_dir_all(&dir); + return Err(format!("{} did not answer within {}s", resolved.label, timeout.as_secs())); + } + }; + let text = if resolved.kind == ProviderKind::Codex { + std::fs::read_to_string(&out_file).unwrap_or_default() + } else { + String::from_utf8_lossy(&output.stdout).into_owned() + }; + let _ = std::fs::remove_dir_all(&dir); + if !output.status.success() { + let err = String::from_utf8_lossy(&output.stderr); + return Err(format!( + "{} exited {}: {}", + resolved.label, + output.status.code().unwrap_or(-1), + err.lines().next().unwrap_or("").trim() + )); + } + let text = text.trim().to_string(); + if text.is_empty() { Err(format!("{} returned nothing", resolved.label)) } else { Ok(text) } +} + +fn tempdir() -> Result { + let dir = std::env::temp_dir().join(format!("lest-ai-{}-{}", std::process::id(), rand::random::())); + std::fs::create_dir_all(&dir).map_err(|e| e.to_string())?; + Ok(dir) +} + +/// Calls made per day, so a configured provider cannot run up cost. +struct Ledger { + path: PathBuf, +} + +impl Ledger { + fn new(data_dir: &Path) -> Self { + Ledger { path: data_dir.join("ai").join("usage.jsonl") } + } + fn today() -> String { + time::OffsetDateTime::now_utc().date().to_string() + } + fn calls_today(&self) -> u32 { + let today = Self::today(); + std::fs::read_to_string(&self.path) + .unwrap_or_default() + .lines() + .filter(|l| l.contains(&format!("\"day\":\"{today}\""))) + .count() as u32 + } + fn record(&self, provider: &str) { + if let Some(p) = self.path.parent() { + let _ = crate::paths::ensure_private_dir(p); + } + if let Ok(mut f) = std::fs::OpenOptions::new().create(true).append(true).open(&self.path) { + let _ = writeln!(f, "{}", serde_json::json!({"day": Self::today(), "provider": provider})); + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[serde(rename_all = "lowercase")] +pub enum Source { + /// Built from the step's own output, with no model involved. + Lest, + Model, +} + +#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)] +#[serde(rename_all = "camelCase")] +pub struct Explanation { + pub text: String, + pub source: Source, + /// The provider that wrote it, for `source: model`. + #[serde(skip_serializing_if = "Option::is_none")] + pub provider: Option, + /// Why the model was not used, when it was configured but failed. + #[serde(skip_serializing_if = "Option::is_none")] + pub note: Option, +} + +const EXPLAIN_SYSTEM: &str = "You explain why a step of an automated end-to-end test failed. \ +You get the step definition, its command, its output and the run's context. \ +Answer in at most three short sentences of plain text: the most likely cause, then the single next action to confirm or fix it. \ +Name the concrete thing (the status code, the missing value, the command). Do not speculate beyond the evidence; \ +if the output does not show the cause, say what to look at."; + +/// Explains a failed step: the deterministic headline, or a model's +/// explanation when a provider is configured, answers in time, and is under +/// the daily limit. Cached by input. +pub fn explain( + cfg: &AiConfig, + data_dir: &Path, + report: &RunReport, + step_id: &str, + flow_source: Option<&str>, + remove: &[String], +) -> Explanation { + let Some(step) = report.find_step(step_id) else { + return Explanation { + text: format!("no step {step_id} in this run"), + source: Source::Lest, + provider: None, + note: None, + }; + }; + let deterministic = Explanation { + text: step + .headline + .clone() + .or_else(|| step.error.clone()) + .unwrap_or_else(|| "This step did not fail.".to_string()), + source: Source::Lest, + provider: None, + note: None, + }; + let Some(resolved) = resolve(cfg) else { return deterministic }; + if !step.status.is_some_and(|s| s.is_failure()) { + return deterministic; + } + let input = context(report, step, flow_source); + // The full configuration is part of the key: a different command or + // model never reuses another's answer. + let key = hex::encode(Sha256::digest( + format!( + "{}\n{}\n{}\n{input}", + resolved.label, + cfg.command.as_deref().unwrap_or(""), + cfg.model.as_deref().unwrap_or("") + ) + .as_bytes(), + )); + let cache = data_dir.join("ai").join("explain").join(format!("{}.json", &key[..32])); + if let Some(e) = std::fs::read(&cache).ok().and_then(|b| serde_json::from_slice::(&b).ok()) { + return e; + } + let ledger = Ledger::new(data_dir); + let limit = cfg.daily_limit.unwrap_or(50); + { + // Check and record together, so concurrent requests in this process + // cannot both take the last call. + static LEDGER: std::sync::Mutex<()> = std::sync::Mutex::new(()); + let _guard = LEDGER.lock().unwrap_or_else(|e| e.into_inner()); + if ledger.calls_today() >= limit { + return Explanation { note: Some(format!("daily limit of {limit} model calls reached")), ..deterministic }; + } + ledger.record(&resolved.label); + } + match ask(cfg, &resolved, EXPLAIN_SYSTEM, &input, Duration::from_secs(90), remove) { + Ok(text) => { + let e = Explanation { text, source: Source::Model, provider: Some(resolved.label.clone()), note: None }; + if let Ok(bytes) = serde_json::to_vec(&e) { + let _ = crate::paths::write_private(&cache, &bytes); + } + e + } + Err(err) => Explanation { note: Some(err), ..deterministic }, + } +} + +fn tail(text: &str, lines: usize, max_chars: usize) -> String { + let all: Vec<&str> = text.lines().collect(); + let t = all[all.len().saturating_sub(lines)..].join("\n"); + if t.chars().count() > max_chars { + t.chars().rev().take(max_chars).collect::>().into_iter().rev().collect() + } else { + t + } +} + +/// Everything someone (or an agent) needs to pick up a failure: where the +/// flow and report are, what failed, the output's tail, and how to rerun. +/// Built only from the redacted report. +pub fn context(report: &RunReport, step: &StepReport, flow_source: Option<&str>) -> String { + let mut s = String::new(); + s.push_str(&format!("Flow: {} ({})\n", report.flow_name, report.flow_id)); + s.push_str(&format!("Flow file: {}/{}\n", report.project_dir, report.flow_path)); + s.push_str(&format!("Run: {} ({}", report.run_id, report.result.as_str())); + if let Some(env) = &report.environment { + s.push_str(&format!(", environment {env}")); + } + s.push_str(")\n"); + if !report.inputs.is_empty() { + let inputs: Vec = report.inputs.iter().map(|(k, v)| format!("{k}={v}")).collect(); + s.push_str(&format!("Inputs: {}\n", inputs.join(" "))); + } + s.push_str(&format!("\nFailed step: {} ({})\n", step.id, step.name)); + if let Some(k) = step.kind { + s.push_str(&format!("Kind: {}\n", k.as_str())); + } + if let Some(r) = &step.resolved { + s.push_str(&format!("Ran: {r}\n")); + } + if let Some(h) = &step.headline { + s.push_str(&format!("Headline: {h}\n")); + } + if let Some(e) = step.error.as_ref().filter(|e| Some(*e) != step.headline.as_ref()) { + s.push_str(&format!("Error: {e}\n")); + } + if let Some(c) = step.exit_code { + s.push_str(&format!("Exit code: {c}\n")); + } + if let Some(c) = step.http_status { + s.push_str(&format!("HTTP status: {c}\n")); + } + s.push_str(&format!("Attempts: {}, duration {} ms\n", step.attempts, step.duration_ms)); + if let Some(n) = &step.notes { + s.push_str(&format!("Author's notes on this step: {n}\n")); + } + if !step.stderr.trim().is_empty() { + s.push_str(&format!("\nstderr (end):\n{}\n", tail(&step.stderr, 40, 4000))); + } + if !step.stdout.trim().is_empty() { + s.push_str(&format!("\nstdout (end):\n{}\n", tail(&step.stdout, 40, 4000))); + } + let earlier: Vec = report + .all_steps() + .into_iter() + .take_while(|x| x.id != step.id) + .filter(|x| x.children.is_empty()) + .map(|x| format!("{} {} ({} ms)", x.id, x.status.map(|s| s.as_str()).unwrap_or("-"), x.duration_ms)) + .collect(); + if !earlier.is_empty() { + s.push_str(&format!("\nEarlier steps: {}\n", earlier.join(", "))); + } + if let Some(src) = flow_source { + s.push_str(&format!("\nFlow definition:\n{}\n", tail(src, 400, 16000))); + } + let top = report.steps.iter().find(|t| t.id == step.id || contains(t, &step.id)).map(|t| t.id.clone()); + s.push_str(&format!( + "\nRerun: lest run {}{}\nFull report: lest runs get {} -o report.json\n", + report.flow_id, + top.map(|t| format!(" --from {t}")).unwrap_or_default(), + report.run_id + )); + s +} + +fn contains(s: &StepReport, id: &str) -> bool { + s.children.iter().any(|c| c.id == id || contains(c, id)) +} + +/// How to continue in the user's own agent. +#[derive(Debug, Clone, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct Handoff { + /// The text to paste anywhere (always available). + pub context: String, + /// The prompt given to the agent. + pub prompt: String, + /// A shell command that starts the agent in the project, when an agent + /// is configured. + pub command: Option, +} + +fn shell_quote(s: &str) -> String { + format!("'{}'", s.replace('\'', "'\\''")) +} + +pub fn handoff(cfg: &AiConfig, report: &RunReport, step_id: Option<&str>) -> Handoff { + let failed = step_id.and_then(|id| report.find_step(id)).or_else(|| report.first_failure()); + let context = match failed { + Some(step) => context(report, step, None), + None => format!( + "Flow: {} ({})\nFlow file: {}/{}\nRun: {} ({})\nRerun: lest run {}\n", + report.flow_name, + report.flow_id, + report.project_dir, + report.flow_path, + report.run_id, + report.result.as_str(), + report.flow_id + ), + }; + let prompt = match failed { + Some(step) => format!( + "The Lest flow {} failed at step {} in run {}. Read the flow at {}/{} and the run with: lest runs get {} -o report.json. \ +Find the cause, and fix the flow or tell me what is wrong with the system under test. The command lest docs agent explains the flow format. \ +Rerun with: lest run {}", + report.flow_id, step.id, report.run_id, report.project_dir, report.flow_path, report.run_id, report.flow_id + ), + None => format!( + "Look at the Lest flow {} ({}/{}). The command lest docs agent explains the flow format.", + report.flow_id, report.project_dir, report.flow_path + ), + }; + let agent = cfg.agent.clone().or_else(|| match resolve(cfg).map(|r| r.kind) { + Some(ProviderKind::Claude) => Some("claude {prompt}".to_string()), + Some(ProviderKind::Codex) => Some("codex {prompt}".to_string()), + _ => None, + }); + // The prompt reaches the agent through an environment variable, so a + // template such as `agent "{prompt}"` cannot turn its text into shell + // code. + let command = agent.map(|a| { + let cmd = a + .replace("\"{prompt}\"", "\"$LEST_AGENT_PROMPT\"") + .replace("'{prompt}'", "\"$LEST_AGENT_PROMPT\"") + .replace("{prompt}", "\"$LEST_AGENT_PROMPT\""); + format!("cd {} && LEST_AGENT_PROMPT={} {cmd}", shell_quote(&report.project_dir), shell_quote(&prompt)) + }); + Handoff { context, prompt, command } +} + +/// The user's settings file (`/config.yml`). +#[derive(Debug, Clone, Default, Serialize, Deserialize, JsonSchema)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct UserConfig { + #[serde(default)] + pub ai: AiConfig, +} + +impl UserConfig { + pub fn load(path: &Path) -> anyhow::Result { + match std::fs::read_to_string(path) { + Ok(t) if t.trim().is_empty() => Ok(UserConfig::default()), + Ok(t) => serde_yaml_ng::from_str(&t).map_err(|e| anyhow::anyhow!("{}: {e}", path.display())), + Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(UserConfig::default()), + Err(e) => Err(anyhow::anyhow!("{}: {e}", path.display())), + } + } + + pub fn save(&self, path: &Path) -> anyhow::Result<()> { + let text = serde_yaml_ng::to_string(self)?; + crate::paths::write_private(path, text.as_bytes())?; + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::report::*; + + fn report() -> RunReport { + let failed = StepReport { + id: "fetch".into(), + name: "Fetch the user".into(), + kind: Some(crate::spec::StepKind::Http), + status: Some(StepStatus::Failed), + attempts: 1, + http_status: Some(404), + resolved: Some("GET http://127.0.0.1:1/users/9".into()), + headline: Some("HTTP 404 from GET http://127.0.0.1:1/users/9".into()), + stdout: r#"{"error":"no such user"}"#.into(), + ..Default::default() + }; + RunReport { + schema_version: 1, + run_id: "r1".into(), + flow_id: "users".into(), + flow_name: "Users".into(), + flow_path: "flows/users.lest.yaml".into(), + flow_sha256: Default::default(), + project_dir: "/work/app".into(), + environment: Some("local".into()), + started_at: String::new(), + finished_at: None, + duration_ms: 1, + result: RunResult::Failed, + error: None, + warnings: vec![], + inputs: Default::default(), + host: HostInfo::current(), + tools: vec![], + steps: vec![ + StepReport { id: "setup".into(), status: Some(StepStatus::Passed), ..Default::default() }, + failed, + ], + finally: vec![], + cleanups: vec![], + services: vec![], + resumed_from: None, + demo: None, + } + } + + #[test] + fn without_a_provider_the_headline_is_the_explanation() { + let dir = tempfile::tempdir().unwrap(); + let e = explain(&AiConfig::default(), dir.path(), &report(), "fetch", None, &[]); + assert_eq!(e.source, Source::Lest); + assert_eq!(e.text, "HTTP 404 from GET http://127.0.0.1:1/users/9"); + } + + #[test] + fn a_command_provider_answers_and_is_cached() { + let dir = tempfile::tempdir().unwrap(); + let counter = dir.path().join("calls"); + let cfg = AiConfig { + provider: ProviderKind::Command, + command: Some(format!("echo x >> {}; echo 'The user does not exist; create it first.'", counter.display())), + ..Default::default() + }; + let e = explain(&cfg, dir.path(), &report(), "fetch", None, &[]); + assert_eq!(e.source, Source::Model, "{e:?}"); + assert_eq!(e.text, "The user does not exist; create it first."); + let again = explain(&cfg, dir.path(), &report(), "fetch", None, &[]); + assert_eq!(again.text, e.text); + assert_eq!(std::fs::read_to_string(&counter).unwrap().lines().count(), 1); + } + + #[test] + fn a_failing_provider_falls_back_with_a_note() { + let dir = tempfile::tempdir().unwrap(); + let cfg = AiConfig { provider: ProviderKind::Command, command: Some("exit 7".into()), ..Default::default() }; + let e = explain(&cfg, dir.path(), &report(), "fetch", None, &[]); + assert_eq!(e.source, Source::Lest); + assert!(e.note.unwrap().contains("exited 7")); + } + + #[test] + fn a_provider_that_never_reads_is_stopped_on_time() { + let cfg = AiConfig { provider: ProviderKind::Command, command: Some("sleep 30".into()), ..Default::default() }; + let r = resolve(&cfg).unwrap(); + let started = std::time::Instant::now(); + let big = "x".repeat(400_000); + let e = ask(&cfg, &r, "sys", &big, Duration::from_secs(1), &[]).unwrap_err(); + assert!(e.contains("did not answer"), "{e}"); + assert!(started.elapsed() < Duration::from_secs(5), "{:?}", started.elapsed()); + } + + #[test] + fn context_names_files_failure_and_rerun() { + let r = report(); + let c = context(&r, r.find_step("fetch").unwrap(), None); + assert!(c.contains("Flow file: /work/app/flows/users.lest.yaml")); + assert!(c.contains("HTTP status: 404")); + assert!(c.contains("lest run users --from fetch")); + assert!(c.contains("Earlier steps: setup passed")); + } + + #[test] + fn handoff_has_a_command_only_with_an_agent() { + let r = report(); + assert!(handoff(&AiConfig::default(), &r, Some("fetch")).command.is_none()); + let cfg = AiConfig { agent: Some("my-agent --ask {prompt}".into()), ..Default::default() }; + let h = handoff(&cfg, &r, Some("fetch")); + let cmd = h.command.unwrap(); + assert!(cmd.starts_with("cd '/work/app' && LEST_AGENT_PROMPT='The Lest flow users failed"), "{cmd}"); + assert!(cmd.ends_with("my-agent --ask \"$LEST_AGENT_PROMPT\""), "{cmd}"); + // A quoted placeholder stays safe. + let quoted = AiConfig { agent: Some("agent \"{prompt}\"".into()), ..Default::default() }; + assert!(handoff("ed, &r, Some("fetch")).command.unwrap().ends_with("agent \"$LEST_AGENT_PROMPT\"")); + } +} diff --git a/crates/lest-core/src/docs.rs b/crates/lest-core/src/docs.rs new file mode 100644 index 0000000..a0e6c46 --- /dev/null +++ b/crates/lest-core/src/docs.rs @@ -0,0 +1,49 @@ +//! Documentation for coding agents (`lest docs agent`): how to write and +//! fix flows, printed by the installed version so it always matches. + +const GUIDE: &str = r#"# Lest: a guide for coding agents + +Lest runs end-to-end flows (`*.lest.yaml`) against real systems and records +demos from them. Use this guide to write, run and fix flows. + +## Working loop + +1. Find flows: `lest list` (ids, names, paths, last result). +2. Read a flow: `lest get ` or open its file. +3. Validate after every edit: `lest validate` (exit 0 means valid). +4. Run: `lest run `; rerun from a failed top-level step with + `lest run --from `. +5. Read results: `lest runs list`, `lest runs get `, or the full report + with `lest runs get -o report.json`. +6. Explain a failure: `lest explain ` and `lest context `. + +## Rules that matter + +- Step ids are `[a-z][a-z0-9_]*`; later steps read `steps..outputs.`. +- `run:` scripts must not contain `${{ }}`: pass values through `env:` and + read `$NAME`. +- Expressions are CEL. Use raw strings for regexes: `self.stdout.capture(r'id=(\w+)')`. +- Secrets: list them under `secrets:` and use `${{ secrets.x }}` only in + `env:`, `http`, browser values and `cleanup.env`. +- Poll with `retry: {attempts, delay, until}`; do not lengthen waits to hide + slowness. A slow or failing step is information: find out why. +- Register cleanup on the step that creates something (`cleanup:`). +- Never weaken an assertion to make a flow pass when it caught a real problem. + +The full references follow. +"#; + +const FLOW_FORMAT: &str = include_str!("../../../docs/flow-format.md"); +const BROWSER: &str = include_str!("../../../docs/browser.md"); +const DEMOS: &str = include_str!("../../../docs/demos.md"); +const CI: &str = include_str!("../../../docs/ci.md"); + +/// The agent guide: rules first, then the reference docs. +pub fn agent_guide() -> String { + let mut s = String::from(GUIDE); + for doc in [FLOW_FORMAT, BROWSER, DEMOS, CI] { + s.push_str("\n\n---\n\n"); + s.push_str(doc); + } + s +} diff --git a/crates/lest-core/src/lib.rs b/crates/lest-core/src/lib.rs index c4cf656..b8f1f0d 100644 --- a/crates/lest-core/src/lib.rs +++ b/crates/lest-core/src/lib.rs @@ -1,9 +1,11 @@ //! Lest core: the flow format, validation, the runner and run reports. pub mod affected; +pub mod ai; pub mod browser; pub mod bundle; pub mod catalog; +pub mod docs; pub mod event; pub mod expr; pub mod headline; diff --git a/crates/lest-server/src/api.rs b/crates/lest-server/src/api.rs index 401d17f..66bf94d 100644 --- a/crates/lest-server/src/api.rs +++ b/crates/lest-server/src/api.rs @@ -59,6 +59,11 @@ pub(crate) fn routes() -> Router> { .route("/runs/{id}/files/{*path}", get(run_file)) .route("/runs/{id}/reveal", post(reveal)) .route("/tools/{name}/login", post(tool_login)) + .route("/settings", get(settings)) + .route("/settings/ai", axum::routing::put(save_ai)) + .route("/runs/{id}/explain", post(explain)) + .route("/runs/{id}/handoff", get(handoff)) + .route("/runs/{id}/agent", post(launch_agent)) .route("/events", get(events)) } @@ -497,6 +502,105 @@ async fn tool_login(State(s): State>, Path(name): Path) -> } } +fn user_config(s: &AppState) -> AppResult { + lest_core::ai::UserConfig::load(&s.roots.config_file()) + .map_err(|e| ApiError(StatusCode::INTERNAL_SERVER_ERROR, e.to_string())) +} + +#[derive(Serialize)] +struct SettingsResponse { + ai: lest_core::ai::AiConfig, + status: lest_core::ai::Status, +} + +async fn settings(State(s): State>) -> AppResult> { + let cfg = user_config(&s)?; + let status = lest_core::ai::status(&cfg.ai, &s.roots.data_dir); + Ok(Json(SettingsResponse { ai: cfg.ai, status })) +} + +async fn save_ai( + State(s): State>, + Json(ai): Json, +) -> AppResult> { + let mut cfg = user_config(&s)?; + cfg.ai = ai; + cfg.save(&s.roots.config_file()).map_err(|e| ApiError(StatusCode::INTERNAL_SERVER_ERROR, e.to_string()))?; + settings(State(s)).await +} + +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +struct StepBody { + step_id: Option, +} + +fn stored_report(s: &AppState, id: &str) -> AppResult { + s.store.find(id).map_err(|e| not_found(e.to_string())) +} + +/// A failed step's explanation: the deterministic headline, or a model's +/// when the user configured one. Never blocks a run; computed on request. +async fn explain( + State(s): State>, + Path(id): Path, + body: Option>, +) -> AppResult> { + let report = stored_report(&s, &id)?; + let step_id = match body.and_then(|b| b.0.step_id) { + Some(id) => id, + None => report.first_failure().map(|f| f.id.clone()).ok_or_else(|| not_found("the run has no failed step"))?, + }; + let cfg = user_config(&s)?; + let data_dir = s.roots.data_dir.clone(); + let source = std::fs::read_to_string(std::path::Path::new(&report.project_dir).join(&report.flow_path)).ok(); + let remove = lest_core::secrets::secret_env_names(&s.project.read().expect("lock").secrets_config()); + let e = tokio::task::spawn_blocking(move || { + lest_core::ai::explain(&cfg.ai, &data_dir, &report, &step_id, source.as_deref(), &remove) + }) + .await + .map_err(|e| ApiError(StatusCode::INTERNAL_SERVER_ERROR, e.to_string()))?; + Ok(Json(e)) +} + +#[derive(Deserialize)] +struct StepQuery { + step: Option, +} + +async fn handoff( + State(s): State>, + Path(id): Path, + Query(q): Query, +) -> AppResult> { + let report = stored_report(&s, &id)?; + let cfg = user_config(&s)?; + Ok(Json(lest_core::ai::handoff(&cfg.ai, &report, q.step.as_deref()))) +} + +/// Opens the user's agent CLI in a terminal, in the project, with the +/// failure's context. The command is built here, never taken from the +/// request. +async fn launch_agent( + State(s): State>, + Path(id): Path, + body: Option>, +) -> AppResult> { + let report = stored_report(&s, &id)?; + let cfg = user_config(&s)?; + let h = lest_core::ai::handoff(&cfg.ai, &report, body.and_then(|b| b.0.step_id).as_deref()); + let Some(command) = h.command else { + return Err(bad_request( + "no agent is configured: set Agent command in Settings, or `lest config set ai.agent ' {prompt}'`", + )); + }; + let cwd = std::path::PathBuf::from(&report.project_dir); + match crate::terminal::open(&command, &cwd) { + Ok(how) => Ok(Json(json!({"launched": true, "how": how, "command": command}))), + Err(e) => Ok(Json(json!({"launched": false, "error": e, "command": command}))), + } +} + async fn events(State(s): State>) -> Sse>> { let rx = s.events.subscribe(); let stream = BroadcastStream::new(rx).map(|item| { diff --git a/crates/lest-server/src/lib.rs b/crates/lest-server/src/lib.rs index dfd7123..6bcce3c 100644 --- a/crates/lest-server/src/lib.rs +++ b/crates/lest-server/src/lib.rs @@ -66,6 +66,7 @@ pub(crate) struct AppState { /// The cookie's name carries the port, so two `lest ui` processes do /// not overwrite each other's token. pub cookie: String, + pub roots: StateRoots, pub store: Store, pub keyring: Arc, pub browser: Arc, @@ -104,7 +105,8 @@ pub async fn bind(opts: Options) -> anyhow::Result { let (events, _) = broadcast::channel(4096); let state = Arc::new(AppState { project: RwLock::new(opts.project), - store: Store::new(opts.roots), + store: Store::new(opts.roots.clone()), + roots: opts.roots, keyring: opts.keyring, cookie: format!("lest_token_{}", addr.port()), // The live view may run from either loopback name. diff --git a/crates/lest/src/args.rs b/crates/lest/src/args.rs index b2b37f7..45ab24d 100644 --- a/crates/lest/src/args.rs +++ b/crates/lest/src/args.rs @@ -54,6 +54,20 @@ pub enum Command { }, /// Open the UI. Ui(UiArgs), + /// Explain why a run's step failed. + Explain(ExplainArgs), + /// Print everything needed to pick up a failed run, ready to paste. + Context(ExplainArgs), + /// Print documentation. + Docs { + #[command(subcommand)] + command: DocsCommand, + }, + /// Show or change settings. + Config { + #[command(subcommand)] + command: ConfigCommand, + }, /// Remove stored run data. Data { #[command(subcommand)] @@ -95,6 +109,31 @@ pub struct RunArgs { pub quiet: bool, } +#[derive(Debug, Args)] +pub struct ExplainArgs { + /// Run id or unique prefix. + pub run: String, + /// The step (default: the first failed step). + #[arg(long)] + pub step: Option, +} + +#[derive(Debug, Subcommand)] +pub enum DocsCommand { + /// The flow authoring guide and references, for coding agents. + Agent, +} + +#[derive(Debug, Subcommand)] +pub enum ConfigCommand { + /// Print the settings and the AI provider they resolve to. + Show, + /// Change a setting: ai.provider (none, auto, claude, codex, command), + /// ai.model, ai.command, ai.agent, ai.dailyLimit. An empty value clears + /// it. + Set { key: String, value: String }, +} + #[derive(Debug, Args)] pub struct UiArgs { /// Port to listen on (default: any free port). diff --git a/crates/lest/src/main.rs b/crates/lest/src/main.rs index e46a670..23c46c6 100644 --- a/crates/lest/src/main.rs +++ b/crates/lest/src/main.rs @@ -117,6 +117,13 @@ async fn dispatch(cli: Cli, style: Style) -> Result { } Command::Data { command } => cmd_data(&ctx, command), Command::Ui(a) => cmd_ui(&ctx, a).await, + Command::Explain(a) => cmd_explain(&ctx, a, false), + Command::Context(a) => cmd_explain(&ctx, a, true), + Command::Docs { command: DocsCommand::Agent } => { + print!("{}", lest_core::docs::agent_guide()); + Ok(0) + } + Command::Config { command } => cmd_config(&ctx, command), } } @@ -875,6 +882,90 @@ fn cmd_secrets(ctx: &Ctx, command: SecretsCommand) -> Result { } } +fn user_config(ctx: &Ctx) -> Result { + lest_core::ai::UserConfig::load(&ctx.roots.config_file()).map_err(|e| fail(exit::ERRORED, e.to_string())) +} + +fn cmd_explain(ctx: &Ctx, a: ExplainArgs, context_only: bool) -> Result { + let report = ctx.store().find(&a.run).map_err(|e| fail(exit::NOT_FOUND, e.to_string()))?; + let step = match &a.step { + Some(id) => report + .find_step(id) + .ok_or_else(|| fail(exit::NOT_FOUND, format!("no step {id} in run {}", report.run_id)))?, + None => report + .first_failure() + .ok_or_else(|| fail(exit::NOT_FOUND, format!("run {} has no failed step", report.run_id)))?, + }; + let flow_source = std::fs::read_to_string(std::path::Path::new(&report.project_dir).join(&report.flow_path)).ok(); + if context_only { + print!("{}", lest_core::ai::context(&report, step, flow_source.as_deref())); + return Ok(0); + } + let cfg = user_config(ctx)?; + let remove = lest_core::secrets::secret_env_names(&ctx.project.secrets_config()); + let e = lest_core::ai::explain(&cfg.ai, &ctx.roots.data_dir, &report, &step.id, flow_source.as_deref(), &remove); + println!("{}", e.text); + match e.source { + lest_core::ai::Source::Model => eprintln!("(written by {})", e.provider.as_deref().unwrap_or("a model")), + lest_core::ai::Source::Lest => {} + } + if let Some(n) = e.note { + eprintln!("note: {n}"); + } + Ok(0) +} + +fn cmd_config(ctx: &Ctx, command: ConfigCommand) -> Result { + let path = ctx.roots.config_file(); + let mut cfg = user_config(ctx)?; + match command { + ConfigCommand::Show => { + println!("Config: {}", path.display()); + println!("Data: {}", ctx.roots.data_dir.display()); + let status = lest_core::ai::status(&cfg.ai, &ctx.roots.data_dir); + println!("ai.provider: {}", serde_json::to_value(cfg.ai.provider)?.as_str().unwrap_or("none")); + println!("ai.model: {}", cfg.ai.model.as_deref().unwrap_or("-")); + println!("ai.command: {}", cfg.ai.command.as_deref().unwrap_or("-")); + println!("ai.agent: {}", cfg.ai.agent.as_deref().unwrap_or("-")); + println!("ai.dailyLimit: {}", status.daily_limit); + println!( + "AI: {}", + match &status.resolved { + Some(r) => format!("{} ({} of {} calls today)", r.label, status.calls_today, status.daily_limit), + None => "off".to_string(), + } + ); + println!( + "Agent CLIs found: {}", + if status.available.is_empty() { "none".into() } else { status.available.join(", ") } + ); + Ok(0) + } + ConfigCommand::Set { key, value } => { + let v = if value.trim().is_empty() { None } else { Some(value.clone()) }; + match key.as_str() { + "ai.provider" => { + cfg.ai.provider = serde_json::from_value(serde_json::Value::String(value.to_lowercase())) + .map_err(|_| fail(exit::USAGE, "ai.provider is one of none, auto, claude, codex, command"))?; + } + "ai.model" => cfg.ai.model = v, + "ai.command" => cfg.ai.command = v, + "ai.agent" => cfg.ai.agent = v, + "ai.dailyLimit" => { + cfg.ai.daily_limit = match v { + Some(n) => Some(n.parse().map_err(|_| fail(exit::USAGE, "ai.dailyLimit is a number"))?), + None => None, + } + } + other => return Err(fail(exit::USAGE, format!("unknown setting '{other}'"))), + } + cfg.save(&path).map_err(|e| fail(exit::ERRORED, e.to_string()))?; + eprintln!("saved {key} to {}", path.display()); + Ok(0) + } + } +} + async fn cmd_ui(ctx: &Ctx, a: UiArgs) -> Result { let roots = ctx.roots.clone(); let project = ctx.project.clone(); diff --git a/docs/ai.md b/docs/ai.md new file mode 100644 index 0000000..45ab435 --- /dev/null +++ b/docs/ai.md @@ -0,0 +1,79 @@ +# AI (optional) + +Lest works the same with or without AI. It is off by default, nothing is +sent anywhere until you choose a provider, and turning it on or off changes +no layout: AI only fills places that already have content, and AI actions +live in menus. + +## What it does + +- **Explains failed steps.** Every failed step already has a deterministic + headline built from its own output (the exit code and the error line, the + HTTP status and the body's message, the expectation and what the page + showed). With a provider, the failure card and `lest explain` add a short + explanation written by the model, marked as such, with the headline kept + beside it. The model gets the step's output tail, its notes, the earlier + steps' results and the flow file, with every resolved secret value + redacted. Output from the system under test is untrusted text: treat the + answer as a suggestion. Each step is explained once; the answer is cached. +- **Hands off to your own agent.** The failure card's Copy context button + and `lest context` give everything needed to pick up a failure: the flow and report paths, the + failing step, its output and the rerun command. With an agent configured, + **Open in agent** starts your agent CLI in a terminal, in the project, with + that task. Lest has no chat of its own: your agent already has tools, + permissions and memory. +- **Agent-readable docs.** `lest docs agent` prints the authoring guide and + the format references for a coding agent to read on demand. This needs no + provider. + +A model never decides whether a run passed, and a run never waits for a model. + +## Choosing a provider + +In the UI: Settings, AI. From the CLI: + +```bash +lest config set ai.provider auto # the first agent CLI found on PATH +lest config set ai.provider command # any program: prompt on stdin, text on stdout +lest config set ai.command 'my-llm --quiet' +lest config set ai.model small-model # optional, passed to agent CLIs +lest config set ai.agent 'my-agent {prompt}' # what Open in agent runs +lest config set ai.dailyLimit 50 +lest config show +``` + +| Provider | Uses | +|---|---| +| `none` | No AI (default). | +| `auto` | The first supported agent CLI on PATH, else none. | +| a supported agent CLI | That CLI, headless, in an empty temporary directory, for one answer. CLIs that can turn tools off run with no tools, no MCP servers and no user customizations; a CLI that cannot runs in its read-only sandbox without the user's config or rules, so it can still read files to answer. | +| `command` | Any program that reads the prompt on stdin and prints the answer. Labeled by the program's name. | + +Every call, writing the prompt included, is stopped after 90 seconds, and +the provider's whole process group is killed. Prompts are cut at 48 KB. The +provider never sees the environment variables the env secret backends read; +its own configuration and sign-in variables are left alone. + +Settings live in `config.yml` in Lest's config directory (`lest config show` +prints the path). Calls are counted per UTC day in the data directory; past +`dailyLimit`, explanations fall back to the headline with a note. Answers are +cached by the provider, command, model and input, so reopening a failure +costs nothing. + +Open in agent passes the task to your agent through the `LEST_AGENT_PROMPT` +environment variable, so the agent command's quoting cannot turn the task's +text into shell code. + +## Trust + +The UI's token is equivalent to running commands as you: anyone holding it +can run the project's flows and, through Settings, set the command the AI +provider runs. Lest's server listens on loopback only, requires the token on +every API call, and accepts changes only from its own page. Do not share the +address `lest ui` prints. + +## Without AI + +Everything else is identical: the same failure card with the deterministic +headline, the same Copy context and rerun actions, the same CLI. Only the +menu items that need an agent are absent. diff --git a/docs/design.md b/docs/design.md index 5cae200..a42a857 100644 --- a/docs/design.md +++ b/docs/design.md @@ -209,8 +209,10 @@ AI is optional and off by default. Turning it on or off changes no layout. agent CLI in the project with that context. Lest does not build a chat. The user's agent already has tools, permissions and memory; Lest gives it a good starting point. -- **Agent docs:** `lest docs agent` prints the authoring guide and schema for - an agent to read on demand. +- **Agent docs:** `lest docs agent` prints the authoring guide and the format + references for an agent to read on demand. + +[ai.md](ai.md) describes the behavior. Rules: no screen region exists only for AI; AI fills slots that already have deterministic content; AI actions live in menus, never as top-level buttons diff --git a/docs/images/flow-failed-ai.png b/docs/images/flow-failed-ai.png new file mode 100644 index 0000000..684b7e7 Binary files /dev/null and b/docs/images/flow-failed-ai.png differ diff --git a/docs/images/settings-ai.png b/docs/images/settings-ai.png new file mode 100644 index 0000000..ac6227c Binary files /dev/null and b/docs/images/settings-ai.png differ diff --git a/docs/ui.md b/docs/ui.md index 3e0cbe5..b4e9754 100644 --- a/docs/ui.md +++ b/docs/ui.md @@ -68,8 +68,8 @@ the step's `notes` under it. The actions: - The overflow menu copies the headline or step id, opens the run folder in the file manager, or opens the run on its own page. -The headline block is laid out so another text can take its place later -without moving anything else on the page. +The headline block is laid out so another text can take its place without +moving anything else on the page (see [AI (optional)](#ai-optional)). The UI follows the system's light or dark setting, or the choice in Settings: @@ -122,8 +122,61 @@ extras. A flow that has not recorded yet says so. ![Settings](images/settings.png) -Theme (System, Light, Dark), and the project's name, root, config file and -the Lest version. +Theme (System, Light, Dark), the AI settings, and the project's name, root, +config file and the Lest version. The theme is saved in the browser; the AI +settings are saved in the Lest settings file (`lest config show` prints its +path). + +## AI (optional) + +AI is off by default, and turning it on or off changes no layout: it fills +places that already hold deterministic content, and its actions live only in +menus. + +![Settings with a custom command provider](images/settings-ai.png) + +Settings, AI: + +- **Provider:** Off, Automatic (the first agent CLI found on PATH), one + choice per agent CLI found on PATH, or Custom command. Custom command shows + a field for the program, which reads the prompt on stdin and prints text. +- **Model:** passed to the agent CLI; a custom command does not receive it. + Empty means the provider's small default. +- **Agent command:** what Open in agent runs in a terminal; `{prompt}` + becomes the task. Empty means the detected agent CLI (a custom command + provider has none). +- **Daily limit:** model calls allowed per day (default 50). +- A status line says whether AI is off, which provider and model are in use + and how many model calls were made today, or why the choice resolves to + nothing (for example, no agent CLI on PATH). + +Every change is saved right away (text fields after a short pause). + +![A failure explained by a model, with the step menu open](images/flow-failed-ai.png) + +On a failed step of a finished run, with a provider in use, the UI asks the +server to explain the failure once per provider, run and step: + +- While it waits, the card shows the deterministic headline and + "Explaining…" in the marker position at the top right. The run is never + held up; a run in progress is explained only after it finishes. +- When the model answers, its text takes the headline's place, the marker + reads "Written by" and the provider's name, and the headline moves to the + line under it. +- When the model cannot be used (it failed, timed out, or the daily limit is + reached), the card keeps the headline and adds "AI unavailable:" with the + reason in small text under it. + +Explanations are cached by the server, so reopening a run does not call the +model again. + +The step's overflow menu gains two items when there is an agent to open +(an agent command, or a detected agent CLI as the provider), and only while a +provider is in use: **Open in agent** starts your agent CLI in a terminal, in the +project, with a prompt naming the flow, the run and the failed step (if no +terminal can be opened, a dialog shows the error and the command with a Copy +button), and **Copy agent command** copies that command. With AI off, these items are absent and the card is the same +as in [Failures](#failures). ## Updating the screenshots @@ -136,5 +189,9 @@ cd ui && LEST_UI_URL='http://127.0.0.1:4790/?token=...' npm run screenshots ``` The script starts runs through the API and waits for each state before it -captures. The project root appears in `settings.png`, so run the server from +captures. It turns AI off for the regular images and restores the AI settings +afterwards. `flow-failed-ai.png` and `settings-ai.png` need +`LEST_UI_AI_COMMAND` set to a program that reads a prompt on stdin and prints +an explanation; without it they are skipped. Start the server with a PATH that +has no agent CLIs on it so the provider choices in the image are generic. The project root appears in `settings.png`, so run the server from a copy of `examples/` at a neutral path if your checkout path is personal. diff --git a/ui/scripts/screenshots.mjs b/ui/scripts/screenshots.mjs index 3205c74..9a7bdc2 100644 --- a/ui/scripts/screenshots.mjs +++ b/ui/scripts/screenshots.mjs @@ -6,6 +6,14 @@ // // Starts no server. Triggers runs through the API and waits for each state // before capturing. Writes PNGs to docs/images/ at 2x. +// +// The AI screenshots need a custom command provider, a program that reads a +// prompt on stdin and prints an explanation: +// +// LEST_UI_AI_COMMAND=/path/to/program npm run screenshots +// +// Without it they are skipped. The script turns AI off for the other images +// and restores the AI settings it found when it ends. import { mkdir } from "node:fs/promises"; import { dirname, join } from "node:path"; @@ -69,8 +77,20 @@ async function go(page, path) { }, path); } +const putAi = (ai) => api("/settings/ai", { method: "PUT", body: JSON.stringify(ai) }); + async function main() { await mkdir(outDir, { recursive: true }); + const original = (await api("/settings")).ai; + try { + await putAi({ provider: "none" }); + await capture(); + } finally { + await putAi(original); + } +} + +async function capture() { const browser = await chromium.launch(); const light = await browser.newContext({ viewport: VIEWPORT, deviceScaleFactor: 2, colorScheme: "light" }); const page = await open(light); @@ -139,6 +159,28 @@ async function main() { await dpage.waitForSelector(".step.open .thumb img"); await shot(dpage, "flow-failed-dark.png"); + const aiCommand = process.env.LEST_UI_AI_COMMAND; + if (aiCommand) { + // A fresh page so the AI settings are read again. + const apage = await open(light); + await go(apage, "/settings"); + await apage.click('[aria-label="AI provider"] [role="radio"]:has-text("Custom command")'); + await apage.fill("#ai-command", aiCommand); + await apage.waitForSelector('.settings-status:has-text("Using")'); + await apage.waitForSelector('.settings-status .saved:has-text("Saved")'); + await shot(apage, "settings-ai.png"); + + await go(apage, "/flows/wrong-password"); + await apage.waitForSelector('.explain-marker:has-text("Written by")', { timeout: 120_000 }); + await apage.click('.step.open [role="tab"]:has-text("Artifacts")'); + await apage.waitForSelector(".step.open .thumb img"); + await apage.click('.explain-actions [aria-haspopup="menu"]'); + await apage.waitForSelector('.menu [role="menuitem"]:has-text("Open in agent")'); + await shot(apage, "flow-failed-ai.png"); + } else { + console.log("LEST_UI_AI_COMMAND is not set; skipping the AI screenshots"); + } + await browser.close(); } diff --git a/ui/src/api.ts b/ui/src/api.ts index f4a79f1..03ff5e9 100644 --- a/ui/src/api.ts +++ b/ui/src/api.ts @@ -1,7 +1,18 @@ // Calls to the Lest server. The page is served same-origin with the auth // cookie already set, so plain fetch works. -import type { FlowDetail, FlowsResponse, RunResponse, RunsResponse, StateResponse } from "./types"; +import type { + AgentLaunch, + AiConfig, + Explanation, + FlowDetail, + FlowsResponse, + Handoff, + RunResponse, + RunsResponse, + SettingsResponse, + StateResponse, +} from "./types"; export class ApiError extends Error { constructor( @@ -68,6 +79,17 @@ export const api = { post<{ cancelled: string }>(`/runs/${encodeURIComponent(id)}/cancel`, { stage }), reveal: (id: string, path?: string) => post<{ revealed: string }>(`/runs/${encodeURIComponent(id)}/reveal`, { path }), toolLogin: (name: string) => post(`/tools/${encodeURIComponent(name)}/login`), + settings: () => call("/settings"), + saveAi: (ai: AiConfig) => + call("/settings/ai", { + method: "PUT", + headers: { "content-type": "application/json" }, + body: JSON.stringify(ai), + }), + explain: (id: string, stepId: string) => post(`/runs/${encodeURIComponent(id)}/explain`, { stepId }), + handoff: (id: string, stepId: string) => + call(`/runs/${encodeURIComponent(id)}/handoff?${new URLSearchParams({ step: stepId })}`), + openAgent: (id: string, stepId: string) => post(`/runs/${encodeURIComponent(id)}/agent`, { stepId }), }; export function fileUrl(runId: string, path: string): string { diff --git a/ui/src/components/RunView.tsx b/ui/src/components/RunView.tsx index 46f90de..7dab63c 100644 --- a/ui/src/components/RunView.tsx +++ b/ui/src/components/RunView.tsx @@ -12,7 +12,15 @@ import { type StepState, } from "../store/runState"; import { useRun } from "../store/store"; -import type { Artifact, RunReport, StepReport, StepStatus, ToolReport } from "../types"; +import type { AiStatus, Artifact, Handoff, RunReport, StepReport, StepStatus, ToolReport } from "../types"; +import { + agentConfigured, + explainSlot, + explanationKey, + requestExplanation, + useAiSettings, + type ExplainState, +} from "../lib/ai"; import { IconCopy, IconExternal, @@ -23,7 +31,7 @@ import { } from "./icons"; import { LiveView } from "./LiveView"; import { ResultBadge, StatusDot, StatusGlyph } from "./status"; -import { CopyButton, copyWithToast, Lightbox, OverflowMenu, toast, useNow } from "./ui"; +import { CopyButton, copyWithToast, Dialog, Lightbox, OverflowMenu, toast, useNow } from "./ui"; export interface RunViewProps { runId: string; @@ -554,8 +562,8 @@ function KvRow({ k, v }: { k: string; v: ReactNode }) { } /** The failure explanation slot. The primary line holds the deterministic - * headline today; the marker and secondary lines keep their place so a - * different primary text can fill it without moving anything. */ + * headline; with AI on, a model's explanation takes its place and the + * headline moves to the line under it. The marker keeps its place either way. */ function FailedCard({ s, ctx }: { s: StepState; ctx: Ctx }) { const r = s.report as StepReport; const { run } = ctx; @@ -570,13 +578,46 @@ function FailedCard({ s, ctx }: { s: StepState; ctx: Ctx }) { inputs: report.inputs, }); const headline = r.headline ?? r.error ?? `${s.name} ${s.status}`; + const ai = useAiSettings(); + // Explanations and handoffs read the stored report, so they wait for the run to finish. + const stored = !ctx.live && !!run.report; + const explainState = useExplanation(ai?.status ?? null, stored ? run.runId : null, s.id); + const slot = explainSlot(headline, explainState); + const handoff = useHandoff(ai?.status ?? null, stored ? run.runId : null, s.id); + const [launch, setLaunch] = useState<{ error: string; command: string | null } | null>(null); + const openAgent = async () => { + try { + const res = await api.openAgent(run.runId, s.id); + if (res.launched) toast(`Opened ${res.how ?? "a terminal"} with the agent`); + else setLaunch({ error: res.error ?? "No terminal could be opened.", command: res.command }); + } catch (e) { + setLaunch({ error: (e as Error).message, command: null }); + } + }; + const agentItems = + stored && agentConfigured(ai?.status, handoff?.command) + ? [ + { label: "Open in agent", icon: , onSelect: () => void openAgent() }, + ...(handoff?.command + ? [ + { + label: "Copy agent command", + icon: , + onSelect: () => void copyWithToast(handoff.command as string, "agent command"), + }, + ] + : []), + ] + : []; return ( -
+
-
{headline}
- +
{slot.primary}
+ {slot.marker}
+ {slot.headline &&
{slot.headline}
} + {slot.note &&
{slot.note}
} {s.notes &&
{s.notes}
}
+ {launch && ( + setLaunch(null)}> +

{launch.error}

+ {launch.command ? ( + <> +

Run this in a terminal yourself:

+
+ 
{launch.command}
+ +
+ + ) : ( +

+ Set the agent to start with lest config set ai.agent '<command> {"{prompt}"}'. +

+ )} + + )}
); } +/** The explanation for a failed step of a finished run, when AI is on. */ +function useExplanation(status: AiStatus | null, runId: string | null, stepId: string): ExplainState { + const key = status?.resolved && runId ? explanationKey(status, runId, stepId) : null; + const [result, setResult] = useState<{ key: string; state: ExplainState } | null>(null); + useEffect(() => { + if (!key || !runId) return; + let alive = true; + requestExplanation(key, runId, stepId).then( + (explanation) => alive && setResult({ key, state: { kind: "done", explanation } }), + (e: Error) => alive && setResult({ key, state: { kind: "error", message: e.message } }), + ); + return () => { + alive = false; + }; + }, [key, runId, stepId]); + if (!key) return { kind: "off" }; + return result?.key === key ? result.state : { kind: "pending" }; +} + +/** The agent handoff for a finished run, fetched when AI may be on. */ +function useHandoff(status: AiStatus | null, runId: string | null, stepId: string): Handoff | null { + const [handoff, setHandoff] = useState(null); + const resolved = status?.resolved?.label ?? ""; + useEffect(() => { + if (!status || !runId || status.configured === "none") { + setHandoff(null); + return; + } + let alive = true; + api.handoff(runId, stepId).then( + (h) => alive && setHandoff(h), + () => alive && setHandoff(null), + ); + return () => { + alive = false; + }; + }, [status?.configured, resolved, runId, stepId]); + return handoff; +} + /** A report-shaped view of a run whose report has not loaded yet. */ function syntheticReport(run: RunState, ctx: Ctx): RunReport { const toReport = (id: string): StepReport => { diff --git a/ui/src/components/ui.tsx b/ui/src/components/ui.tsx index 9d5c257..80a4920 100644 --- a/ui/src/components/ui.tsx +++ b/ui/src/components/ui.tsx @@ -143,6 +143,31 @@ export function Lightbox({ src, caption, onClose }: { src: string; caption: stri ); } +/** A small modal with a title, a body and a Close button. */ +export function Dialog({ title, onClose, children }: { title: string; onClose: () => void; children: ReactNode }) { + const closeRef = useRef(null); + useEffect(() => { + const onKey = (e: KeyboardEvent) => e.key === "Escape" && onClose(); + document.addEventListener("keydown", onKey); + closeRef.current?.focus(); + return () => document.removeEventListener("keydown", onKey); + }, [onClose]); + return ( +
+
e.stopPropagation()}> +
+ {title} + +
+
{children}
+
+
+ ); +} + export function Switch({ checked, onChange, diff --git a/ui/src/lib/ai.ts b/ui/src/lib/ai.ts new file mode 100644 index 0000000..5a63e08 --- /dev/null +++ b/ui/src/lib/ai.ts @@ -0,0 +1,156 @@ +// Optional AI: the settings store, the client-side explanation cache, and +// the pure decisions about what the failure card and Settings show. + +import { useEffect, useSyncExternalStore } from "react"; +import { api } from "../api"; +import type { AiConfig, AiStatus, Explanation, SettingsResponse } from "../types"; + +// Settings, fetched once per page load and replaced on every save. + +let settings: SettingsResponse | null = null; +let loading: Promise | null = null; +const listeners = new Set<() => void>(); + +function emit() { + listeners.forEach((l) => l()); +} + +function load(): Promise { + loading ??= api.settings().then( + (s) => { + settings = s; + emit(); + }, + () => { + // AI stays off for this page when settings cannot be read. + loading = null; + }, + ); + return loading; +} + +export function useAiSettings(): SettingsResponse | null { + const s = useSyncExternalStore( + (l) => { + listeners.add(l); + return () => listeners.delete(l); + }, + () => settings, + ); + useEffect(() => { + if (!settings) void load(); + }, []); + return s; +} + +/** Saves the AI settings. The agent command has no field in the UI, so the + * one in the settings file is kept. */ +export async function saveAi(ai: AiConfig): Promise { + const s = await api.saveAi(ai); + settings = s; + emit(); + return s; +} + +// Explanations, requested once per provider, run and step. + +const explanations = new Map>(); + +export function explanationKey(status: AiStatus, runId: string, stepId: string): string { + return [status.resolved?.label ?? "", status.resolved?.model ?? "", runId, stepId].join("\n"); +} + +export function requestExplanation(key: string, runId: string, stepId: string): Promise { + let p = explanations.get(key); + if (!p) { + p = api.explain(runId, stepId); + // A failed request is not cached, so a later view can try again. + p.catch(() => explanations.delete(key)); + explanations.set(key, p); + } + return p; +} + +// What the failure card's explanation slot shows. + +export type ExplainState = + | { kind: "off" } + | { kind: "pending" } + | { kind: "done"; explanation: Explanation } + | { kind: "error"; message: string }; + +export interface Slot { + /** The main line. */ + primary: string; + /** The small text in the marker position, empty when AI is off. */ + marker: string; + /** The deterministic headline when a model wrote the main line. */ + headline: string | null; + /** Why the model's text is not shown, in small muted text. */ + note: string | null; +} + +export function explainSlot(headline: string, state: ExplainState): Slot { + const base: Slot = { primary: headline, marker: "", headline: null, note: null }; + switch (state.kind) { + case "off": + return base; + case "pending": + return { ...base, marker: "Explaining…" }; + case "error": + return { ...base, note: `AI unavailable: ${state.message}` }; + case "done": { + const e = state.explanation; + if (e.source === "model" && e.text.trim()) { + return { primary: e.text.trim(), marker: `Written by ${e.provider ?? "the model"}`, headline, note: null }; + } + return { ...base, note: e.note ? `AI unavailable: ${e.note}` : null }; + } + } +} + +// Settings copy. + +export function aiStatusLine(ai: AiConfig, status: AiStatus): string { + const r = status.resolved; + if (r) { + const model = r.model ? `, model ${r.model}` : ""; + return `Using ${r.label}${model}. ${status.callsToday} of ${status.dailyLimit} model calls today.`; + } + switch (ai.provider) { + case "none": + return "AI is off. Failure explanations come from the step's own output."; + case "auto": + return "No agent CLI found on PATH; AI stays off."; + case "command": + return "Enter a command; AI stays off until then."; + default: + return `${ai.provider} is not on PATH; AI stays off.`; + } +} + +export interface ProviderOption { + id: AiConfig["provider"]; + label: string; +} + +/** Off, Automatic, each agent CLI found on PATH, and Custom command. A + * configured CLI that is not on PATH stays listed so the choice is visible. */ +export function providerOptions(ai: AiConfig, status: AiStatus): ProviderOption[] { + const clis = [...status.available]; + if (!["none", "auto", "command"].includes(ai.provider) && !clis.includes(ai.provider)) clis.push(ai.provider); + return [ + { id: "none", label: "Off" }, + { id: "auto", label: "Automatic" }, + ...clis.map((c) => ({ id: c, label: c })), + { id: "command", label: "Custom command" }, + ]; +} + +/** Whether the step menu offers the agent items. */ +export function agentConfigured(status: AiStatus | null | undefined, handoffCommand: string | null | undefined): boolean { + // Only a command Lest can run counts: a provider alone (a custom + // command) has no interactive agent to open. + void status; + return !!handoffCommand; +} diff --git a/ui/src/pages/Settings.tsx b/ui/src/pages/Settings.tsx index 5842b7c..080ba6f 100644 --- a/ui/src/pages/Settings.tsx +++ b/ui/src/pages/Settings.tsx @@ -1,7 +1,9 @@ -import { useState, type ReactNode } from "react"; +import { useEffect, useRef, useState, type ReactNode } from "react"; import { IconDisplay, IconMoon, IconSun } from "../components/icons"; import { CopyButton } from "../components/ui"; +import { aiStatusLine, providerOptions, saveAi, useAiSettings } from "../lib/ai"; import { readTheme, saveTheme, useProject, type Theme } from "../lib/project"; +import type { AiConfig, SettingsResponse } from "../types"; const THEMES: { id: Theme; label: string; icon: ReactNode }[] = [ { id: "system", label: "System", icon: }, @@ -17,7 +19,7 @@ export function SettingsPage() {

Settings

-
Saved in this browser.
+
Appearance is saved in this browser; AI in your Lest settings file.
@@ -49,6 +51,7 @@ export function SettingsPage() {
+
Project @@ -74,3 +77,175 @@ export function SettingsPage() {
); } + +const TEXT_SAVE_DELAY_MS = 600; + +function AiSection() { + const settings = useAiSettings(); + return ( +
+
+ AI +
+ {settings ? :
Loading
} +
+ ); +} + +function AiForm({ settings }: { settings: SettingsResponse }) { + // The form edits a local copy; every change is saved, text fields after a pause. + const [ai, setAi] = useState(settings.ai); + const [limitText, setLimitText] = useState(settings.ai.dailyLimit ? String(settings.ai.dailyLimit) : ""); + const [saved, setSaved] = useState<"" | "saving" | "saved">(""); + const [saveError, setSaveError] = useState(null); + const timer = useRef | undefined>(undefined); + const latest = useRef(ai); + latest.current = ai; + + const save = async (next: AiConfig) => { + setSaved("saving"); + try { + await saveAi(clean(next)); + setSaveError(null); + if (latest.current === next) setSaved("saved"); + } catch (e) { + setSaved(""); + setSaveError(`Not saved: ${(e as Error).message}`); + } + }; + const change = (patch: Partial, debounce: boolean) => { + const next = { ...latest.current, ...patch }; + setAi(next); + clearTimeout(timer.current); + if (debounce) timer.current = setTimeout(() => void save(next), TEXT_SAVE_DELAY_MS); + else void save(next); + }; + useEffect(() => () => clearTimeout(timer.current), []); + useEffect(() => { + if (saved !== "saved") return; + const t = setTimeout(() => setSaved(""), 2000); + return () => clearTimeout(t); + }, [saved]); + + const status = settings.status; + const options = providerOptions(ai, status); + return ( + <> +
+
+
Provider
+
+ Explains failed steps in plain words and can open your own agent CLI in the project with the failure's + context. Nothing is sent anywhere unless you choose a provider here. +
+
+
+ {options.map((o) => ( + + ))} +
+
+ {ai.provider === "command" && ( +
+
+ +
Reads the prompt on stdin and prints text.
+
+ change({ command: e.target.value }, true)} + /> +
+ )} +
+
+ +
Passed to the agent CLI; a custom command does not receive it.
+
+ change({ model: e.target.value }, true)} + /> +
+
+
+ +
+ What Open in agent runs in a terminal, in the project; {"{prompt}"} becomes the task. Empty: + the detected agent CLI (a custom command provider has none). +
+
+ change({ agent: e.target.value }, true)} + /> +
+
+
+ +
Model calls allowed per day. Each failed step of a run is explained once and then cached.
+
+ { + setLimitText(e.target.value); + change({ dailyLimit: parseLimit(e.target.value) }, true); + }} + /> +
+
+ {saveError ?? aiStatusLine(settings.ai, status)} + {saved === "saved" ? "Saved" : ""} +
+ + ); +} + +function parseLimit(text: string): number | undefined { + const n = Number(text.trim()); + return Number.isInteger(n) && n > 0 ? n : undefined; +} + +/** Drops empty optional fields so the settings file keeps only real values. */ +function clean(ai: AiConfig): AiConfig { + const out: AiConfig = { provider: ai.provider }; + if (ai.agent?.trim()) out.agent = ai.agent.trim(); + if (ai.model?.trim()) out.model = ai.model.trim(); + if (ai.command?.trim()) out.command = ai.command.trim(); + if (ai.dailyLimit) out.dailyLimit = ai.dailyLimit; + return out; +} diff --git a/ui/src/styles.css b/ui/src/styles.css index 21ba725..e69939c 100644 --- a/ui/src/styles.css +++ b/ui/src/styles.css @@ -1332,6 +1332,13 @@ a.card:hover { font-size: 12.5px; white-space: pre-wrap; } +.explain-marker { + text-align: right; +} +.explain-note { + color: var(--text-3); + font-size: 11.5px; +} .explain-actions { display: flex; gap: 6px; @@ -1734,6 +1741,49 @@ a.card:hover { outline: none; } +.dialog-scrim { + position: fixed; + inset: 0; + z-index: 50; + background: var(--scrim); + display: grid; + place-items: center; + padding: 40px 16px; +} +.dialog { + width: min(560px, 100%); + box-shadow: var(--shadow-lg); +} +.dialog-body { + padding: 12px 14px 14px; + display: flex; + flex-direction: column; + gap: 8px; + font-size: 13px; +} +.dialog-body p { + margin: 0; +} +.dialog-command { + display: flex; + align-items: flex-start; + gap: 8px; +} +.dialog-command pre { + flex: 1; + min-width: 0; + margin: 0; + padding: 8px 10px; + border-radius: 6px; + background: var(--sunken); + border: 1px solid var(--border); + white-space: pre-wrap; + overflow-wrap: anywhere; + max-height: 200px; + overflow: auto; + font-size: 12px; +} + .lightbox { position: fixed; inset: 0; @@ -2073,6 +2123,35 @@ table.runs tr.running td { color: var(--text-3); font-size: 12.5px; } +.settings-row .input { + width: 280px; + flex-shrink: 0; +} +.settings-row .seg { + flex-shrink: 0; +} +.settings-status { + display: flex; + align-items: center; + gap: 16px; + height: 44px; + padding: 0 16px; + border-top: 1px solid var(--border); + color: var(--text-2); + font-size: 12.5px; +} +.settings-status .msg { + flex: 1; + min-width: 0; + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; +} +.settings-status .saved { + width: 4em; + text-align: right; + color: var(--text-3); +} @media (max-width: 1100px) { .run-layout, diff --git a/ui/src/test/ai.test.ts b/ui/src/test/ai.test.ts new file mode 100644 index 0000000..460ebdb --- /dev/null +++ b/ui/src/test/ai.test.ts @@ -0,0 +1,112 @@ +import { describe, expect, it } from "vitest"; +import { agentConfigured, aiStatusLine, explainSlot, providerOptions } from "../lib/ai"; +import type { AiConfig, AiStatus } from "../types"; + +const headline = 'Expected the dashboard, but the page shows "Wrong email or password".'; + +const off: AiStatus = { configured: "none", resolved: null, available: [], callsToday: 0, dailyLimit: 50 }; +const on: AiStatus = { + configured: "command", + resolved: { kind: "command", label: "local-model", model: null }, + available: [], + callsToday: 3, + dailyLimit: 50, +}; + +describe("explanation slot", () => { + it("shows today's card when AI is off", () => { + expect(explainSlot(headline, { kind: "off" })).toEqual({ primary: headline, marker: "", headline: null, note: null }); + }); + + it("keeps the headline while the explanation is pending", () => { + expect(explainSlot(headline, { kind: "pending" })).toEqual({ + primary: headline, + marker: "Explaining…", + headline: null, + note: null, + }); + }); + + it("puts the model's text first and moves the headline under it", () => { + const slot = explainSlot(headline, { + kind: "done", + explanation: { text: " The password is wrong. \n", source: "model", provider: "local-model" }, + }); + expect(slot).toEqual({ + primary: "The password is wrong.", + marker: "Written by local-model", + headline, + note: null, + }); + }); + + it("keeps the headline and shows the note on a fallback", () => { + const slot = explainSlot(headline, { + kind: "done", + explanation: { text: headline, source: "lest", note: "daily limit of 50 model calls reached" }, + }); + expect(slot).toEqual({ + primary: headline, + marker: "", + headline: null, + note: "AI unavailable: daily limit of 50 model calls reached", + }); + }); + + it("treats an empty model answer and a failed request as fallbacks", () => { + expect(explainSlot(headline, { kind: "done", explanation: { text: " ", source: "model", provider: "x" } }).primary).toBe( + headline, + ); + expect(explainSlot(headline, { kind: "error", message: "500 Internal Server Error" })).toEqual({ + primary: headline, + marker: "", + headline: null, + note: "AI unavailable: 500 Internal Server Error", + }); + }); +}); + +describe("AI settings", () => { + const cfg = (provider: string, extra: Partial = {}): AiConfig => ({ provider, ...extra }); + + it("describes each state in one line", () => { + expect(aiStatusLine(cfg("none"), off)).toBe("AI is off. Failure explanations come from the step's own output."); + expect(aiStatusLine(cfg("auto"), { ...off, configured: "auto" })).toBe("No agent CLI found on PATH; AI stays off."); + expect(aiStatusLine(cfg("command"), { ...off, configured: "command" })).toBe( + "Enter a command; AI stays off until then.", + ); + expect(aiStatusLine(cfg("command", { command: "/x/local-model" }), on)).toBe( + "Using local-model. 3 of 50 model calls today.", + ); + expect( + aiStatusLine(cfg("command"), { ...on, resolved: { kind: "command", label: "local-model", model: "small" } }), + ).toBe("Using local-model, model small. 3 of 50 model calls today."); + }); + + it("lists the detected CLIs between Automatic and Custom command", () => { + const labels = (ai: AiConfig, st: AiStatus) => providerOptions(ai, st).map((o) => o.label); + expect(labels(cfg("none"), off)).toEqual(["Off", "Automatic", "Custom command"]); + expect(labels(cfg("none"), { ...off, available: ["agent-a", "agent-b"] })).toEqual([ + "Off", + "Automatic", + "agent-a", + "agent-b", + "Custom command", + ]); + // A configured CLI that is no longer on PATH stays visible. + expect(labels(cfg("agent-b"), { ...off, available: ["agent-a"] })).toEqual([ + "Off", + "Automatic", + "agent-a", + "agent-b", + "Custom command", + ]); + }); + + it("offers the agent items only when an agent or provider is configured", () => { + expect(agentConfigured(off, null)).toBe(false); + expect(agentConfigured(null, undefined)).toBe(false); + expect(agentConfigured(on, null)).toBe(false); + expect(agentConfigured(off, "cd '/p' && agent 'go'")).toBe(true); + }); +}); diff --git a/ui/src/types.ts b/ui/src/types.ts index 310c657..98f9046 100644 --- a/ui/src/types.ts +++ b/ui/src/types.ts @@ -264,3 +264,56 @@ export type EventBody = }; export type RunEvent = EventBase & EventBody; + +// Optional AI (crates/lest-core/src/ai.rs). + +/** "none", "auto", "command", or the name of a built-in agent CLI. */ +export type AiProvider = string; + +export interface AiConfig { + provider: AiProvider; + model?: string; + command?: string; + agent?: string; + dailyLimit?: number; +} + +export interface AiResolved { + kind: AiProvider; + label: string; + model: string | null; +} + +export interface AiStatus { + configured: AiProvider; + resolved: AiResolved | null; + /** Agent CLIs found on PATH. */ + available: string[]; + callsToday: number; + dailyLimit: number; +} + +export interface SettingsResponse { + ai: AiConfig; + status: AiStatus; +} + +export interface Explanation { + text: string; + source: "lest" | "model"; + provider?: string; + note?: string; +} + +export interface Handoff { + context: string; + prompt: string; + command: string | null; +} + +export interface AgentLaunch { + launched: boolean; + how?: string; + error?: string; + command: string; +}