From 9761631e3e97a009c2eece555592105ab5626cb7 Mon Sep 17 00:00:00 2001 From: CLAUDE <355856132+HoTT-stuff@users.noreply.github.com> Date: Fri, 13 Nov 2026 10:58:36 +0000 Subject: [PATCH] feat(panicbot): run the full panic-attack suite unless the directives say otherwise Owner ruling 2026-11-13: panicbot ran `assail` only. It now runs every class (a) panic-attack subcommand (scan/analysis, non-interactive, findings or verdict), inventoried in bots/panicbot/src/suite.rs with the reason for each subcommand that is out (interactive, infrastructure, or held back). - AllowedCommands covers assail, assay, adjudicate, diagnostics, attack, assault, ambush, amuck, abduct, axial; the default is all of them. - Directives: no file gets the full suite. A file keeps exactly what it allows and denies (no `allow` key keeps the static trio it always granted; `deny` now narrows; deny wins). Unknown names are ignored with a printed note. A directive file that does not parse stops the run instead of falling back to the (now full) default. - Each part ends ran / not_applicable / denied / error, never a silent skip. Dynamic modes need an explicit --target (an executable inside the scanned directory); without one they report not_applicable with the reason. They are refused under pull_request_target and workflow_run. - New findings name their file: assay hits (PB-ASSAY-*), dynamic results and the adjudicated verdict (PB-DYN-*, PB-ADJUDICATE-VERDICT) name the target. - A failed part makes `scan --format json` print an error object and exit 3; a run that analysed nothing prints not_run and exits 4. The report-back driver maps both to envelope statuses; a findings array stays a complete run. - Whole-suite budget (240 s, under the driver's 300 s limit); each invocation gets the smaller of its timeout and what is left. panic-attack runs in a scratch cwd so a scan does not write into the repository. - `adjudicate` prints `pass (processed 0, failed 1)` and exits 0 for a missing or unrecognised report; panicbot treats that as an error, not a pass. - `diagnostics` --version had no timeout and could hang the lane; it is timed. Advisory gating is unchanged (owner ruling G2). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_0136P7ZkERqsQWMogHRpM2z8 --- bots/panicbot/src/cli.rs | 21 +- bots/panicbot/src/config.rs | 284 +++- bots/panicbot/src/directives.rs | 227 +++- bots/panicbot/src/fleet.rs | 89 +- bots/panicbot/src/lib.rs | 1 + bots/panicbot/src/main.rs | 109 +- bots/panicbot/src/scanner.rs | 453 ++++++- bots/panicbot/src/suite.rs | 1279 +++++++++++++++++++ bots/panicbot/tests/envelope_parity_test.rs | 161 ++- docs/REPORT-BACK.adoc | 18 +- scripts/fleet-report-back.sh | 35 +- tests/fleet-report-back-test.sh | 37 + 12 files changed, 2604 insertions(+), 110 deletions(-) create mode 100644 bots/panicbot/src/suite.rs diff --git a/bots/panicbot/src/cli.rs b/bots/panicbot/src/cli.rs index 5594da96..822b8f3c 100644 --- a/bots/panicbot/src/cli.rs +++ b/bots/panicbot/src/cli.rs @@ -2,7 +2,7 @@ //! CLI Interface — clap-based command-line argument parsing for panicbot. //! //! Subcommands: -//! - `scan ` — Run panic-attack assail and translate findings +//! - `scan ` — Run the allowed panic-attack suite and translate findings //! - `fleet ` — Run as a gitbot-fleet member //! - `report ` — Generate A2ML debt register @@ -49,11 +49,24 @@ pub enum OutputFormat { /// Available subcommands. #[derive(Subcommand, Debug)] pub enum Command { - /// Run panic-attack assail on a directory and translate findings. + /// Run the allowed panic-attack suite on a directory and translate findings. + /// + /// Full suite by default; the repository's bot directives can narrow it. + /// Exits 3 if any part failed (findings of the parts that ran are still + /// printed) and 4 if no part analysed anything. Scan { /// Directory or binary to scan. #[arg(default_value = ".")] dir: PathBuf, + + /// A built executable inside DIR for the dynamic modes (assault, ambush). + /// They EXECUTE it. Without it they report not_applicable. + #[arg(long)] + target: Option, + + /// Wall-clock budget for the whole suite, in seconds (default 240). + #[arg(long)] + budget_seconds: Option, }, /// Run as a gitbot-fleet member. @@ -65,6 +78,10 @@ pub enum Command { /// Bot operating mode (advisor, auditor, guardian). #[arg(short, long, default_value = "advisor")] mode: String, + + /// A built executable inside DIR for the dynamic modes. They EXECUTE it. + #[arg(long)] + target: Option, }, /// Generate A2ML debt register for a directory. diff --git a/bots/panicbot/src/config.rs b/bots/panicbot/src/config.rs index 98f684b4..4c10dc7f 100644 --- a/bots/panicbot/src/config.rs +++ b/bots/panicbot/src/config.rs @@ -12,7 +12,7 @@ use std::time::Duration; /// Top-level panicbot configuration. /// /// Controls which `panic-attack` subcommands run, severity thresholds, -/// and subprocess timeout behaviour. +/// and subprocess timeout and budget behaviour. #[derive(Debug, Clone, Serialize, Deserialize)] pub struct PanicbotConfig { /// Which panic-attack subcommands are allowed to execute. @@ -25,25 +25,207 @@ pub struct PanicbotConfig { /// Maximum wall-clock time for a single `panic-attack` invocation. pub timeout: Duration, + /// Wall-clock budget for the WHOLE suite. Each invocation runs for at most + /// the smaller of `timeout` and what is left of the budget; a part that + /// cannot start inside the budget reports `error`, never a silent skip. + /// Kept under the report-back driver's 300 s per-adapter limit. + pub budget: Duration, + + /// Things the directive loader wants the operator to see (unknown command + /// names, a deny that overrides an allow). Printed by the CLI. + #[serde(default)] + pub notes: Vec, + /// Per-category confidence overrides. /// If absent, `translator::default_confidence()` is used. pub confidence_overrides: Vec, } +/// A panic-attack subcommand panicbot can run (class (a) of the inventory in +/// [`crate::suite`]). +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[serde(rename_all = "lowercase")] +pub enum Subcommand { + /// Static analysis of a directory (weak points). + Assail, + /// Proven-library substitution survey of a directory. + Assay, + /// Tool preflight: binary present, version, health. + Diagnostics, + /// Verdict over dynamic-mode reports. + Adjudicate, + /// Single-axis attack on a built program (covered by `assault`). + Attack, + /// Assail plus multi-axis attack on a built program. + Assault, + /// Program run under ambient stressors. + Ambush, + /// File mutation with an execution check. + Amuck, + /// File isolation with an execution check. + Abduct, + /// Reaction observation from an execution or reports. + Axial, +} + +impl Subcommand { + /// Every class (a) subcommand, in the order the suite runs them. + pub const ALL: [Subcommand; 10] = [ + Subcommand::Assail, + Subcommand::Assay, + Subcommand::Assault, + Subcommand::Ambush, + Subcommand::Attack, + Subcommand::Amuck, + Subcommand::Abduct, + Subcommand::Axial, + Subcommand::Adjudicate, + Subcommand::Diagnostics, + ]; + + /// The subcommand as panic-attack and the directive files spell it. + pub fn name(self) -> &'static str { + match self { + Subcommand::Assail => "assail", + Subcommand::Assay => "assay", + Subcommand::Diagnostics => "diagnostics", + Subcommand::Adjudicate => "adjudicate", + Subcommand::Attack => "attack", + Subcommand::Assault => "assault", + Subcommand::Ambush => "ambush", + Subcommand::Amuck => "amuck", + Subcommand::Abduct => "abduct", + Subcommand::Axial => "axial", + } + } + + /// Parse a directive spelling. Unknown names are `None`, never a guess. + pub fn parse(s: &str) -> Option { + Subcommand::ALL.into_iter().find(|c| c.name() == s) + } + + /// Whether the subcommand executes code from the scanned repository + /// (or a program built from it). Dynamic modes need an explicit target + /// and are refused under privileged CI triggers (see [`crate::suite`]). + pub fn is_dynamic(self) -> bool { + matches!( + self, + Subcommand::Attack + | Subcommand::Assault + | Subcommand::Ambush + | Subcommand::Amuck + | Subcommand::Abduct + | Subcommand::Axial + ) + } +} + /// Which panic-attack subcommands panicbot is allowed to invoke. /// -/// Dynamic attack modes (`attack`, `assault`, `ambush`, `amuck`, `abduct`, -/// `axial`) are **denied by default** — they execute the target binary and -/// can cause side effects. Static analysis (`assail`, `adjudicate`, -/// `diagnostics`) is safe and enabled by default. -#[derive(Debug, Clone, Serialize, Deserialize)] +/// The default is the FULL SUITE: every class (a) subcommand (owner ruling +/// 2026-11-13). A repository narrows it with its directive file, which can +/// deny any of them or replace the set with an explicit allow list (see +/// [`crate::directives`]). Allowed is not the same as run: a dynamic mode +/// that is allowed but has no target reports `not_applicable` with its reason. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct AllowedCommands { - /// `panic-attack assail` — core static analysis (safe, default: true). + /// `panic-attack assail` — core static analysis. pub assail: bool, - /// `panic-attack adjudicate` — multi-report verdict (safe, default: true). + /// `panic-attack assay` — proven-library substitution survey. + pub assay: bool, + /// `panic-attack adjudicate` — verdict over dynamic-mode reports. pub adjudicate: bool, - /// `panic-attack diagnostics` — preflight health check (safe, default: true). + /// `panic-attack diagnostics` — preflight health check. pub diagnostics: bool, + /// `panic-attack attack` — single-axis attack (covered by `assault`). + pub attack: bool, + /// `panic-attack assault` — assail plus multi-axis attack on a program. + pub assault: bool, + /// `panic-attack ambush` — program under ambient stressors. + pub ambush: bool, + /// `panic-attack amuck` — file mutation with an execution check. + pub amuck: bool, + /// `panic-attack abduct` — file isolation with an execution check. + pub abduct: bool, + /// `panic-attack axial` — reaction observation. + pub axial: bool, +} + +impl AllowedCommands { + /// Every class (a) subcommand allowed: the default. + pub fn all() -> Self { + Self { + assail: true, + assay: true, + adjudicate: true, + diagnostics: true, + attack: true, + assault: true, + ambush: true, + amuck: true, + abduct: true, + axial: true, + } + } + + /// Nothing allowed. + pub fn none() -> Self { + Self { + assail: false, + assay: false, + adjudicate: false, + diagnostics: false, + attack: false, + assault: false, + ambush: false, + amuck: false, + abduct: false, + axial: false, + } + } + + /// What a directive file granted before the full-suite ruling: the static + /// trio. A directive file that says nothing about `allow` keeps exactly + /// this, so a repository's scope is never widened behind its back. + pub fn legacy_static() -> Self { + let mut a = Self::none(); + a.assail = true; + a.adjudicate = true; + a.diagnostics = true; + a + } + + /// Whether `cmd` is allowed. + pub fn allows(&self, cmd: Subcommand) -> bool { + match cmd { + Subcommand::Assail => self.assail, + Subcommand::Assay => self.assay, + Subcommand::Diagnostics => self.diagnostics, + Subcommand::Adjudicate => self.adjudicate, + Subcommand::Attack => self.attack, + Subcommand::Assault => self.assault, + Subcommand::Ambush => self.ambush, + Subcommand::Amuck => self.amuck, + Subcommand::Abduct => self.abduct, + Subcommand::Axial => self.axial, + } + } + + /// Allow or deny one subcommand. + pub fn set(&mut self, cmd: Subcommand, allowed: bool) { + match cmd { + Subcommand::Assail => self.assail = allowed, + Subcommand::Assay => self.assay = allowed, + Subcommand::Diagnostics => self.diagnostics = allowed, + Subcommand::Adjudicate => self.adjudicate = allowed, + Subcommand::Attack => self.attack = allowed, + Subcommand::Assault => self.assault = allowed, + Subcommand::Ambush => self.ambush = allowed, + Subcommand::Amuck => self.amuck = allowed, + Subcommand::Abduct => self.abduct = allowed, + Subcommand::Axial => self.axial = allowed, + } + } } /// Minimum severity filter for reported findings. @@ -73,27 +255,28 @@ pub struct ConfidenceOverride { pub confidence: f64, } +impl Default for AllowedCommands { + fn default() -> Self { + Self::all() + } +} + +/// Default wall-clock budget for the whole suite (see [`PanicbotConfig::budget`]). +pub const DEFAULT_BUDGET: Duration = Duration::from_secs(240); + impl Default for PanicbotConfig { fn default() -> Self { Self { allowed_commands: AllowedCommands::default(), min_severity: MinSeverity::Low, timeout: Duration::from_secs(300), + budget: DEFAULT_BUDGET, + notes: Vec::new(), confidence_overrides: Vec::new(), } } } -impl Default for AllowedCommands { - fn default() -> Self { - Self { - assail: true, - adjudicate: true, - diagnostics: true, - } - } -} - impl MinSeverity { /// Returns the numeric threshold for this severity level. /// Findings with a severity value >= this threshold are reported. @@ -145,6 +328,11 @@ mod tests { assert!(config.allowed_commands.diagnostics); assert_eq!(config.min_severity, MinSeverity::Low); assert_eq!(config.timeout, Duration::from_secs(300)); + assert_eq!(config.budget, DEFAULT_BUDGET); + assert!( + config.budget < Duration::from_secs(300), + "must stay under the driver's 300 s adapter limit" + ); assert!(config.confidence_overrides.is_empty()); } @@ -176,3 +364,61 @@ mod tests { assert_eq!(config.confidence_for("PanicPath"), None); } } + +#[cfg(test)] +mod suite_tests { + use super::*; + + #[test] + fn default_is_the_full_suite() { + let allowed = PanicbotConfig::default().allowed_commands; + for cmd in Subcommand::ALL { + assert!( + allowed.allows(cmd), + "{} must be allowed by default", + cmd.name() + ); + } + assert_eq!(allowed, AllowedCommands::all()); + } + + #[test] + fn all_lists_each_subcommand_once_and_names_round_trip() { + let mut seen = std::collections::HashSet::new(); + for cmd in Subcommand::ALL { + assert!(seen.insert(cmd), "{} listed twice", cmd.name()); + assert_eq!(Subcommand::parse(cmd.name()), Some(cmd)); + } + assert_eq!(Subcommand::parse("tui"), None); + assert_eq!(Subcommand::parse("Assail"), None); + } + + #[test] + fn set_and_allows_agree_for_every_subcommand() { + for cmd in Subcommand::ALL { + let mut a = AllowedCommands::none(); + assert!(!a.allows(cmd)); + a.set(cmd, true); + assert!(a.allows(cmd)); + for other in Subcommand::ALL.into_iter().filter(|o| *o != cmd) { + assert!( + !a.allows(other), + "{} leaked into {}", + cmd.name(), + other.name() + ); + } + } + } + + #[test] + fn legacy_static_is_exactly_the_old_default() { + let a = AllowedCommands::legacy_static(); + let on: Vec<&str> = Subcommand::ALL + .into_iter() + .filter(|c| a.allows(*c)) + .map(|c| c.name()) + .collect(); + assert_eq!(on, ["assail", "adjudicate", "diagnostics"]); + } +} diff --git a/bots/panicbot/src/directives.rs b/bots/panicbot/src/directives.rs index 53547f87..2bb4c703 100644 --- a/bots/panicbot/src/directives.rs +++ b/bots/panicbot/src/directives.rs @@ -1,12 +1,25 @@ // SPDX-License-Identifier: MPL-2.0 //! Directives — per-repo configuration parser for `.machine_readable/bot_directives/panicbot.a2ml`. //! -//! Each repository can customise panicbot's behaviour via a TOML-shaped A2ML -//! file. If no directive file exists, safe defaults are used: -//! - Allow: `assail`, `adjudicate`, `diagnostics` (static analysis only) -//! - Deny: all dynamic attack modes (`attack`, `assault`, `ambush`, etc.) -//! - Min severity: `low` (report everything) -//! - Timeout: 300 seconds +//! Each repository can customise panicbot's behaviour via a TOML-shaped file. +//! A2ML is retired and the deed has no per-bot directive clause yet (the +//! standards mapping only covers the `instructions-dir` pointer), so this +//! module only READS the legacy file; it never writes or extends one. +//! +//! ## What a repository gets +//! +//! - **No directive file: the full suite** — every class (a) panic-attack +//! subcommand (see [`crate::suite`]), all severities, 300 s per command +//! inside the suite budget (owner ruling 2026-11-13). +//! - **A directive file keeps exactly what it allows and denies.** Before the +//! full-suite ruling a file with no `allow` key granted the static trio +//! (`assail`, `adjudicate`, `diagnostics`); it still grants exactly that, so +//! no repository is widened behind its back. `allow = [...]` replaces that +//! set; `deny = [...]` removes from it, and a deny wins over an allow of the +//! same name. Widening needs an explicit `allow`. +//! - **A file that exists but cannot be parsed is an error**, not a fallback to +//! the (now full) default: failing open would widen a repository whose +//! directive is broken. Callers must surface the error. //! //! ## Directive Format (A2ML / TOML-shaped) //! @@ -22,9 +35,14 @@ //! timeout-seconds = 300 //! ``` //! +//! Names are panic-attack subcommands: `assail`, `assay`, `adjudicate`, +//! `diagnostics`, `attack`, `assault`, `ambush`, `amuck`, `abduct`, `axial`. +//! An unknown name is ignored with a note (it can only ever narrow, never +//! widen), and the note is printed. +//! //! The SCM form was retired 2026-04-17; no fallback is supported. -use crate::config::{AllowedCommands, MinSeverity, PanicbotConfig}; +use crate::config::{AllowedCommands, MinSeverity, PanicbotConfig, Subcommand}; use anyhow::{Context, Result}; use serde::Deserialize; use std::path::Path; @@ -35,15 +53,15 @@ const DIRECTIVE_PATH: &str = ".machine_readable/bot_directives/panicbot.a2ml"; /// Load panicbot configuration from a repository's directive file. /// -/// If the file doesn't exist, returns the default config (static analysis -/// only, all severities, 300s timeout). If the file exists but can't be -/// parsed, returns an error. +/// If the file doesn't exist, returns the default config: the full suite, all +/// severities, 300 s per command. If the file exists but can't be read or +/// parsed, returns an error (never the default: see the module docs). pub fn load_directives(repo_path: &Path) -> Result { let directive_file = repo_path.join(DIRECTIVE_PATH); if !directive_file.exists() { tracing::debug!( - "No directive file at {}, using defaults", + "No directive file at {}, using the full suite", directive_file.display() ); return Ok(PanicbotConfig::default()); @@ -62,6 +80,8 @@ struct DirectiveFile { #[serde(default)] allow: Option>, #[serde(default)] + deny: Option>, + #[serde(default)] config: Option, #[serde(default, rename = "min-severity")] min_severity_top: Option, @@ -78,17 +98,50 @@ struct ConfigBlock { } /// Parse a panicbot directive from its A2ML/TOML content. +/// +/// The content is a directive FILE, so the baseline is what a file granted +/// before the full-suite ruling (the static trio), not the no-file default. fn parse_directive(content: &str) -> Result { - let mut config = PanicbotConfig::default(); + let mut config = PanicbotConfig { + allowed_commands: AllowedCommands::legacy_static(), + ..PanicbotConfig::default() + }; let file: DirectiveFile = toml::from_str(content).context("Failed to parse TOML")?; - // Allow list - if let Some(allowed) = file.allow { - config.allowed_commands = AllowedCommands { - assail: allowed.iter().any(|s| s == "assail"), - adjudicate: allowed.iter().any(|s| s == "adjudicate"), - diagnostics: allowed.iter().any(|s| s == "diagnostics"), - }; + // Allow list: an explicit allow REPLACES the baseline. + if let Some(allowed) = &file.allow { + config.allowed_commands = AllowedCommands::none(); + for name in allowed { + match Subcommand::parse(name) { + Some(cmd) => config.allowed_commands.set(cmd, true), + None => config + .notes + .push(format!("directive allow: unknown command '{name}' ignored")), + } + } + } + + // Deny list: removes from whatever is allowed; a deny wins. + if let Some(denied) = &file.deny { + for name in denied { + match Subcommand::parse(name) { + Some(cmd) => { + if file + .allow + .as_ref() + .is_some_and(|a| a.iter().any(|n| n == name)) + { + config.notes.push(format!( + "directive lists '{name}' in both allow and deny: deny wins" + )); + } + config.allowed_commands.set(cmd, false); + } + None => config + .notes + .push(format!("directive deny: unknown command '{name}' ignored")), + } + } } // min-severity: prefer [config] block, fall back to top-level kebab key. @@ -196,3 +249,139 @@ timeout-seconds = 120 assert_eq!(config.min_severity, MinSeverity::Low); } } + +#[cfg(test)] +mod suite_tests { + use super::*; + + fn allowed_names(config: &PanicbotConfig) -> Vec<&'static str> { + Subcommand::ALL + .into_iter() + .filter(|c| config.allowed_commands.allows(*c)) + .map(|c| c.name()) + .collect() + } + + fn repo_with_directive(content: &str) -> tempfile::TempDir { + let d = tempfile::tempdir().unwrap(); + let dir = d.path().join(".machine_readable/bot_directives"); + std::fs::create_dir_all(&dir).unwrap(); + std::fs::write(dir.join("panicbot.a2ml"), content).unwrap(); + d + } + + #[test] + fn no_directive_file_means_the_full_suite() { + let d = tempfile::tempdir().unwrap(); + let config = load_directives(d.path()).unwrap(); + assert_eq!(config.allowed_commands, AllowedCommands::all()); + assert_eq!(allowed_names(&config).len(), Subcommand::ALL.len()); + } + + #[test] + fn a_directive_file_that_matches_todays_denies_keeps_exactly_todays_scope() { + // The directive shape the estate ships: static trio allowed, the six + // dynamic modes denied. `assay` is new and is NOT in the file, so it + // must stay off: the file's scope is not silently widened. + let d = repo_with_directive( + r#" +schema_version = "1.0" +bot = "panicbot" +scope = "static-analysis" +allow = ["assail", "adjudicate", "diagnostics"] +deny = ["attack", "assault", "ambush", "amuck", "abduct", "axial"] + +[config] +min-severity = "low" +timeout-seconds = 300 +"#, + ); + let config = load_directives(d.path()).unwrap(); + assert_eq!( + allowed_names(&config), + ["assail", "adjudicate", "diagnostics"] + ); + assert!(!config.allowed_commands.assay); + assert!(config.notes.is_empty()); + } + + #[test] + fn a_file_with_no_allow_key_keeps_the_static_trio_and_is_not_widened() { + let d = repo_with_directive("[config]\nmin-severity = \"high\"\n"); + let config = load_directives(d.path()).unwrap(); + assert_eq!( + allowed_names(&config), + ["assail", "adjudicate", "diagnostics"] + ); + assert_eq!(config.min_severity, MinSeverity::High); + } + + #[test] + fn deny_alone_narrows_the_baseline() { + let config = parse_directive("deny = [\"adjudicate\"]\n").unwrap(); + assert_eq!(allowed_names(&config), ["assail", "diagnostics"]); + } + + #[test] + fn an_explicit_allow_can_widen_to_any_subcommand() { + let config = parse_directive("allow = [\"assail\", \"assay\", \"assault\"]\n").unwrap(); + assert_eq!(allowed_names(&config), ["assail", "assay", "assault"]); + } + + #[test] + fn deny_can_remove_any_subcommand_from_an_allow() { + for cmd in Subcommand::ALL { + let content = format!( + "allow = [{}]\ndeny = [\"{}\"]\n", + Subcommand::ALL + .iter() + .map(|c| format!("\"{}\"", c.name())) + .collect::>() + .join(", "), + cmd.name() + ); + let config = parse_directive(&content).unwrap(); + assert!( + !config.allowed_commands.allows(cmd), + "{} not denied", + cmd.name() + ); + for other in Subcommand::ALL.into_iter().filter(|o| *o != cmd) { + assert!( + config.allowed_commands.allows(other), + "{} lost", + other.name() + ); + } + } + } + + #[test] + fn a_name_in_both_lists_is_denied_and_noted() { + let config = parse_directive("allow = [\"assail\"]\ndeny = [\"assail\"]\n").unwrap(); + assert!(!config.allowed_commands.assail); + assert!( + config.notes.iter().any(|n| n.contains("deny wins")), + "{:?}", + config.notes + ); + } + + #[test] + fn unknown_names_are_ignored_loudly_and_never_widen() { + let config = + parse_directive("allow = [\"assail\", \"tui\"]\ndeny = [\"asail\"]\n").unwrap(); + assert_eq!(allowed_names(&config), ["assail"]); + assert_eq!(config.notes.len(), 2, "{:?}", config.notes); + assert!(config.notes.iter().any(|n| n.contains("'tui'"))); + assert!(config.notes.iter().any(|n| n.contains("'asail'"))); + } + + #[test] + fn a_broken_directive_file_is_an_error_not_the_default() { + // Falling back to the default would hand a repository whose directive + // is broken the FULL suite. + let d = repo_with_directive("allow = [ this is not toml\n"); + assert!(load_directives(d.path()).is_err()); + } +} diff --git a/bots/panicbot/src/fleet.rs b/bots/panicbot/src/fleet.rs index 0846861a..f17c4e33 100644 --- a/bots/panicbot/src/fleet.rs +++ b/bots/panicbot/src/fleet.rs @@ -4,8 +4,9 @@ //! Registers as `BotId::Panicbot` in gitbot-shared-context. Provides the //! standard fleet entry point (`run_fleet_scan`) that: //! 1. Reads per-repo directives (`.machine_readable/bot_directives/panicbot.scm`) -//! 2. Invokes `panic-attack assail` via the scanner module -//! 3. Translates WeakPoints into Finding structs +//! 2. Runs the allowed panic-attack suite (full suite by default, see `suite`) +//! 3. Translates each part's results into Finding structs (assail WeakPoints, +//! assay candidates, dynamic-mode results when a target is given) //! 4. Classifies fixable vs unfixable //! 5. Writes unfixable to `.panicbot/PANICBOT-FINDINGS.a2ml` //! 6. Returns FindingSet for the fleet pipeline @@ -13,6 +14,7 @@ use crate::a2ml_writer; use crate::directives; use crate::scanner; +use crate::suite::{self, SuiteOptions, SuiteRun}; use crate::translator; use gitbot_shared_context::bot::{BotExecution, BotId, BotInfo}; use gitbot_shared_context::finding::FindingSet; @@ -28,11 +30,12 @@ pub fn bot_info() -> BotInfo { /// Controls scan depth and whether findings can block PRs. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum BotMode { - /// Report findings, don't block. Runs `assail` only. + /// Report findings, don't block. Runs the allowed suite. Advisor, - /// Full suite: `assail` + `adjudicate` + `diagnostics`. + /// Report findings, don't block. Same scope as Advisor since the full-suite + /// ruling (2026-11-13); the name is kept for callers. Auditor, - /// Block PRs on critical findings. Runs `assail` + strict thresholds. + /// Block PRs on critical findings. Runs the allowed suite + strict thresholds. Guardian, } @@ -57,50 +60,54 @@ impl BotMode { /// /// Performs the complete scan pipeline: /// 1. Load per-repo directives -/// 2. Run panic-attack assail -/// 3. Translate findings -/// 4. Classify fixable/unfixable -/// 5. Write A2ML debt register -/// 6. Return FindingSet + BotExecution +/// 2. Run the allowed panic-attack suite (see [`crate::suite`]) +/// 3. Classify fixable/unfixable +/// 4. Write A2ML debt register +/// 5. Return FindingSet + BotExecution pub fn run_fleet_scan(dir: &Path, mode: BotMode) -> (FindingSet, BotExecution) { + let (set, execution, _) = run_fleet_suite(dir, mode, &SuiteOptions::from_env(None)); + (set, execution) +} + +/// [`run_fleet_scan`] that also returns the per-part results and takes the +/// suite options (target for the dynamic modes). +pub fn run_fleet_suite( + dir: &Path, + mode: BotMode, + opts: &SuiteOptions, +) -> (FindingSet, BotExecution, Option) { let mut execution = BotExecution::new(BotId::Panicbot); execution.start(); - // Step 1: Load per-repo directives + // Step 1: Load per-repo directives. A directive file that exists but does + // not parse is a failure, never the (full-suite) default: failing open + // would widen the scope of a repository whose directive is broken. let config = match directives::load_directives(dir) { Ok(c) => c, Err(e) => { - tracing::warn!("Failed to load directives: {}, using defaults", e); - crate::config::PanicbotConfig::default() - } - }; - - // Step 2: Run panic-attack assail - if !config.allowed_commands.assail { - tracing::info!("assail command disabled by directives, skipping scan"); - execution.skip("assail disabled by bot directives"); - return (FindingSet::new(), execution); - } - - let assail_result = scanner::run_assail(dir, config.timeout); - let report = match assail_result { - Ok(r) => r, - Err(e) => { - let msg = format!("panic-attack assail failed: {}", e); + let msg = format!("bot directives unreadable, nothing run: {e:#}"); tracing::error!("{}", msg); execution.fail(&msg); - return (FindingSet::new(), execution); + return (FindingSet::new(), execution, None); } }; - // Step 3: Translate WeakPoints into Findings - let findings = translator::translate_all(&report.weak_points, &config); - let findings_count = findings.len(); + // Step 2: Run the allowed suite. + let run = suite::run_suite(dir, &config, opts); + for line in run.table() { + tracing::info!("{}", line); + } + if !run.analysed() && run.errored().is_empty() { + execution.skip("no panic-attack part ran: all denied by bot directives or not applicable"); + return (FindingSet::new(), execution, Some(run)); + } - // Step 4: Classify fixable vs unfixable + // Step 3: Classify fixable vs unfixable + let findings = run.findings.clone(); + let findings_count = findings.len(); let (fixable, unfixable) = translator::classify_fixability(&findings); - // Step 5: Write A2ML debt register (if there are unfixable findings) + // Step 4: Write A2ML debt register (if there are findings) let repo_name = dir .file_name() .map(|n| n.to_string_lossy().to_string()) @@ -124,14 +131,15 @@ pub fn run_fleet_scan(dir: &Path, mode: BotMode) -> (FindingSet, BotExecution) { } } - // Step 6: Build FindingSet for fleet pipeline + // Step 5: Build FindingSet for fleet pipeline let mut finding_set = FindingSet::new(); finding_set.extend(findings); // Count files from weak point locations - let files_analyzed: std::collections::HashSet<_> = report - .weak_points + let files_analyzed: std::collections::HashSet<_> = run + .assail .iter() + .flat_map(|r| r.weak_points.iter()) .filter_map(|wp| wp.location.as_ref()) .filter_map(|loc| loc.split(':').next()) .collect(); @@ -139,6 +147,11 @@ pub fn run_fleet_scan(dir: &Path, mode: BotMode) -> (FindingSet, BotExecution) { let error_count = finding_set.errors().len(); execution.complete(findings_count, error_count, files_analyzed.len()); + // A failed part is a failed run, whatever the other parts found. + if let Some(summary) = run.error_summary() { + execution.fail(&summary); + } + // In guardian mode, mark as failed if there are critical findings if mode.blocks_on_error() && finding_set.has_errors() { execution.fail(&format!( @@ -147,7 +160,7 @@ pub fn run_fleet_scan(dir: &Path, mode: BotMode) -> (FindingSet, BotExecution) { )); } - (finding_set, execution) + (finding_set, execution, Some(run)) } #[cfg(test)] diff --git a/bots/panicbot/src/lib.rs b/bots/panicbot/src/lib.rs index 399cc887..d0f8522b 100644 --- a/bots/panicbot/src/lib.rs +++ b/bots/panicbot/src/lib.rs @@ -41,4 +41,5 @@ pub mod config; pub mod directives; pub mod fleet; pub mod scanner; +pub mod suite; pub mod translator; diff --git a/bots/panicbot/src/main.rs b/bots/panicbot/src/main.rs index 680ca977..30bee570 100644 --- a/bots/panicbot/src/main.rs +++ b/bots/panicbot/src/main.rs @@ -5,7 +5,7 @@ //! subcommand handler based on command-line arguments. //! //! Subcommands: -//! - `scan ` — Run panic-attack assail and show findings +//! - `scan ` — Run the allowed panic-attack suite and show findings //! - `fleet ` — Run as a fleet member (produces FindingSet + A2ML) //! - `report ` — Generate A2ML debt register only @@ -18,6 +18,7 @@ use panicbot::cli::{Cli, Command, OutputFormat}; use panicbot::directives; use panicbot::fleet; use panicbot::scanner; +use panicbot::suite::{self, SuiteOptions}; use panicbot::translator; fn main() -> Result<()> { @@ -37,21 +38,61 @@ fn main() -> Result<()> { .init(); match cli.command { - Command::Scan { dir } => cmd_scan(&dir, cli.format), - Command::Fleet { dir, mode } => cmd_fleet(&dir, &mode, cli.format), + Command::Scan { + dir, + target, + budget_seconds, + } => cmd_scan(&dir, target, budget_seconds, cli.format), + Command::Fleet { dir, mode, target } => cmd_fleet(&dir, &mode, target, cli.format), Command::Report { dir, output } => cmd_report(&dir, output.as_deref(), cli.format), } } -/// Scan a directory with panic-attack and display translated findings. -fn cmd_scan(dir: &std::path::Path, format: OutputFormat) -> Result<()> { +/// Exit status when at least one panic-attack part failed. The findings of the +/// parts that ran are still printed; the report-back driver reads this status +/// as an `error` row carrying the failure list. +const EXIT_PART_ERROR: i32 = 3; +/// Exit status when no part analysed anything (all denied by the directives or +/// not applicable). The driver reads it as `not_run`, never as a clean `ran`. +const EXIT_NOTHING_RAN: i32 = 4; + +const NOTHING_RAN: &str = + "no part analysed the repository (all denied by its bot directives or not applicable)"; + +/// Scan a directory with the allowed panic-attack suite and display translated +/// findings. +fn cmd_scan( + dir: &std::path::Path, + target: Option, + budget_seconds: Option, + format: OutputFormat, +) -> Result<()> { tracing::info!("Scanning: {}", dir.display()); - let config = directives::load_directives(dir).unwrap_or_default(); - let report = - scanner::run_assail(dir, config.timeout).context("Failed to run panic-attack assail")?; + // A directive file that exists but does not parse stops the run: falling + // back to the default (now the full suite) would widen the repository. + let mut config = + directives::load_directives(dir).context("bot directives unreadable, nothing run")?; + if let Some(secs) = budget_seconds { + config.budget = std::time::Duration::from_secs(secs); + } - let findings = translator::translate_all(&report.weak_points, &config); + let run = suite::run_suite(dir, &config, &SuiteOptions::from_env(target)); + let findings = run.findings.clone(); + + // Failure summary FIRST on stderr: the report-back driver quotes the first + // line as the row's reason. + let part_error = run.error_summary(); + if let Some(summary) = &part_error { + eprintln!("panicbot: {summary}"); + } + for note in &config.notes { + eprintln!("panicbot: directive note: {note}"); + } + for line in run.table() { + eprintln!("panicbot: {line}"); + } + eprintln!("panicbot: suite wall time {} ms", run.wall_ms); match format { OutputFormat::Text => { @@ -83,12 +124,40 @@ fn cmd_scan(dir: &std::path::Path, format: OutputFormat) -> Result<()> { } } OutputFormat::Json => { + // A findings ARRAY means every allowed part that applies ran. A run + // that did not is an object naming its status and reason (the same + // form ponsbot uses), never an array: the report-back driver turns + // it into an `error` or `not_run` row, so nothing reads as clean. + let value = if let Some(summary) = &part_error { + serde_json::json!({ + "status": "error", + "reason": summary, + "findings": findings, + "parts": run.parts, + }) + } else if !run.analysed() { + serde_json::json!({ + "status": "not_run", + "reason": NOTHING_RAN, + "findings": findings, + "parts": run.parts, + }) + } else { + serde_json::to_value(&findings).context("Failed to serialize findings")? + }; let json = - serde_json::to_string_pretty(&findings).context("Failed to serialize findings")?; + serde_json::to_string_pretty(&value).context("Failed to serialize findings")?; println!("{}", json); } } + if part_error.is_some() { + std::process::exit(EXIT_PART_ERROR); + } + if !run.analysed() { + eprintln!("panicbot: {NOTHING_RAN}"); + std::process::exit(EXIT_NOTHING_RAN); + } // Exit with error code if there are blocking issues if findings.iter().any(|f| f.severity.blocks_release()) { std::process::exit(1); @@ -98,7 +167,12 @@ fn cmd_scan(dir: &std::path::Path, format: OutputFormat) -> Result<()> { } /// Run as a fleet member. -fn cmd_fleet(dir: &std::path::Path, mode_str: &str, format: OutputFormat) -> Result<()> { +fn cmd_fleet( + dir: &std::path::Path, + mode_str: &str, + target: Option, + format: OutputFormat, +) -> Result<()> { let mode = fleet::BotMode::parse(mode_str).unwrap_or_else(|| { tracing::warn!("Unknown mode '{}', defaulting to advisor", mode_str); fleet::BotMode::Advisor @@ -106,7 +180,8 @@ fn cmd_fleet(dir: &std::path::Path, mode_str: &str, format: OutputFormat) -> Res tracing::info!("Running as fleet member in {:?} mode", mode); - let (finding_set, execution) = fleet::run_fleet_scan(dir, mode); + let (finding_set, execution, run) = + fleet::run_fleet_suite(dir, mode, &SuiteOptions::from_env(target)); match format { OutputFormat::Text => { @@ -117,6 +192,12 @@ fn cmd_fleet(dir: &std::path::Path, mode_str: &str, format: OutputFormat) -> Res println!("Errors: {}", finding_set.errors().len()); println!("Warnings: {}", finding_set.warnings().len()); println!(); + if let Some(run) = &run { + for line in run.table() { + println!(" {line}"); + } + println!(); + } for finding in finding_set.into_iter() { let location = finding.location_string().unwrap_or_default(); @@ -134,6 +215,7 @@ fn cmd_fleet(dir: &std::path::Path, mode_str: &str, format: OutputFormat) -> Res "bot_info": fleet::bot_info(), "execution": execution, "findings": finding_set, + "parts": run.as_ref().map(|r| &r.parts), }); let json = serde_json::to_string_pretty(&output) .context("Failed to serialize fleet output")?; @@ -152,7 +234,8 @@ fn cmd_report( ) -> Result<()> { tracing::info!("Generating A2ML report for: {}", dir.display()); - let config = directives::load_directives(dir).unwrap_or_default(); + let config = + directives::load_directives(dir).context("bot directives unreadable, nothing run")?; let report = scanner::run_assail(dir, config.timeout).context("Failed to run panic-attack assail")?; diff --git a/bots/panicbot/src/scanner.rs b/bots/panicbot/src/scanner.rs index 9bc0883a..9868332d 100644 --- a/bots/panicbot/src/scanner.rs +++ b/bots/panicbot/src/scanner.rs @@ -85,6 +85,10 @@ pub struct WeakPoint { } /// Report from `panic-attack adjudicate` — cross-report verdict. +/// +/// NOTE: this models a report shape panic-attack 2.5.5 does not write (it +/// writes [`AdjudicateVerdict`]). Kept because it is public; the suite uses +/// [`run_adjudicate_verdict`]. #[derive(Debug, Clone, Serialize, Deserialize)] pub struct AdjudicateReport { /// Number of reports adjudicated. @@ -311,18 +315,23 @@ pub fn run_adjudicate(reports: &[PathBuf], timeout: Duration) -> Result Result { - // First check if the binary exists at all - let binary = find_panic_attack_binary()?; + // Fail early, and clearly, when the binary is not there at all. + find_panic_attack_binary()?; - // Try `--version` for a lightweight health check - let version_output = Command::new(&binary) - .arg("--version") - .output() + // `--version` is the lightweight health check. Timed like every other invocation: a hung binary must not hang the lane. + let version_run = run_panic_attack_plain(&["--version"], timeout) .context("Failed to execute panic-attack --version")?; - let version = String::from_utf8_lossy(&version_output.stdout) + // First line only, bounded: this ends up in a part's detail. + let version: String = version_run + .stdout + .lines() + .next() + .unwrap_or("") .trim() - .to_string(); + .chars() + .take(80) + .collect(); // If diagnostics subcommand exists, use it match invoke_panic_attack(&["diagnostics"], timeout) { @@ -344,7 +353,7 @@ pub fn run_diagnostics(timeout: Duration) -> Result { } else { version }, - healthy: version_output.status.success(), + healthy: version_run.status.success(), supported_languages: Vec::new(), messages: vec!["Diagnostics constructed from --version check".to_string()], }) @@ -402,6 +411,281 @@ pub fn run_migration_diff( .context("Failed to parse panic-attack migration-diff output as JSON") } +// ── subcommands beyond `assail` (the full suite) ──────────────────────────── +// +// These all write their report with `-o` into a private scratch directory and +// run with that directory as the working directory. panic-attack otherwise +// drops `reports/`, `runtime/` and similar into the CWD, and the report-back +// driver's CWD is the scanned repository: a scan must not change the tree. + +/// A private scratch directory, removed on drop. +pub struct Scratch { + path: PathBuf, +} + +impl Scratch { + /// Create a fresh directory under the system temp dir. + pub fn new() -> Result { + let path = std::env::temp_dir().join(format!("panicbot-{}", uuid::Uuid::new_v4())); + std::fs::create_dir_all(&path) + .with_context(|| format!("Failed to create scratch dir {}", path.display()))?; + Ok(Self { path }) + } + + /// The directory. + pub fn path(&self) -> &Path { + &self.path + } +} + +impl Drop for Scratch { + fn drop(&mut self) { + let _ = std::fs::remove_dir_all(&self.path); + } +} + +/// One proven-library substitution candidate from `panic-attack assay`. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct AssayCandidate { + /// Stable candidate id (e.g. `safe-path`). + pub id: String, + /// Name of the proven replacement. + pub proven_name: String, + /// The proof artifact backing the replacement. + pub proof_backing: String, + /// `offered`, `already-applied` or `no-replacement-source`. + pub status: String, + /// Catalogue confidence (0.0 - 1.0). + pub confidence: f64, + /// Where the unproven pattern occurs. + #[serde(default)] + pub target_hits: Vec, +} + +/// One occurrence of an unproven pattern. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct AssayHit { + /// File (absolute or relative to the target). + pub file: PathBuf, + /// 1-based line. + pub line: u32, +} + +/// Report from `panic-attack assay `. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct AssayReport { + /// Candidates found. + pub candidates: Vec, + /// How many catalogue entries were checked. + #[serde(default)] + pub catalogue_entries: usize, + /// Free-text notes from the survey. + #[serde(default)] + pub notes: Vec, +} + +/// One per-axis result of a dynamic run. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct AxisResult { + /// Attack axis (`cpu`, `memory`, ...). + pub axis: String, + /// Whether the program survived the axis. + pub success: bool, + /// The axis did not run. + #[serde(default)] + pub skipped: bool, + /// Why it did not run. + #[serde(default)] + pub skip_reason: Option, + /// The program's exit code, if it exited. + #[serde(default)] + pub exit_code: Option, + /// Crashes observed. + #[serde(default)] + pub crashes: Vec, + /// Bug signatures detected. + #[serde(default)] + pub signatures_detected: Vec, +} + +/// Report from `panic-attack assault` or `panic-attack ambush`. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct DynamicReport { + /// Per-axis results. + pub attack_results: Vec, + /// Total crashes. + #[serde(default)] + pub total_crashes: usize, + /// Total bug signatures. + #[serde(default)] + pub total_signatures: usize, +} + +/// A verdict from `panic-attack adjudicate` (the real 2.5 report shape). +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct AdjudicateVerdict { + /// `pass`, `warn` or `fail`. + pub verdict: String, + /// Reports it understood. + pub processed_reports: usize, + /// Reports it could not read or did not recognise. + pub failed_reports: usize, + /// Per-report problems. + #[serde(default)] + pub notes: Vec, + /// Prioritised conclusions. + #[serde(default)] + pub priorities: Vec, +} + +/// One prioritised conclusion of an adjudication. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct AdjudicatePriority { + /// `high`, `medium`, `info`. + pub level: String, + /// What it says. + pub message: String, +} + +/// Run a report-writing panic-attack subcommand in `scratch` and return the +/// parsed JSON report it wrote to `-o`. +/// +/// Any non-zero exit, a timeout, a missing report file or a report that is not +/// JSON is an error: a part never reads as clean because it printed nothing. +fn run_to_report(args: &[&str], scratch: &Scratch, timeout: Duration) -> Result { + let out = scratch.path().join("report.json"); + let out_s = out.display().to_string(); + let mut full: Vec<&str> = args.to_vec(); + full.push("-o"); + full.push(&out_s); + let run = run_panic_attack_raw_in(&full, timeout, Some(scratch.path()))?; + if !run.status.success() { + bail!( + "panic-attack {} exited with {}: {}", + args.first().copied().unwrap_or("?"), + run.status, + first_diagnostic(&run.stderr) + ); + } + let raw = std::fs::read_to_string(&out).with_context(|| { + format!( + "panic-attack {} exited 0 without writing its report", + args.first().copied().unwrap_or("?") + ) + })?; + serde_json::from_str(extract_json(&raw)).with_context(|| { + format!( + "panic-attack {} wrote a report that is not JSON", + args.first().copied().unwrap_or("?") + ) + }) +} + +/// Run `panic-attack assay `: the proven-library substitution survey. +pub fn run_assay(target: &Path, timeout: Duration) -> Result { + let scratch = Scratch::new()?; + let t = target.display().to_string(); + let value = run_to_report(&["assay", &t], &scratch, timeout)?; + parse_assay_report(value) +} + +/// Validate and parse an `assay` report. A report with no `candidates` array is +/// not an empty survey, it is a malformed one. +pub fn parse_assay_report(value: serde_json::Value) -> Result { + if !value.get("candidates").is_some_and(|c| c.is_array()) { + bail!("panic-attack assay wrote a report without a candidates array"); + } + serde_json::from_value(value).context("panic-attack assay wrote an unexpected report shape") +} + +/// Run `panic-attack assault` or `panic-attack ambush` against a built program. +/// +/// EXECUTES `program`. The caller is responsible for the security gate (see +/// `suite::dynamic_refusal`). `source` is the directory both modes run their +/// own assail phase over; only the attack results are used here, so the +/// static findings are not reported twice. +/// The raw report is copied to `keep` for `adjudicate`. +pub fn run_dynamic( + mode: &str, + program: &Path, + source: &Path, + axis_seconds: u64, + timeout: Duration, + keep: &Path, +) -> Result { + if mode != "assault" && mode != "ambush" { + bail!("run_dynamic: '{mode}' is not a dynamic mode panicbot runs"); + } + let scratch = Scratch::new()?; + let p = program.display().to_string(); + let s = source.display().to_string(); + let d = axis_seconds.to_string(); + let value = run_to_report( + &[mode, &p, "--source", &s, "-d", &d, "--quiet"], + &scratch, + timeout, + )?; + // Keep the raw report: `adjudicate` reads it from disk. + std::fs::write(keep, serde_json::to_vec(&value)?) + .with_context(|| format!("Failed to keep the {mode} report at {}", keep.display()))?; + parse_dynamic_report(value) +} + +/// Validate and parse an `assault`/`ambush` report. No per-axis results means +/// nothing was attacked, which is an error, never a clean run. +pub fn parse_dynamic_report(value: serde_json::Value) -> Result { + let report: DynamicReport = serde_json::from_value(value) + .context("panic-attack dynamic run wrote an unexpected report shape")?; + if report.attack_results.is_empty() { + bail!("panic-attack dynamic run produced no per-axis results (nothing was attacked)"); + } + Ok(report) +} + +/// Run `panic-attack adjudicate` over dynamic-mode reports and return its +/// verdict. +/// +/// panic-attack 2.5.5 prints `pass (processed 0, failed 1)` and exits 0 for a +/// missing or unrecognised report, so exit status is not evidence of a verdict: +/// [`parse_adjudicate_verdict`] requires every input to have been processed. +pub fn run_adjudicate_verdict(reports: &[PathBuf], timeout: Duration) -> Result { + let scratch = Scratch::new()?; + let mut args: Vec = vec!["adjudicate".to_string()]; + args.extend(reports.iter().map(|r| r.display().to_string())); + let arg_refs: Vec<&str> = args.iter().map(String::as_str).collect(); + let value = run_to_report(&arg_refs, &scratch, timeout)?; + parse_adjudicate_verdict(value, reports.len()) +} + +/// Validate and parse an `adjudicate` report for `expected` inputs. +pub fn parse_adjudicate_verdict( + value: serde_json::Value, + expected: usize, +) -> Result { + let v: AdjudicateVerdict = serde_json::from_value(value) + .context("panic-attack adjudicate wrote an unexpected report shape")?; + if v.failed_reports > 0 || v.processed_reports < expected { + bail!( + "panic-attack adjudicate understood {} of {} report(s) ({} failed){}; its '{}' verdict is not evidence", + v.processed_reports, + expected, + v.failed_reports, + v.notes + .first() + .map(|n| format!(": {n}")) + .unwrap_or_default(), + v.verdict + ); + } + if !matches!(v.verdict.as_str(), "pass" | "warn" | "fail") { + bail!( + "panic-attack adjudicate wrote unknown verdict {:?}", + v.verdict + ); + } + Ok(v) +} + /// Locate the `panic-attack` binary on PATH. /// /// Returns the path to the binary, or an error if not found. @@ -455,6 +739,27 @@ struct RawRun { /// the call fails: a hung scanner must not hang the lane or read as a run). /// The exit status is returned for the caller to judge. fn run_panic_attack_raw(args: &[&str], timeout: Duration) -> Result { + run_panic_attack_raw_in(args, timeout, None) +} + +/// [`run_panic_attack_raw`] with an explicit working directory. +fn run_panic_attack_raw_in(args: &[&str], timeout: Duration, cwd: Option<&Path>) -> Result { + run_panic_attack_cmd(args, timeout, cwd, true) +} + +/// Run `panic-attack ` as given (no `--output-format`), with the same +/// enforced timeout. For `--version`. +fn run_panic_attack_plain(args: &[&str], timeout: Duration) -> Result { + run_panic_attack_cmd(args, timeout, None, false) +} + +/// The one place panic-attack is spawned and its timeout enforced. +fn run_panic_attack_cmd( + args: &[&str], + timeout: Duration, + cwd: Option<&Path>, + json: bool, +) -> Result { use std::io::Read; let binary = find_panic_attack_binary()?; @@ -467,8 +772,13 @@ fn run_panic_attack_raw(args: &[&str], timeout: Duration) -> Result { let mut cmd = Command::new(&binary); cmd.args(args); - cmd.arg("--output-format"); - cmd.arg("json"); + if let Some(dir) = cwd { + cmd.current_dir(dir); + } + if json { + cmd.arg("--output-format"); + cmd.arg("json"); + } cmd.stdin(std::process::Stdio::null()); cmd.stdout(std::process::Stdio::piped()); cmd.stderr(std::process::Stdio::piped()); @@ -784,3 +1094,124 @@ mod parity_tests { assert!(parse_assail_output(json).is_err()); } } + +#[cfg(test)] +mod suite_parse_tests { + use super::*; + use serde_json::json; + + #[test] + fn adjudicate_that_understood_nothing_is_an_error_not_a_pass() { + // Real panic-attack 2.5.5 output for a missing/unrecognised report: + // exit 0, verdict "pass", processed 0, failed 1. + let v = json!({ + "verdict": "pass", "processed_reports": 0, "failed_reports": 1, + "notes": ["assail.json: unsupported report format"], "priorities": [] + }); + let err = parse_adjudicate_verdict(v, 1).unwrap_err().to_string(); + assert!(err.contains("understood 0 of 1"), "{err}"); + assert!(err.contains("unsupported report format"), "{err}"); + assert!(err.contains("not evidence"), "{err}"); + } + + #[test] + fn adjudicate_that_understood_only_some_inputs_is_an_error() { + let v = + json!({"verdict": "fail", "processed_reports": 1, "failed_reports": 1, "notes": []}); + assert!(parse_adjudicate_verdict(v, 2).is_err()); + } + + #[test] + fn adjudicate_verdict_is_read_when_every_input_was_understood() { + let v = json!({ + "verdict": "fail", "processed_reports": 2, "failed_reports": 0, "notes": [], + "priorities": [{"level": "high", "message": "1 crashes detected across assault reports"}], + "totals": {"total_crashes": 1} + }); + let ok = parse_adjudicate_verdict(v, 2).unwrap(); + assert_eq!(ok.verdict, "fail"); + assert_eq!(ok.priorities[0].level, "high"); + } + + #[test] + fn adjudicate_with_an_unknown_verdict_is_an_error() { + let v = json!({"verdict": "shrug", "processed_reports": 1, "failed_reports": 0}); + assert!(parse_adjudicate_verdict(v, 1).is_err()); + } + + #[test] + fn assay_report_needs_a_candidates_array() { + assert!(parse_assay_report(json!({})).is_err()); + assert!(parse_assay_report(json!({"candidates": "x"})).is_err()); + // The real "nothing found" report is a valid, empty survey. + let ok = parse_assay_report(json!({ + "candidates": [], "catalogue_entries": 2, "notes": ["no substitution candidates found for this target"] + })) + .unwrap(); + assert!(ok.candidates.is_empty()); + } + + #[test] + fn assay_candidate_with_hits_parses_from_the_real_shape() { + let ok = parse_assay_report(json!({ + "candidates": [{ + "id": "safe-path", "proven_name": "SafePath", + "proof_backing": "proven: src/Proven/SafePath/Operations.idr", + "dest_rel": "src/safe_path.rs", "port_present": false, + "target_hits": [{"file": "src/a.rs", "line": 3, "snippet": "canonicalize(p).unwrap_or_else"}], + "equivalence_basis": "x", "perf_note": "y", + "substitution_kind": "port-to-rust", "confidence": 0.9, "status": "offered" + }] + })) + .unwrap(); + assert_eq!(ok.candidates[0].target_hits[0].line, 3); + assert_eq!(ok.candidates[0].status, "offered"); + } + + #[test] + fn a_dynamic_report_with_no_axis_results_is_an_error() { + assert!(parse_dynamic_report(json!({"attack_results": [], "total_crashes": 0})).is_err()); + assert!(parse_dynamic_report(json!({})).is_err()); + } + + #[test] + fn a_dynamic_report_parses_from_the_real_shape() { + let ok = parse_dynamic_report(json!({ + "schema_version": "2.5", "total_crashes": 1, "total_signatures": 0, + "attack_results": [{ + "axis": "cpu", "program": "./crash.sh", "success": false, "skipped": false, + "exit_code": null, "peak_memory": 0, "duration": {"secs": 0, "nanos": 1}, + "crashes": [{"signal": null, "stderr": "", "stdout": ""}], + "signatures_detected": [] + }] + })) + .unwrap(); + assert_eq!(ok.attack_results[0].crashes.len(), 1); + assert!(!ok.attack_results[0].success); + } + + #[test] + fn only_assault_and_ambush_are_dynamic_modes_panicbot_runs() { + let err = run_dynamic( + "attack", + Path::new("p"), + Path::new("."), + 1, + Duration::from_secs(1), + Path::new("k"), + ) + .unwrap_err() + .to_string(); + assert!(err.contains("not a dynamic mode"), "{err}"); + } + + #[test] + fn scratch_is_removed_on_drop() { + let path = { + let s = Scratch::new().unwrap(); + assert!(s.path().is_dir()); + s.path().to_path_buf() + }; + assert!(!path.exists()); + } +} diff --git a/bots/panicbot/src/suite.rs b/bots/panicbot/src/suite.rs new file mode 100644 index 00000000..c64f397c --- /dev/null +++ b/bots/panicbot/src/suite.rs @@ -0,0 +1,1279 @@ +// SPDX-License-Identifier: MPL-2.0 +//! Suite — runs the allowed panic-attack subcommands and reports each part. +//! +//! Owner ruling 2026-11-13: panicbot runs the **full suite** unless the +//! repository's bot directives say otherwise. "Full suite" is every class (a) +//! subcommand below. Each subcommand is one *part*; every part ends as `ran`, +//! `not_applicable` (with its reason), `denied` (by the directives) or `error` +//! (loudly). Nothing is skipped silently and nothing is a fake pass. +//! +//! ## Inventory of panic-attack 2.5.5 subcommands +//! +//! Source: `src/main.rs` of panic-attack, each subcommand run against +//! `rsr-template-repo` on 2026-11-13 unless the reason says otherwise. +//! +//! **Class (a)**: a scan or analysis that runs non-interactively and yields +//! findings or a verdict. This is the suite. +//! +//! | Subcommand | Input | Yields | Panicbot runs it | +//! |---|---|---|---| +//! | `assail` | repo directory | weak points | always (if allowed) | +//! | `assay` | repo directory | proven-library substitution candidates (file and line) | always (if allowed) | +//! | `diagnostics` | none | tool health | always (if allowed); preflight only, no findings | +//! | `assault` | built program | per-axis crashes and bug signatures | only with `--target` (see below) | +//! | `ambush` | built program | same, under ambient stressors | only with `--target` | +//! | `adjudicate` | assault/ambush/amuck/abduct reports | pass/warn/fail verdict | only when a dynamic part produced a report | +//! | `attack` | built program, one axis | one axis of `assault` | `not_applicable`: `assault` runs every axis | +//! | `amuck` | one file and an exec program | mutation outcomes | `not_applicable`: panicbot has no source for the file/exec pairing | +//! | `abduct` | one file and an exec program | isolated-run outcome | `not_applicable`: same | +//! | `axial` | an exec program or existing reports | observed reactions | `not_applicable`: same (it rejects an assail report) | +//! +//! **Class (b)**: interactive or UI, never run: `tui`, `gui` (need a terminal +//! or display), `groove` (starts a long-lived server). +//! +//! **Class (c)**: infrastructure, side effects, or no repository scan; not part +//! of the suite: +//! +//! | Subcommand | Why not class (a) | +//! |---|---| +//! | `assimilate` | rewrites the tree (stages proven modules); an action, not a scan | +//! | `notify`, `sweep-tracker`, `query`, `campaign` (`register-pr`, `dismiss`, `status`, `poll`) | issue/PR side effects or a store of past findings, not a scan | +//! | `verisim-push`, `assemblyline`, `image` | push to a database, or batch-scan many repos and write a store | +//! | `attest`, `verify` | check an attestation sidecar; no repository analysis | +//! | `temporal` (`list`, `diff`) | reads the local scan-history store; errors without one | +//! | `diff`, `report`, `panll`, `a2ml-export`, `a2ml-import`, `manifest`, `analyze`, `aggregate` | consume or convert reports or prover output panicbot does not hold | +//! | `fingerprint` | a BLAKE3 hash of the tree; yields neither findings nor a verdict | +//! | `migration-snapshot`, `migration-diff` | ReScript-migration *metrics* (their embedded assail report duplicates `assail`); no findings; `--build-time` runs `rescript build` | +//! | `describe-contract`, `completions`, `help` | describe the CLI | +//! | `bridge` (`triage`, `status`) | class (a) in kind (CVE triage with reachability) but **held out**: owner ruling D4 of 2026-11-13 keeps the bridge gate outside panicbot (CVE-mitigation lifecycle, a sanctioned direct use), it needs the `http` feature and OSV network access, and it errors on a repo with no supported lockfile. Revisit with the owner | +//! +//! ## Dynamic modes: where the target comes from +//! +//! `assault` and `ambush` execute a program. Panicbot does not build the +//! scanned repository, so there is no program unless the operator gives one: +//! `--target PATH` (a built executable inside the scanned directory). Without +//! it the dynamic parts report `not_applicable` with that reason. None of the +//! CI callers passes a target, so in CI the dynamic parts are `not_applicable`. +//! +//! The security gate (checked 2026-11-13 over gitbot-fleet +//! `fleet-report-back-reusable.yml` and standards `fork-governance.yml`): both +//! run on plain `pull_request` with `contents: read` and no secrets, which is +//! unprivileged. A reusable workflow cannot know its caller's trigger, though, +//! so [`dynamic_refusal`] makes panicbot refuse to execute repository code when +//! `GITHUB_EVENT_NAME` is `pull_request_target` or `workflow_run`. +//! +//! ## Rule ids +//! +//! `assail` findings keep panic-attack's canonical `PAxxx` ids. The other +//! subcommands have no canonical id, so their findings use panicbot-local ids +//! with a `PB-` prefix (`PB-ASSAY-`, `PB-DYN-CRASH`, +//! `PB-DYN-SIGNATURE`, `PB-DYN-FAIL`, `PB-ADJUDICATE-VERDICT`) rather than a +//! second numbering inside `PA`. Every finding names a file (gitbot-fleet #668): +//! assay names the hit's file, dynamic findings and the verdict name the target. +//! +//! ## Budget +//! +//! Parts run one after another. Each invocation gets the smaller of the +//! per-command timeout and what is left of the suite budget +//! ([`PanicbotConfig::budget`], 240 s, under the driver's 300 s adapter limit). +//! A part that cannot start inside the budget is an `error`. + +use crate::config::{PanicbotConfig, Subcommand}; +use crate::scanner::{ + self, AdjudicateVerdict, AssailReport, AssayReport, DiagnosticsReport, DynamicReport, Scratch, +}; +use crate::translator; +use anyhow::Result; +use gitbot_shared_context::bot::BotId; +use gitbot_shared_context::finding::{Finding, Severity}; +use gitbot_shared_context::triangle::TriangleTier; +use serde::Serialize; +use std::path::{Path, PathBuf}; +use std::time::{Duration, Instant}; + +/// Seconds each attack axis runs for in the dynamic modes. +pub const DYNAMIC_AXIS_SECONDS: u64 = 10; + +/// How one part of the suite ended. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum PartStatus { + /// It ran to a usable result (with or without findings). + Ran, + /// It cannot apply here; the detail says why. + NotApplicable, + /// The repository's directives deny it. + Denied, + /// It failed. Always loud. + Error, +} + +/// The outcome of one subcommand. +#[derive(Debug, Clone, Serialize)] +pub struct Part { + /// Which subcommand. + pub command: Subcommand, + /// How it ended. + pub status: PartStatus, + /// The reason (never empty unless it ran with nothing to add). + pub detail: String, + /// Findings it contributed after the severity filter. + pub findings: usize, + /// Wall time spent in it. + pub wall_ms: u128, +} + +/// Everything a suite run produced. +#[derive(Debug, Clone, Serialize)] +pub struct SuiteRun { + /// One entry per subcommand, in run order. + pub parts: Vec, + /// The findings of every part that ran. + #[serde(skip)] + pub findings: Vec, + /// The assail report, when assail ran (statistics for callers). + #[serde(skip)] + pub assail: Option, + /// Total wall time. + pub wall_ms: u128, +} + +impl SuiteRun { + /// The parts that ended in `error`. + pub fn errored(&self) -> Vec<&Part> { + self.parts + .iter() + .filter(|p| p.status == PartStatus::Error) + .collect() + } + + /// Whether any part that analyses the repository ran. A preflight alone + /// (`diagnostics`) is not an analysis: a run where nothing else ran must + /// not read as a clean scan. + pub fn analysed(&self) -> bool { + self.parts + .iter() + .any(|p| p.status == PartStatus::Ran && p.command != Subcommand::Diagnostics) + } + + /// One line naming every failed part, or `None` when none failed. + pub fn error_summary(&self) -> Option { + let errs = self.errored(); + if errs.is_empty() { + return None; + } + let list: Vec = errs + .iter() + .map(|p| format!("{}: {}", p.command.name(), p.detail)) + .collect(); + Some(format!( + "{} of {} panic-attack part(s) failed: {}", + errs.len(), + self.parts.len(), + list.join("; ") + )) + } + + /// One line per part, for logs and the text output. + pub fn table(&self) -> Vec { + self.parts + .iter() + .map(|p| { + let status = match p.status { + PartStatus::Ran => "ran", + PartStatus::NotApplicable => "not_applicable", + PartStatus::Denied => "denied", + PartStatus::Error => "ERROR", + }; + format!( + "{:<12} {:<15} {:>4} finding(s) {:>6} ms {}", + p.command.name(), + status, + p.findings, + p.wall_ms, + p.detail + ) + }) + .collect() + } +} + +/// How a suite run is invoked. +#[derive(Debug, Clone, Default)] +pub struct SuiteOptions { + /// A built program for the dynamic modes. None: they are `not_applicable`. + pub target: Option, + /// The CI event name (`GITHUB_EVENT_NAME`), for the dynamic-mode gate. + pub ci_event: Option, +} + +impl SuiteOptions { + /// Options from the environment, with an optional explicit target. + pub fn from_env(target: Option) -> Self { + Self { + target, + ci_event: std::env::var("GITHUB_EVENT_NAME").ok(), + } + } +} + +/// The panic-attack side of the suite, as a trait so the orchestration can be +/// tested without the binary. +pub trait Engine { + /// `panic-attack assail`. + fn assail(&self, dir: &Path, timeout: Duration) -> Result; + /// `panic-attack assay`. + fn assay(&self, dir: &Path, timeout: Duration) -> Result; + /// `panic-attack assault|ambush`, keeping the raw report at `keep`. + fn dynamic( + &self, + mode: &str, + program: &Path, + source: &Path, + timeout: Duration, + keep: &Path, + ) -> Result; + /// `panic-attack adjudicate`. + fn adjudicate(&self, reports: &[PathBuf], timeout: Duration) -> Result; + /// The tool's health. + fn diagnostics(&self, timeout: Duration) -> Result; +} + +/// The real panic-attack binary. +pub struct PanicAttack; + +impl Engine for PanicAttack { + fn assail(&self, dir: &Path, timeout: Duration) -> Result { + scanner::run_assail(dir, timeout) + } + fn assay(&self, dir: &Path, timeout: Duration) -> Result { + scanner::run_assay(dir, timeout) + } + fn dynamic( + &self, + mode: &str, + program: &Path, + source: &Path, + timeout: Duration, + keep: &Path, + ) -> Result { + scanner::run_dynamic(mode, program, source, DYNAMIC_AXIS_SECONDS, timeout, keep) + } + fn adjudicate(&self, reports: &[PathBuf], timeout: Duration) -> Result { + scanner::run_adjudicate_verdict(reports, timeout) + } + fn diagnostics(&self, timeout: Duration) -> Result { + scanner::run_diagnostics(timeout) + } +} + +/// Why panicbot must not execute repository code right now, if it must not. +/// +/// A reusable workflow cannot see its caller's trigger policy, and the +/// privileged triggers run with secrets or a write token. Refuse to execute +/// the scanned repository's code under them. +pub fn dynamic_refusal(opts: &SuiteOptions) -> Option { + match opts.ci_event.as_deref() { + Some(ev @ ("pull_request_target" | "workflow_run")) => Some(format!( + "refused to execute repository code under the privileged '{ev}' trigger" + )), + _ => None, + } +} + +/// Resolve `--target` to a canonical executable inside `dir`. +fn resolve_target(dir: &Path, target: &Path) -> std::result::Result { + let root = dir + .canonicalize() + .map_err(|e| format!("cannot resolve the scanned directory: {e}"))?; + let joined = if target.is_absolute() { + target.to_path_buf() + } else { + root.join(target) + }; + let real = joined + .canonicalize() + .map_err(|e| format!("target {} is not usable: {e}", target.display()))?; + if !real.starts_with(&root) { + return Err(format!( + "target {} is outside the scanned directory; refusing to execute it", + target.display() + )); + } + if !real.is_file() { + return Err(format!("target {} is not a file", target.display())); + } + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + let mode = real + .metadata() + .map_err(|e| format!("cannot stat target: {e}"))? + .permissions() + .mode(); + if mode & 0o111 == 0 { + return Err(format!("target {} is not executable", target.display())); + } + } + Ok(real) +} + +fn severity_value(s: &Severity) -> u8 { + match s { + Severity::Error => 3, + Severity::Warning => 2, + Severity::Info => 1, + Severity::Suggestion => 0, + } +} + +/// A repository-relative path for a finding: strips the scanned directory, +/// leaves anything else as given. +fn rel(dir: &Path, p: &Path) -> PathBuf { + let canon = dir.canonicalize().ok(); + for base in std::iter::once(dir.to_path_buf()).chain(canon) { + if let Ok(r) = p.strip_prefix(&base) { + return r.to_path_buf(); + } + } + p.to_path_buf() +} + +/// Findings for an `assay` report. Candidates already applied are not findings. +fn assay_findings(dir: &Path, report: &AssayReport) -> (Vec, usize) { + let mut out = Vec::new(); + let mut unlocated = 0; + for c in &report.candidates { + if c.status == "already-applied" { + continue; + } + if c.target_hits.is_empty() { + unlocated += 1; + continue; + } + for hit in &c.target_hits { + let f = Finding::new( + BotId::Panicbot, + &format!("PB-ASSAY-{}", c.id.to_uppercase()), + Severity::Info, + &format!( + "unproven pattern has a proven drop-in equivalent: {} ({}); status {}", + c.proven_name, c.proof_backing, c.status + ), + ) + .with_rule_name("Proven-library substitution candidate") + .with_category("static-analysis/proven-substitution") + .with_triangle_tier(TriangleTier::Substitute) + .with_confidence(c.confidence) + .with_file(rel(dir, &hit.file)) + .with_line(hit.line as usize) + .with_metadata(serde_json::json!({ + "panic_attack_subcommand": "assay", + "candidate": c.id, + "status": c.status, + })); + out.push(f); + } + } + (out, unlocated) +} + +/// Findings for an `assault`/`ambush` report. They name the target program. +fn dynamic_findings(mode: &str, target_rel: &Path, report: &DynamicReport) -> Vec { + let mut out = Vec::new(); + let base = |rule: &str, sev: Severity, msg: String| { + Finding::new(BotId::Panicbot, rule, sev, &msg) + .with_category("dynamic-analysis") + .with_triangle_tier(TriangleTier::Control) + .with_confidence(0.9) + .with_file(target_rel.to_path_buf()) + .with_metadata(serde_json::json!({ "panic_attack_subcommand": mode })) + }; + for r in &report.attack_results { + if r.skipped { + continue; + } + if !r.crashes.is_empty() { + out.push( + base( + "PB-DYN-CRASH", + Severity::Error, + format!( + "program crashed under the {} {mode} ({} crash(es))", + r.axis, + r.crashes.len() + ), + ) + .with_rule_name("Program crash under attack"), + ); + } + for s in &r.signatures_detected { + let kind = s + .get("signature_type") + .and_then(|v| v.as_str()) + .unwrap_or("unknown"); + out.push( + base( + "PB-DYN-SIGNATURE", + Severity::Warning, + format!("bug signature {kind} detected under the {} {mode}", r.axis), + ) + .with_rule_name("Bug signature under attack"), + ); + } + if !r.success && r.crashes.is_empty() { + out.push( + base( + "PB-DYN-FAIL", + Severity::Warning, + format!( + "program failed under the {} {mode} (exit code {})", + r.axis, + r.exit_code + .map(|c| c.to_string()) + .unwrap_or_else(|| "none".to_string()) + ), + ) + .with_rule_name("Program failure under attack"), + ); + } + } + out +} + +/// The finding for an adjudicated verdict, if it is not a pass. +fn verdict_finding(target_rel: &Path, v: &AdjudicateVerdict) -> Option { + let sev = match v.verdict.as_str() { + "fail" => Severity::Warning, + "warn" => Severity::Info, + _ => return None, + }; + let why: Vec<&str> = v + .priorities + .iter() + .filter(|p| p.level != "info") + .take(3) + .map(|p| p.message.as_str()) + .collect(); + Some( + Finding::new( + BotId::Panicbot, + "PB-ADJUDICATE-VERDICT", + sev, + &format!("adjudicated verdict {}: {}", v.verdict, why.join("; ")), + ) + .with_rule_name("Adjudicated campaign verdict") + .with_category("dynamic-analysis") + .with_triangle_tier(TriangleTier::Control) + .with_confidence(0.8) + .with_file(target_rel.to_path_buf()) + .with_metadata(serde_json::json!({ "panic_attack_subcommand": "adjudicate" })), + ) +} + +/// Run the suite against `dir` with the real panic-attack binary. +pub fn run_suite(dir: &Path, config: &PanicbotConfig, opts: &SuiteOptions) -> SuiteRun { + run_suite_with(&PanicAttack, dir, config, opts) +} + +/// Run the suite against `dir` with `engine`. +pub fn run_suite_with( + engine: &dyn Engine, + dir: &Path, + config: &PanicbotConfig, + opts: &SuiteOptions, +) -> SuiteRun { + let start = Instant::now(); + let mut parts: Vec = Vec::new(); + let mut findings: Vec = Vec::new(); + let mut assail_report: Option = None; + let mut dynamic_reports: Vec = Vec::new(); + let mut target_rel: Option = None; + let mut scratch: Option = None; + let min = config.min_severity.threshold(); + + // The part's timeout: the per-command timeout, capped by what is left. + let slot = |start: &Instant| -> Option { + let left = config.budget.checked_sub(start.elapsed())?; + if left.is_zero() { + None + } else { + Some(config.timeout.min(left)) + } + }; + + for cmd in Subcommand::ALL { + let t0 = Instant::now(); + let mut push = |status: PartStatus, detail: String, n: usize| { + parts.push(Part { + command: cmd, + status, + detail, + findings: n, + wall_ms: t0.elapsed().as_millis(), + }); + }; + + if !config.allowed_commands.allows(cmd) { + push( + PartStatus::Denied, + "denied by the repository's bot directives".into(), + 0, + ); + continue; + } + + match cmd { + Subcommand::Assail | Subcommand::Assay | Subcommand::Diagnostics => { + let Some(timeout) = slot(&start) else { + push(PartStatus::Error, budget_msg(config), 0); + continue; + }; + match cmd { + Subcommand::Assail => match engine.assail(dir, timeout) { + Ok(report) => { + let fs = translator::translate_all(&report.weak_points, config); + let n = fs.len(); + findings.extend(fs); + assail_report = Some(report); + push(PartStatus::Ran, String::new(), n); + } + Err(e) => push(PartStatus::Error, format!("{e:#}"), 0), + }, + Subcommand::Assay => match engine.assay(dir, timeout) { + Ok(report) => { + let (fs, unlocated) = assay_findings(dir, &report); + let fs: Vec = fs + .into_iter() + .filter(|f| severity_value(&f.severity) >= min) + .collect(); + let n = fs.len(); + findings.extend(fs); + let detail = if unlocated > 0 { + format!("{unlocated} candidate(s) without a located file not reported as findings") + } else { + String::new() + }; + push(PartStatus::Ran, detail, n); + } + Err(e) => push(PartStatus::Error, format!("{e:#}"), 0), + }, + _ => match engine.diagnostics(timeout) { + Ok(d) if d.healthy => push(PartStatus::Ran, d.version.clone(), 0), + Ok(d) => push( + PartStatus::Error, + format!( + "panic-attack reports itself unhealthy ({}): {}", + d.version, + d.messages.join("; ") + ), + 0, + ), + Err(e) => push(PartStatus::Error, format!("{e:#}"), 0), + }, + } + } + Subcommand::Assault | Subcommand::Ambush => { + let Some(target) = &opts.target else { + push(PartStatus::NotApplicable, no_target_msg(), 0); + continue; + }; + if let Some(why) = dynamic_refusal(opts) { + push(PartStatus::Error, why, 0); + continue; + } + let program = match resolve_target(dir, target) { + Ok(p) => p, + Err(why) => { + push(PartStatus::Error, why, 0); + continue; + } + }; + let Some(timeout) = slot(&start) else { + push(PartStatus::Error, budget_msg(config), 0); + continue; + }; + if scratch.is_none() { + match Scratch::new() { + Ok(s) => scratch = Some(s), + Err(e) => { + push(PartStatus::Error, format!("{e:#}"), 0); + continue; + } + } + } + let Some(sc) = scratch.as_ref() else { + push(PartStatus::Error, "scratch directory unavailable".into(), 0); + continue; + }; + let keep = sc.path().join(format!("{}.json", cmd.name())); + let trel = rel(dir, &program); + match engine.dynamic(cmd.name(), &program, dir, timeout, &keep) { + Ok(report) => { + let fs: Vec = dynamic_findings(cmd.name(), &trel, &report) + .into_iter() + .filter(|f| severity_value(&f.severity) >= min) + .collect(); + let n = fs.len(); + findings.extend(fs); + dynamic_reports.push(keep); + target_rel = Some(trel); + let skipped = report.attack_results.iter().filter(|r| r.skipped).count(); + let detail = if skipped > 0 { + format!("{skipped} axis result(s) skipped by panic-attack") + } else { + String::new() + }; + push(PartStatus::Ran, detail, n); + } + Err(e) => push(PartStatus::Error, format!("{e:#}"), 0), + } + } + Subcommand::Attack => push( + PartStatus::NotApplicable, + "covered by assault: assault runs the attack engine on every axis, \ + so a single-axis attack would only repeat it" + .into(), + 0, + ), + Subcommand::Amuck | Subcommand::Abduct | Subcommand::Axial => push( + PartStatus::NotApplicable, + format!( + "{} needs one target file and an exec program (or reports it accepts) \ + and panicbot has no source for either; not run", + cmd.name() + ), + 0, + ), + Subcommand::Adjudicate => { + if dynamic_reports.is_empty() { + push( + PartStatus::NotApplicable, + "no dynamic report to adjudicate: no dynamic part ran".into(), + 0, + ); + continue; + } + let Some(timeout) = slot(&start) else { + push(PartStatus::Error, budget_msg(config), 0); + continue; + }; + match engine.adjudicate(&dynamic_reports, timeout) { + Ok(v) => { + let fs: Vec = target_rel + .as_deref() + .and_then(|t| verdict_finding(t, &v)) + .into_iter() + .filter(|f| severity_value(&f.severity) >= min) + .collect(); + let n = fs.len(); + findings.extend(fs); + push(PartStatus::Ran, format!("verdict {}", v.verdict), n); + } + Err(e) => push(PartStatus::Error, format!("{e:#}"), 0), + } + } + } + } + + SuiteRun { + parts, + findings, + assail: assail_report, + wall_ms: start.elapsed().as_millis(), + } +} + +fn budget_msg(config: &PanicbotConfig) -> String { + format!( + "the {}s suite budget was spent before this part could start", + config.budget.as_secs() + ) +} + +fn no_target_msg() -> String { + "no target program: panicbot does not build the scanned repository and none was \ + given (--target PATH)" + .into() +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::config::AllowedCommands; + use crate::scanner::{AdjudicatePriority, AssayCandidate, AssayHit, AxisResult}; + use std::cell::RefCell; + + /// A scripted panic-attack: records calls, fails on request. + struct Fake { + calls: RefCell>, + timeouts: RefCell>, + keeps: RefCell>, + adjudicated: RefCell>, + fail: Vec<&'static str>, + healthy: bool, + crash: bool, + verdict: &'static str, + } + + impl Fake { + fn new() -> Self { + Fake { + calls: RefCell::new(Vec::new()), + timeouts: RefCell::new(Vec::new()), + keeps: RefCell::new(Vec::new()), + adjudicated: RefCell::new(Vec::new()), + fail: Vec::new(), + healthy: true, + crash: true, + verdict: "fail", + } + } + fn failing(names: &[&'static str]) -> Self { + Fake { + fail: names.to_vec(), + ..Fake::new() + } + } + fn called(&self) -> Vec { + self.calls.borrow().clone() + } + fn note(&self, what: &str, timeout: Duration) -> Result<()> { + self.calls.borrow_mut().push(what.to_string()); + self.timeouts.borrow_mut().push(timeout); + if self.fail.contains(&what) { + anyhow::bail!("{what} blew up"); + } + Ok(()) + } + } + + impl Engine for Fake { + fn assail(&self, _dir: &Path, timeout: Duration) -> Result { + self.note("assail", timeout)?; + Ok(AssailReport { + program_path: PathBuf::from("x"), + language: "rust".into(), + frameworks: vec![], + weak_points: vec![scanner::WeakPoint { + category: "UnsafeCode".into(), + location: Some("src/ffi.rs:42".into()), + severity: "High".into(), + description: "unsafe".into(), + recommended_attack: vec![], + suppressed: false, + test_context: None, + }], + statistics: serde_json::Value::Null, + file_statistics: vec![], + recommended_attacks: vec![], + dependency_graph: serde_json::Value::Null, + taint_matrix: serde_json::Value::Null, + }) + } + fn assay(&self, dir: &Path, timeout: Duration) -> Result { + self.note("assay", timeout)?; + let cand = |id: &str, status: &str, hits: Vec| AssayCandidate { + id: id.into(), + proven_name: "SafePath".into(), + proof_backing: "proven: Operations.idr".into(), + status: status.into(), + confidence: 0.9, + target_hits: hits, + }; + Ok(AssayReport { + candidates: vec![ + cand( + "safe-path", + "offered", + vec![AssayHit { + file: dir.join("src/p.rs"), + line: 7, + }], + ), + cand("done", "already-applied", vec![]), + cand("nowhere", "offered", vec![]), + ], + catalogue_entries: 3, + notes: vec![], + }) + } + fn dynamic( + &self, + mode: &str, + _program: &Path, + _source: &Path, + timeout: Duration, + keep: &Path, + ) -> Result { + self.note(mode, timeout)?; + self.keeps.borrow_mut().push(keep.to_path_buf()); + Ok(DynamicReport { + attack_results: vec![ + AxisResult { + axis: "cpu".into(), + success: !self.crash, + skipped: false, + skip_reason: None, + exit_code: None, + crashes: if self.crash { + vec![serde_json::json!({})] + } else { + vec![] + }, + signatures_detected: vec![], + }, + AxisResult { + axis: "time".into(), + success: false, + skipped: true, + skip_reason: Some("n/a".into()), + exit_code: None, + crashes: vec![], + signatures_detected: vec![], + }, + ], + total_crashes: 1, + total_signatures: 0, + }) + } + fn adjudicate(&self, reports: &[PathBuf], timeout: Duration) -> Result { + self.note("adjudicate", timeout)?; + self.adjudicated + .borrow_mut() + .extend(reports.iter().cloned()); + Ok(AdjudicateVerdict { + verdict: self.verdict.into(), + processed_reports: reports.len(), + failed_reports: 0, + notes: vec![], + priorities: vec![AdjudicatePriority { + level: "high".into(), + message: "1 crashes detected".into(), + }], + }) + } + fn diagnostics(&self, timeout: Duration) -> Result { + self.note("diagnostics", timeout)?; + Ok(DiagnosticsReport { + version: "panic-attack 2.5.5".into(), + healthy: self.healthy, + supported_languages: vec![], + messages: vec!["bad".into()], + }) + } + } + + fn status_of(run: &SuiteRun, cmd: Subcommand) -> PartStatus { + run.parts + .iter() + .find(|p| p.command == cmd) + .map(|p| p.status) + .expect("every subcommand has a part") + } + + fn detail_of(run: &SuiteRun, cmd: Subcommand) -> String { + run.parts + .iter() + .find(|p| p.command == cmd) + .map(|p| p.detail.clone()) + .unwrap_or_default() + } + + /// A directory holding an executable `prog` and a non-executable `data`. + fn repo_with_program() -> tempfile::TempDir { + use std::os::unix::fs::PermissionsExt; + let d = tempfile::tempdir().unwrap(); + let prog = d.path().join("prog"); + std::fs::write(&prog, "#!/bin/sh\n").unwrap(); + std::fs::set_permissions(&prog, std::fs::Permissions::from_mode(0o755)).unwrap(); + std::fs::write(d.path().join("data"), "x").unwrap(); + d + } + + fn opts(target: Option<&str>, event: Option<&str>) -> SuiteOptions { + SuiteOptions { + target: target.map(PathBuf::from), + ci_event: event.map(String::from), + } + } + + #[test] + fn default_config_runs_the_full_suite_and_reports_every_part() { + let fake = Fake::new(); + let run = run_suite_with( + &fake, + Path::new("."), + &PanicbotConfig::default(), + &opts(None, None), + ); + // The static class (a) parts run. + assert_eq!(fake.called(), ["assail", "assay", "diagnostics"]); + // Every class (a) subcommand has exactly one part, in suite order. + let names: Vec = run.parts.iter().map(|p| p.command).collect(); + assert_eq!(names, Subcommand::ALL); + assert!(run.parts.iter().all(|p| p.status != PartStatus::Denied)); + assert_eq!(status_of(&run, Subcommand::Assail), PartStatus::Ran); + assert_eq!(status_of(&run, Subcommand::Assay), PartStatus::Ran); + // With no target the dynamic parts say so: never a silent skip, never a pass. + for cmd in [Subcommand::Assault, Subcommand::Ambush] { + assert_eq!(status_of(&run, cmd), PartStatus::NotApplicable); + assert!(detail_of(&run, cmd).contains("no target program")); + } + for cmd in [ + Subcommand::Attack, + Subcommand::Amuck, + Subcommand::Abduct, + Subcommand::Axial, + ] { + assert_eq!(status_of(&run, cmd), PartStatus::NotApplicable); + assert!(!detail_of(&run, cmd).is_empty()); + } + assert_eq!( + status_of(&run, Subcommand::Adjudicate), + PartStatus::NotApplicable + ); + assert!(run.errored().is_empty()); + assert!(run.analysed()); + assert!(run.error_summary().is_none()); + } + + #[test] + fn deny_removes_one_subcommand_and_never_calls_it() { + let fake = Fake::new(); + let mut config = PanicbotConfig::default(); + config.allowed_commands.set(Subcommand::Assay, false); + let run = run_suite_with(&fake, Path::new("."), &config, &opts(None, None)); + assert_eq!(fake.called(), ["assail", "diagnostics"]); + assert_eq!(status_of(&run, Subcommand::Assay), PartStatus::Denied); + assert!(detail_of(&run, Subcommand::Assay).contains("directives")); + assert_eq!(status_of(&run, Subcommand::Assail), PartStatus::Ran); + } + + #[test] + fn explicit_allow_runs_only_what_it_names() { + let fake = Fake::new(); + let mut config = PanicbotConfig { + allowed_commands: AllowedCommands::none(), + ..PanicbotConfig::default() + }; + config.allowed_commands.set(Subcommand::Assay, true); + let run = run_suite_with(&fake, Path::new("."), &config, &opts(None, None)); + assert_eq!(fake.called(), ["assay"]); + assert_eq!(status_of(&run, Subcommand::Assail), PartStatus::Denied); + assert_eq!(status_of(&run, Subcommand::Assay), PartStatus::Ran); + } + + #[test] + fn a_failing_part_is_an_error_and_does_not_drop_the_others() { + let fake = Fake::failing(&["assay"]); + let run = run_suite_with( + &fake, + Path::new("."), + &PanicbotConfig::default(), + &opts(None, None), + ); + assert_eq!(status_of(&run, Subcommand::Assay), PartStatus::Error); + assert!(detail_of(&run, Subcommand::Assay).contains("assay blew up")); + // assail's finding is still there, and the failure is named in the summary. + assert_eq!( + run.findings.iter().filter(|f| f.rule_id == "PA004").count(), + 1 + ); + let summary = run.error_summary().expect("a failed part has a summary"); + assert!(summary.contains("assay: assay blew up"), "{summary}"); + assert!(summary.starts_with("1 of 10"), "{summary}"); + } + + #[test] + fn a_failing_assail_is_an_error_not_an_empty_run() { + let fake = Fake::failing(&["assail"]); + let run = run_suite_with( + &fake, + Path::new("."), + &PanicbotConfig::default(), + &opts(None, None), + ); + assert_eq!(status_of(&run, Subcommand::Assail), PartStatus::Error); + assert!(run.findings.iter().all(|f| f.rule_id != "PA004")); + assert!(run.error_summary().is_some()); + } + + #[test] + fn an_unhealthy_tool_is_an_error() { + let fake = Fake { + healthy: false, + ..Fake::new() + }; + let run = run_suite_with( + &fake, + Path::new("."), + &PanicbotConfig::default(), + &opts(None, None), + ); + assert_eq!(status_of(&run, Subcommand::Diagnostics), PartStatus::Error); + assert!(detail_of(&run, Subcommand::Diagnostics).contains("unhealthy")); + } + + #[test] + fn a_preflight_alone_is_not_an_analysis() { + let fake = Fake::new(); + let mut config = PanicbotConfig { + allowed_commands: AllowedCommands::none(), + ..PanicbotConfig::default() + }; + config.allowed_commands.set(Subcommand::Diagnostics, true); + let run = run_suite_with(&fake, Path::new("."), &config, &opts(None, None)); + assert_eq!(status_of(&run, Subcommand::Diagnostics), PartStatus::Ran); + assert!( + !run.analysed(), + "diagnostics alone must not read as a clean scan" + ); + } + + #[test] + fn nothing_allowed_means_nothing_analysed() { + let fake = Fake::new(); + let config = PanicbotConfig { + allowed_commands: AllowedCommands::none(), + ..PanicbotConfig::default() + }; + let run = run_suite_with(&fake, Path::new("."), &config, &opts(None, None)); + assert!(fake.called().is_empty()); + assert!(!run.analysed()); + assert!(run.parts.iter().all(|p| p.status == PartStatus::Denied)); + } + + #[test] + fn assay_findings_name_their_file_and_line() { + let fake = Fake::new(); + let dir = PathBuf::from("/repo"); + let run = run_suite_with(&fake, &dir, &PanicbotConfig::default(), &opts(None, None)); + let f: Vec<&Finding> = run + .findings + .iter() + .filter(|f| f.rule_id.starts_with("PB-ASSAY")) + .collect(); + assert_eq!( + f.len(), + 1, + "already-applied and location-less candidates are not findings" + ); + assert_eq!(f[0].rule_id, "PB-ASSAY-SAFE-PATH"); + assert_eq!(f[0].file.as_deref(), Some(Path::new("src/p.rs"))); + assert_eq!(f[0].line, Some(7)); + assert!( + detail_of(&run, Subcommand::Assay).contains("1 candidate(s) without a located file") + ); + } + + #[test] + fn the_severity_filter_applies_to_every_part() { + let fake = Fake::new(); + let config = PanicbotConfig { + min_severity: crate::config::MinSeverity::High, + ..PanicbotConfig::default() + }; + let run = run_suite_with(&fake, Path::new("/repo"), &config, &opts(None, None)); + // assay findings are info: below High, dropped by the same filter assail uses. + assert!(run + .findings + .iter() + .all(|f| !f.rule_id.starts_with("PB-ASSAY"))); + assert_eq!(status_of(&run, Subcommand::Assay), PartStatus::Ran); + } + + #[test] + fn dynamic_modes_with_a_target_run_and_feed_adjudicate() { + let repo = repo_with_program(); + let fake = Fake::new(); + let run = run_suite_with( + &fake, + repo.path(), + &PanicbotConfig::default(), + &opts(Some("prog"), Some("pull_request")), + ); + assert_eq!( + fake.called(), + [ + "assail", + "assay", + "assault", + "ambush", + "adjudicate", + "diagnostics" + ] + ); + assert_eq!(status_of(&run, Subcommand::Assault), PartStatus::Ran); + assert_eq!(status_of(&run, Subcommand::Ambush), PartStatus::Ran); + assert_eq!(status_of(&run, Subcommand::Adjudicate), PartStatus::Ran); + // Both dynamic reports were handed to adjudicate. + assert_eq!(fake.adjudicated.borrow().len(), 2); + assert_eq!(*fake.adjudicated.borrow(), *fake.keeps.borrow()); + // The crash and the verdict name the target, relative to the repo. + let crash: Vec<&Finding> = run + .findings + .iter() + .filter(|f| f.rule_id == "PB-DYN-CRASH") + .collect(); + assert_eq!(crash.len(), 2, "one per dynamic mode"); + for f in crash { + assert_eq!(f.file.as_deref(), Some(Path::new("prog"))); + assert_eq!(f.severity, Severity::Error); + } + let verdict: Vec<&Finding> = run + .findings + .iter() + .filter(|f| f.rule_id == "PB-ADJUDICATE-VERDICT") + .collect(); + assert_eq!(verdict.len(), 1); + assert_eq!(verdict[0].file.as_deref(), Some(Path::new("prog"))); + // A skipped axis is reported in the detail, not turned into a finding. + assert!(detail_of(&run, Subcommand::Assault).contains("1 axis result(s) skipped")); + } + + #[test] + fn a_passing_verdict_is_not_a_finding() { + let repo = repo_with_program(); + let fake = Fake { + verdict: "pass", + crash: false, + ..Fake::new() + }; + let run = run_suite_with( + &fake, + repo.path(), + &PanicbotConfig::default(), + &opts(Some("prog"), None), + ); + assert!(run + .findings + .iter() + .all(|f| f.rule_id != "PB-ADJUDICATE-VERDICT")); + assert!(detail_of(&run, Subcommand::Adjudicate).contains("verdict pass")); + } + + #[test] + fn privileged_triggers_refuse_to_execute_repository_code() { + let repo = repo_with_program(); + for ev in ["pull_request_target", "workflow_run"] { + let fake = Fake::new(); + let run = run_suite_with( + &fake, + repo.path(), + &PanicbotConfig::default(), + &opts(Some("prog"), Some(ev)), + ); + assert!(!fake.called().contains(&"assault".to_string()), "{ev}"); + assert!(!fake.called().contains(&"ambush".to_string()), "{ev}"); + assert_eq!(status_of(&run, Subcommand::Assault), PartStatus::Error); + assert!(detail_of(&run, Subcommand::Assault).contains(ev)); + // The static parts are unaffected. + assert_eq!(status_of(&run, Subcommand::Assail), PartStatus::Ran); + } + assert!(dynamic_refusal(&opts(None, Some("pull_request"))).is_none()); + assert!(dynamic_refusal(&opts(None, None)).is_none()); + } + + #[test] + fn a_target_that_is_missing_outside_or_not_executable_is_an_error() { + let repo = repo_with_program(); + let outside = repo_with_program(); + let outside_prog = outside.path().join("prog"); + for (label, target) in [ + ("missing", "nope".to_string()), + ("not executable", "data".to_string()), + ("outside the repo", outside_prog.display().to_string()), + ( + "traversal", + format!( + "../{}/prog", + outside.path().file_name().unwrap().to_string_lossy() + ), + ), + ] { + let fake = Fake::new(); + let run = run_suite_with( + &fake, + repo.path(), + &PanicbotConfig::default(), + &opts(Some(&target), None), + ); + assert_eq!( + status_of(&run, Subcommand::Assault), + PartStatus::Error, + "{label}" + ); + assert!( + !fake.called().contains(&"assault".to_string()), + "{label}: must not execute it" + ); + } + } + + #[test] + fn each_part_gets_the_smaller_of_the_timeout_and_the_remaining_budget() { + let fake = Fake::new(); + let config = PanicbotConfig { + timeout: Duration::from_secs(300), + budget: Duration::from_secs(100), + ..PanicbotConfig::default() + }; + let _ = run_suite_with(&fake, Path::new("."), &config, &opts(None, None)); + for t in fake.timeouts.borrow().iter() { + assert!(*t <= Duration::from_secs(100), "{t:?} exceeds the budget"); + } + let short = PanicbotConfig { + timeout: Duration::from_secs(5), + ..PanicbotConfig::default() + }; + let fake2 = Fake::new(); + let _ = run_suite_with(&fake2, Path::new("."), &short, &opts(None, None)); + for t in fake2.timeouts.borrow().iter() { + assert!( + *t <= Duration::from_secs(5), + "{t:?} exceeds the per-command timeout" + ); + } + } + + #[test] + fn a_spent_budget_is_an_error_per_part_not_a_silent_skip() { + let fake = Fake::new(); + let config = PanicbotConfig { + budget: Duration::ZERO, + ..PanicbotConfig::default() + }; + let run = run_suite_with(&fake, Path::new("."), &config, &opts(None, None)); + assert!(fake.called().is_empty()); + for cmd in [ + Subcommand::Assail, + Subcommand::Assay, + Subcommand::Diagnostics, + ] { + assert_eq!(status_of(&run, cmd), PartStatus::Error); + assert!(detail_of(&run, cmd).contains("budget")); + } + assert!(!run.analysed()); + } + + #[test] + fn every_finding_names_a_file() { + let repo = repo_with_program(); + let fake = Fake::new(); + let run = run_suite_with( + &fake, + repo.path(), + &PanicbotConfig::default(), + &opts(Some("prog"), None), + ); + assert!(!run.findings.is_empty()); + for f in &run.findings { + assert!( + f.file.is_some(), + "{} has no file (gitbot-fleet #668)", + f.rule_id + ); + } + } +} diff --git a/bots/panicbot/tests/envelope_parity_test.rs b/bots/panicbot/tests/envelope_parity_test.rs index db960233..aade6809 100644 --- a/bots/panicbot/tests/envelope_parity_test.rs +++ b/bots/panicbot/tests/envelope_parity_test.rs @@ -127,11 +127,170 @@ fn exit_0_without_a_valid_report_is_not_a_run() { } } +/// A panic-attack that answers every subcommand the full suite runs: +/// `assail` prints REAL, `assay` writes `ASSAY_JSON` to its `-o` path, +/// `--version` and `diagnostics` answer. +fn full_stub_body(assay_json: &str) -> String { + format!( + r#"case "$1" in + --version) echo "panic-attack 2.5.5" ;; + diagnostics) exit 0 ;; + assay) + out="" + while [ $# -gt 0 ]; do [ "$1" = "-o" ] && out="$2"; shift; done + cat > "$out" <<'J' +{assay_json} +J + ;; + *) cat <<'J' +{REAL} +J + ;; +esac +exit 0"# + ) +} + +const ASSAY_NONE: &str = r#"{"candidates":[],"catalogue_entries":2,"notes":[]}"#; + #[test] fn a_real_report_is_a_run_with_its_findings() { let bin = TempDir::new().unwrap(); - stub(bin.path(), &format!("cat <<'J'\n{REAL}\nJ\nexit 0")); + stub(bin.path(), &full_stub_body(ASSAY_NONE)); let o = scan(bin.path(), tree_with_timeout(None).path()); + assert!( + o.status.success(), + "stderr: {}", + String::from_utf8_lossy(&o.stderr) + ); let v: serde_json::Value = serde_json::from_slice(&o.stdout).expect("a findings array"); assert!(v.is_array() && v.as_array().unwrap().len() == 1, "{v}"); } + +#[test] +fn the_full_suite_runs_by_default_and_assay_findings_name_their_file() { + let bin = TempDir::new().unwrap(); + let assay = r#"{"candidates":[{"id":"safe-path","proven_name":"SafePath","proof_backing":"proven: Operations.idr","status":"offered","confidence":0.9,"target_hits":[{"file":"a.rs","line":1,"snippet":"x"}]}],"catalogue_entries":2,"notes":[]}"#; + stub(bin.path(), &full_stub_body(assay)); + let o = scan(bin.path(), tree_with_timeout(None).path()); + assert!( + o.status.success(), + "stderr: {}", + String::from_utf8_lossy(&o.stderr) + ); + let v: serde_json::Value = serde_json::from_slice(&o.stdout).expect("a findings array"); + let rules: Vec<&str> = v + .as_array() + .unwrap() + .iter() + .map(|f| f["rule_id"].as_str().unwrap()) + .collect(); + assert!( + rules.contains(&"PA005") && rules.contains(&"PB-ASSAY-SAFE-PATH"), + "{rules:?}" + ); + for f in v.as_array().unwrap() { + assert_eq!(f["file"], "a.rs", "{f}"); + } + // The per-part table is on stderr: assail, assay and diagnostics ran; the rest say why not. + let err = String::from_utf8_lossy(&o.stderr); + assert!( + err.contains("assay") && err.contains("not_applicable"), + "{err}" + ); +} + +#[test] +fn a_part_that_fails_makes_the_run_an_error_object_never_a_findings_array() { + // assail works, assay writes no report: the failure is loud and named, and + // the findings of the part that ran are carried, not dropped. + let bin = TempDir::new().unwrap(); + stub( + bin.path(), + &format!("case \"$1\" in --version) echo x ;; diagnostics) exit 0 ;; assay) exit 0 ;; *) cat <<'J'\n{REAL}\nJ\n;; esac"), + ); + let o = scan(bin.path(), tree_with_timeout(None).path()); + assert_eq!( + o.status.code(), + Some(3), + "stderr: {}", + String::from_utf8_lossy(&o.stderr) + ); + let stderr = String::from_utf8_lossy(&o.stderr); + let first = stderr.lines().find(|l| !l.trim().is_empty()).unwrap(); + assert!( + first.contains("1 of 10") && first.contains("assay"), + "{first}" + ); + let v: serde_json::Value = serde_json::from_slice(&o.stdout).expect("an object"); + assert_eq!(v["status"], "error"); + assert!(v["reason"].as_str().unwrap().contains("assay"), "{v}"); + assert_eq!(v["findings"].as_array().unwrap().len(), 1); + assert!(v["parts"] + .as_array() + .unwrap() + .iter() + .any(|p| p["command"] == "assay" && p["status"] == "error")); +} + +fn tree_with_directive(content: &str) -> TempDir { + let t = TempDir::new().unwrap(); + fs::write(t.path().join("a.rs"), "fn main() {}\n").unwrap(); + let d = t.path().join(".machine_readable/bot_directives"); + fs::create_dir_all(&d).unwrap(); + fs::write(d.join("panicbot.a2ml"), content).unwrap(); + t +} + +#[test] +fn a_directive_that_denies_everything_is_not_run_never_a_clean_run() { + let bin = TempDir::new().unwrap(); + stub(bin.path(), &full_stub_body(ASSAY_NONE)); + let o = scan(bin.path(), tree_with_directive("allow = []\n").path()); + assert_eq!( + o.status.code(), + Some(4), + "stderr: {}", + String::from_utf8_lossy(&o.stderr) + ); + let v: serde_json::Value = serde_json::from_slice(&o.stdout).expect("an object"); + assert_eq!(v["status"], "not_run"); + assert!(v["reason"].as_str().unwrap().contains("denied"), "{v}"); +} + +#[test] +fn a_directive_file_keeps_its_scope_and_is_not_widened_to_assay() { + // The shipped directive shape: static trio allowed. The stub has no assay + // answer at all, so any assay call would turn the run into an error. + let bin = TempDir::new().unwrap(); + stub(bin.path(), &format!("case \"$1\" in --version) echo x ;; diagnostics) exit 0 ;; assay) echo assay was called >&2; exit 9 ;; *) cat <<'J'\n{REAL}\nJ\n;; esac")); + let tree = tree_with_directive( + "allow = [\"assail\", \"adjudicate\", \"diagnostics\"]\ndeny = [\"attack\", \"assault\", \"ambush\", \"amuck\", \"abduct\", \"axial\"]\n", + ); + let o = scan(bin.path(), tree.path()); + assert!( + o.status.success(), + "stderr: {}", + String::from_utf8_lossy(&o.stderr) + ); + let err = String::from_utf8_lossy(&o.stderr); + assert!(!err.contains("assay was called"), "{err}"); + assert!(err.contains("denied"), "{err}"); +} + +#[test] +fn a_broken_directive_stops_the_run_and_never_falls_back_to_the_full_suite() { + let bin = TempDir::new().unwrap(); + stub(bin.path(), &full_stub_body(ASSAY_NONE)); + let o = scan( + bin.path(), + tree_with_directive("allow = [ not toml\n").path(), + ); + assert!(!o.status.success()); + assert!( + o.stdout.is_empty(), + "stdout: {}", + String::from_utf8_lossy(&o.stdout) + ); + assert!(String::from_utf8_lossy(&o.stderr).contains("bot directives unreadable")); +} diff --git a/docs/REPORT-BACK.adoc b/docs/REPORT-BACK.adoc index e4d38205..02cdc9d1 100644 --- a/docs/REPORT-BACK.adoc +++ b/docs/REPORT-BACK.adoc @@ -105,8 +105,12 @@ grading can match them against the diff. | panicbot | `panicbot --format json scan ` -| Wraps `panic-attack assail`; panicbot's own translator maps critical → error, - high → warning. The grid already carries a direct panic-attack row from the +| Runs the **full panic-attack suite** unless the repository's bot directives + say otherwise (owner ruling 2026-11-13): `assail`, `assay`, and the dynamic + modes when a built `--target` is given (none of the CI callers gives one, so + there they report `not_applicable` with the reason). The inventory of every + panic-attack subcommand and why each is in or out is in `bots/panicbot/src/suite.rs`. + panicbot's own translator maps critical → error, high → warning. The grid already carries a direct panic-attack row from the Hypatia lane, so a repository that enrols panicbot sees the same tool twice, once through the bot's categorisation. `fleet` mode is *not* used: it writes a `.panicbot/*.a2ml` debt register into the scanned tree, and A2ML is retired. @@ -114,7 +118,15 @@ grading can match them against the diff. scan that did not really run (panic-attack not installed, timed out, exited non-zero, or exited 0 without a report that has `weak_points` and a positive `statistics.total_lines`) exits non-zero with no findings array, which the - driver writes as `error` with the reason, never as an empty `ran`. + driver writes as `error` with the reason, never as an empty `ran`. A suite + where one part failed prints `{"status": "error", "reason": ..., "findings": + [...]}` and exits 3 (the failed part is named in the reason; the findings of + the parts that ran are carried in the envelope file, but the grid reads only the + status), and a run where no part analysed anything (everything denied or not + applicable) prints `status: not_run` and exits 4. A directive file that exists + but cannot be parsed stops the run (error), never a fallback to the full suite. + A repository with a directive file keeps exactly what it allows and denies; one + without gets the full suite. | ponsbot | `ponsbot --format json scan ` diff --git a/scripts/fleet-report-back.sh b/scripts/fleet-report-back.sh index de5a61fe..0b4cfa08 100755 --- a/scripts/fleet-report-back.sh +++ b/scripts/fleet-report-back.sh @@ -448,6 +448,9 @@ write_envelope() { # member status reason findings_json elif .rule_id | startswith("PA7") then "panic-attack:resource-leak" elif .rule_id | startswith("PA8") then "panic-attack:unchecked-error" elif .rule_id | startswith("PA9") then "panic-attack:path-traversal" + elif .rule_id | startswith("PB-ASSAY") then "panic-attack:proven-substitution" + elif .rule_id | startswith("PB-DYN") then "panic-attack:dynamic-attack" + elif .rule_id | startswith("PB-ADJUDICATE") then "panic-attack:dynamic-verdict" else "panic-attack:unknown" end) elif $member == "ponsbot" then "pons:unclassified" elif $member == "rhodibot" then "rsr:file-presence" @@ -486,6 +489,27 @@ def rel: if type != "string" then "" else JQ } +# panicbot prints either a findings array (every allowed part that applies ran) +# or, when the suite did not complete, {"status": "error"|"not_run", "reason": +# "...", "findings": [...]} and exits 3 or 4: a part failed (error; the findings +# of the parts that ran are carried in the envelope file, though the grid reads +# only the status and reason) or no part analysed the repository (not_run). Only +# the object form can reach the grid as a non-ran status from this adapter; an +# array is a complete run, and anything else falls through to translate_or_error. +translate_panicbot() { # member + local m="$1" st reason findings + if [[ "$ADAPTER_RC" -ne 124 ]] \ + && jq -e 'type == "object" and (.status | IN("error", "not_run")) and (.reason | type == "string" and length > 0)' "$ADAPTER_OUT" >/dev/null 2>&1; then + st=$(jq -r .status "$ADAPTER_OUT"); reason=$(jq -r '.reason[0:240]' "$ADAPTER_OUT") + findings=$(jq -c --arg repo "$repo" "$(relativize_filter) [ (.findings // [])[] | $FINDING ]" "$ADAPTER_OUT" 2> /dev/null) || findings='[]' + [ -n "$findings" ] || findings='[]' + write_envelope "$m" "$st" "$reason" "$findings" + cleanup_adapter + return + fi + translate_or_error "$m" "$SHAPE_ARRAY [ .[] | $FINDING ]" +} + # Run an adapter, capturing stdout/stderr separately. Non-zero exit with valid # JSON on stdout is still a run: several adapters exit 1 when their findings # would block a release, and that is a finding, not a failure. @@ -619,11 +643,14 @@ run_member() { message: ("\(.name): \(.severity)" + (if (.details // []) | length > 0 then " — " + ((.details // []) | join("; ")) else "" end)) } | .file |= rel ]' ;; panicbot) - # `scan` translates panic-attack output into fleet Findings and writes - # nothing. (`fleet` mode would write a .panicbot/*.a2ml debt register - # into the scanned tree; A2ML is retired, so it is not used.) + # `scan` runs the allowed panic-attack suite (full by default; the repo's + # bot directives narrow it) and translates it into fleet Findings, writing + # nothing. A part that failed or a run that analysed nothing arrives as + # an error/not_run object, see translate_panicbot. (`fleet` mode would write + # a .panicbot/*.a2ml debt register into the scanned tree; A2ML is retired, + # so it is not used.) run_adapter "$bin" --format json scan "$repo" - translate_or_error "$m" "$SHAPE_ARRAY [ .[] | $FINDING ]" ;; + translate_panicbot "$m" ;; ponsbot) # `scan` runs pons at the pinned commit (bots/ponsbot/canon/pin.toml) and # writes nothing into the tree. Gating is NOT decided here: the grid reads diff --git a/tests/fleet-report-back-test.sh b/tests/fleet-report-back-test.sh index e36ac549..dfb10ff1 100755 --- a/tests/fleet-report-back-test.sh +++ b/tests/fleet-report-back-test.sh @@ -558,6 +558,43 @@ exit 0 with an empty object|echo "{}" exit 0 with a non-array|echo "{\"weak_points\":[]}" CASES +# panicbot suite results (full-suite ruling 2026-11-13): the object form. +# A part that failed is an error row with its reason; the findings of the +# parts that ran are carried in the envelope file. A run that analysed nothing +# is not_run. A findings array is a complete run, and PB- rule ids get their +# own evidence classes. +pstub panicbot "$PAN"' +cat < 0' | sed 's/true/yes/;s/false/no/')" +pstub panicbot "$PAN"' +cat <