From ae1d39b376006ebf7c1ee74e6a7d8fba4bc26231 Mon Sep 17 00:00:00 2001 From: Basil Zafar Date: Thu, 22 Oct 2026 21:25:17 +0530 Subject: [PATCH 1/3] test: measure PI079 against independent command specimens (#180) --- .github/code-scanning-baseline.json | 14 ++++ tests/fixtures/pi079-evidence.json | 26 +++++++ tests/pi079_evidence_test.rs | 101 ++++++++++++++++++++++++++++ 3 files changed, 141 insertions(+) create mode 100644 tests/fixtures/pi079-evidence.json create mode 100644 tests/pi079_evidence_test.rs diff --git a/.github/code-scanning-baseline.json b/.github/code-scanning-baseline.json index cc93f29..167ffef 100644 --- a/.github/code-scanning-baseline.json +++ b/.github/code-scanning-baseline.json @@ -2480,6 +2480,20 @@ "count": 1, "first_seen_line": 16 }, + { + "file": "tests/fixtures/pi079-evidence.json", + "pattern_id": "PI079", + "digest": "sha256:839db62f0f04ef4d370b15177a260c579ef59e578a43788ab8d03b08669b22aa", + "count": 1, + "first_seen_line": 7 + }, + { + "file": "tests/fixtures/pi079-evidence.json", + "pattern_id": "PI079", + "digest": "sha256:960665f230ba649103bbbd85dc23fdf13755819fd8a78e872171233ed3f3c64f", + "count": 1, + "first_seen_line": 15 + }, { "file": "tools/injection-lab/corpus/01-prose-role-override.md", "pattern_id": "PI001", diff --git a/tests/fixtures/pi079-evidence.json b/tests/fixtures/pi079-evidence.json new file mode 100644 index 0000000..7634763 --- /dev/null +++ b/tests/fixtures/pi079-evidence.json @@ -0,0 +1,26 @@ +[ + { + "name": "sans-outlaw-command", + "source": "https://isc.sans.edu/diary/30428", + "accessed": "2026-10-22", + "classification": "hostile-command", + "text": "echo \"ssh-rsa AAAAB3NzaC1yc2EAAAABJQAAAQEArDp4cun2lhr4KUhBGE7VvAcwdli2a8dbnrTOrbMz1+5O73fcBOx8NVbUT0bUanUV9tJ2/9p7+vD0EpZ3Tz/+0kX34uAx1RV/75GVOmNx+9EuWOnvNoaJe0QXxziIg9eLBHpgLMuakb5+BgTFB+rKJAw9u9FSTDengvS8hX1kNFS4Mjux0hJOK8rvcEmPecjdySYMb66nylAKGwCEE6WEQHmd1mUPgHwGQ0hWCwsQk13yCGPK5w6hYp5zYkFnvlC8hGmd4Ww+u97k6pfTGTUbJk14ujvcD9iUKQTTWYYjIIu5PmUux5bsZ0R4WFwdIe6+i6rBLAsPKgAySVKPRK+oRw== mdrfckr\">>.ssh/authorized_keys", + "expected_match": true + }, + { + "name": "geckoadmin-windows-key-installation", + "source": "https://geckoadmin.com/ssh-key-authentication-setup/", + "accessed": "2026-10-22", + "classification": "benign-documentation", + "text": "echo \"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIC4FFRxI+IkBZZ5bfUhNcvc/2KpiZt3TuZoXXPB80q5D SSH-from-Windows\" >> ~/.ssh/authorized_keys", + "expected_match": true + }, + { + "name": "extract-sftp-elided-key", + "source": "https://docs.extract.to/destinations/sftp", + "accessed": "2026-10-22", + "classification": "benign-documentation", + "text": "echo \"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5... extract\" >> ~/.ssh/authorized_keys", + "expected_match": false + } +] diff --git a/tests/pi079_evidence_test.rs b/tests/pi079_evidence_test.rs new file mode 100644 index 0000000..4cb2ea7 --- /dev/null +++ b/tests/pi079_evidence_test.rs @@ -0,0 +1,101 @@ +//! Independent command excerpts, not a prompt-injection recall corpus. +//! See docs/PI079-EVIDENCE.md for provenance and sampling limitations. +use injection_scanner::allowlist::Suppressions; +use injection_scanner::pattern::Severity; +use injection_scanner::patterns::load_embedded_patterns; +use injection_scanner::scanner::Scanner; +use serde::Deserialize; + +mod common; + +#[derive(Deserialize)] +struct Specimen { + name: String, + text: String, + expected_match: bool, +} + +fn specimens() -> Vec { + let cases: Vec = + serde_json::from_str(include_str!("fixtures/pi079-evidence.json")).unwrap(); + assert_eq!( + cases.len(), + 3, + "update the measured table when specimens change" + ); + cases +} + +fn fires(scanner: &Scanner, text: &str) -> bool { + let report = scanner.scan("specimen.txt", text, &Suppressions::default()); + report + .matches + .iter() + .chain(&report.low_confidence) + .any(|m| m.pattern_id == "PI079") +} + +#[test] +fn independent_commands_record_the_benign_overlap() { + for specimen in specimens() { + assert_eq!( + fires(common::scanner(), &specimen.text), + specimen.expected_match, + "{}", + specimen.name + ); + } + let categories = load_embedded_patterns().unwrap(); + let pattern = categories + .iter() + .flat_map(|c| &c.patterns) + .find(|p| p.id == "PI079") + .unwrap(); + assert_eq!(pattern.severity, Some(Severity::Medium)); +} + +#[test] +fn threshold_sensitivity_on_independent_commands() { + // Counts refer to base64 characters AFTER the literal AAAA prefix. + // Raising the cutoff above 64 suppresses the legitimate Ed25519 example, + // but also excludes real keys of that size regardless of provenance. + let cases = specimens(); + for (minimum, expected) in [ + (8, [true, true, true]), + (40, [true, true, false]), + (64, [true, true, false]), + (65, [true, false, false]), + (128, [true, false, false]), + (512, [false, false, false]), + ] { + let mut categories = load_embedded_patterns().unwrap(); + for category in &mut categories { + category.patterns.retain(|p| p.id == "PI079"); + for pattern in &mut category.patterns { + assert_eq!(pattern.pattern.matches("{40,}").count(), 1); + pattern.pattern = pattern.pattern.replace("{40,}", &format!("{{{minimum},}}")); + } + } + let scanner = Scanner::new(&categories).unwrap(); + for (specimen, expected) in cases.iter().zip(expected) { + assert_eq!( + fires(&scanner, &specimen.text), + expected, + "minimum={minimum}: {}", + specimen.name + ); + } + } +} + +#[test] +fn threshold_boundary_is_44_total_base64_characters() { + // Synthetic probes pin mechanics only; they are not independent evidence. + for (after_prefix, expected) in [(39, false), (40, true), (41, true)] { + let text = format!( + "echo 'ssh-ed25519 AAAA{}' >> ~/.ssh/authorized_keys", + "B".repeat(after_prefix) + ); + assert_eq!(fires(common::scanner(), &text), expected, "{after_prefix}"); + } +} From 3fe8d34a94427a581028e84a9ed2efe549f5dc64 Mon Sep 17 00:00:00 2001 From: Basil Zafar Date: Thu, 22 Oct 2026 21:25:34 +0530 Subject: [PATCH 2/3] fix: make PI079 remediation account for authorized key setup (#180) --- docs/PATTERN-CATALOGUE.md | 2 +- patterns/core/persistence-lifecycle-hijack.yaml | 6 +++++- tests/pattern_test.rs | 4 ++-- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/docs/PATTERN-CATALOGUE.md b/docs/PATTERN-CATALOGUE.md index 110cc5f..7258680 100644 --- a/docs/PATTERN-CATALOGUE.md +++ b/docs/PATTERN-CATALOGUE.md @@ -2078,7 +2078,7 @@ Does **not** catch: cat ~/.ssh/id_ed25519.pub >> ~/.ssh/authorized_keys ``` -**Fix:** Remove the line. Adding a key to authorized_keys grants login access, so it is the user's decision, made outside the document being read. +**Fix:** Verify that the user explicitly authorized this key installation outside the document being read. Remove the line if they did not. Legitimate SSH setup instructions can match this rule; the key's length does not establish intent.
Regex diff --git a/patterns/core/persistence-lifecycle-hijack.yaml b/patterns/core/persistence-lifecycle-hijack.yaml index 52d71cb..044b47f 100644 --- a/patterns/core/persistence-lifecycle-hijack.yaml +++ b/patterns/core/persistence-lifecycle-hijack.yaml @@ -583,11 +583,15 @@ patterns: counter_example: "cat ~/.ssh/id_ed25519.pub >> ~/.ssh/authorized_keys" name: persistence-command-with-payload severity: MEDIUM + # Independent hostile and benign matches: docs/PI079-EVIDENCE.md (#180). + # {40,} counts AFTER AAAA: 44 total base64 characters, excluding padding. + # This is an elision heuristic, not key validation or a measured optimum. + # Keep MEDIUM: real key-installation documentation has the same syntax. # relaxed_pattern drops the literal-blob requirement and keeps only the # append into an authorized-keys file, so the counter_example (the tutorial # line appending a key FILE, with no blob on the line) goes red under it. relaxed_pattern: "(?:>>\\s*|\\btee\\s+(?:-[a-z]+\\s+)*-[a-z]*a[a-z]*\\s+)[\\\"']?[^\\s\\\"'|;&]{0,40}?authorized_keys2?\\b" pattern: "(?:ssh-(?:rsa|ed25519|dss)|ecdsa-sha2-nistp(?:256|384|521))\\s+(?-i:AAAA)[A-Za-z0-9+/]{40,}={0,3}[^\\n]{0,250}?(?:>>\\s*|\\btee\\s+(?:-[a-z]+\\s+)*-[a-z]*a[a-z]*\\s+)[\\\"']?[^\\s\\\"'|;&]{0,40}?authorized_keys2?\\b" description: "A line carries a literal public-key blob of a named key type and, on the same line, appends it to an authorized-keys file with a redirect or an append-mode tee" - remediation: "Remove the line. Adding a key to authorized_keys grants login access, so it is the user's decision, made outside the document being read." + remediation: "Verify that the user explicitly authorized this key installation outside the document being read. Remove the line if they did not. Legitimate SSH setup instructions can match this rule; the key's length does not establish intent." tags: [persistence, ssh, authorized-keys] diff --git a/tests/pattern_test.rs b/tests/pattern_test.rs index 5b26d2b..648a0bd 100644 --- a/tests/pattern_test.rs +++ b/tests/pattern_test.rs @@ -1654,8 +1654,8 @@ fn test_pi077_lifecycle_hook_sensitive_command() { #[test] fn test_pi079_persistence_command_with_payload() { - // A real-shaped ed25519 public-key blob (68 base64 characters after the - // `AAAA` prefix, as the format produces). + // A real-shaped ed25519 public-key blob (68 base64 characters total, + // including the four-character `AAAA` prefix). assert_positives( "PI079", &[ From fc189cb154c3cdbb21b114c5b3989d7511cf3185 Mon Sep 17 00:00:00 2001 From: Basil Zafar Date: Thu, 22 Oct 2026 21:25:39 +0530 Subject: [PATCH 3/3] docs: record PI079 threshold measurements and limitations (#180) --- docs/DETECTION-BACKLOG.md | 4 +++ docs/PI079-EVIDENCE.md | 66 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 70 insertions(+) create mode 100644 docs/PI079-EVIDENCE.md diff --git a/docs/DETECTION-BACKLOG.md b/docs/DETECTION-BACKLOG.md index f55aea8..4187831 100644 --- a/docs/DETECTION-BACKLOG.md +++ b/docs/DETECTION-BACKLOG.md @@ -116,6 +116,10 @@ CRITICAL (D-06); `PI073`, `PI074` and `PI079` are MEDIUM and the other six are H `patterns/core/persistence-lifecycle-hijack.yaml` for the full set and its header comment, which carries every accepted-cost measurement named below. +PI079 now has [independent command evidence and a threshold sensitivity test](PI079-EVIDENCE.md) +(#180): a captured hostile key append and a legitimate key-installation example both match. +MEDIUM is retained; the length minimum is still a heuristic, not an optimized classifier. + The five bullets, as shipped: - `write this to CLAUDE.md`, `append to your memory`, `remember this permanently` — **shipped**: diff --git a/docs/PI079-EVIDENCE.md b/docs/PI079-EVIDENCE.md new file mode 100644 index 0000000..0e1e9b0 --- /dev/null +++ b/docs/PI079-EVIDENCE.md @@ -0,0 +1,66 @@ +# PI079: independent specimens and threshold sensitivity + +Issue [#180](https://github.com/UnityInFlow/injection-scanner/issues/180), +measured 2026-10-22 against the pattern on upstream `edec71c`. + +The committed excerpts in `tests/fixtures/pi079-evidence.json` supply one +independent hostile command and two independent benign documentation examples. +They were selected by searching for literal key-installation commands, not +sampled randomly. None was written for this scanner. HTML entities were decoded; +the SANS excerpt selects the contiguous echo-and-append command from a longer +command chain. No key was generated, expanded from an ellipsis, or replaced. +The tests treat these commands as text and never execute them. + +| Specimen and source | Classification | Shipped PI079 | +|---|---|---| +| [SANS ISC, Alex Rodriguez's DShield honeypot diary, 2023-11-27](https://isc.sans.edu/diary/30428), captured RSA key append | Hostile command | Match | +| [GeckoAdmin, SSH Key Authentication on Linux: Setup From Any Client](https://geckoadmin.com/ssh-key-authentication-setup/), Windows Ed25519 key installation | Benign documentation with a complete 68-character blob | Match (known false positive) | +| [Extract's SFTP connector documentation](https://docs.extract.to/destinations/sftp), manual key installation with an elided blob | Benign documentation | No match | + +These are command excerpts, not independently observed prompt injections. The +SANS article itself is defensive documentation, not a hostile document. Its +captured command is evidence for the command shape only. This does not add an +independent prompt-injection true positive or change the existing recall pins. +The benign overlap is recorded outside the zero-finding clean corpus because +the identical command can be authorized or unauthorized. No existing clean +specimen or suppression was changed to accommodate it. + +## Measurement + +Run `cargo test --locked --test pi079_evidence_test`. The sensitivity test changes +only PI079's minimum in memory, and uses the production `Scanner`, including its +normalization and decoding passes. It does not alter the shipped regex. + +The quantifier counts characters **after** the four literal `AAAA` characters; +the current minimum is therefore 44 base64 characters total, excluding padding. + +| Minimum after prefix | Hostile command matched (of 1) | Benign documents matched (of 2) | +|---|---|---| +| 8 | 1 | 2 | +| 40 (shipped) | 1 | 1 | +| 64 | 1 | 1 | +| 65 | 1 | 0 | +| 128 | 1 | 0 | +| 512 | 0 | 0 | + +The apparent improvement at 65 comes from excluding the complete Ed25519 key +(64 characters after the prefix). That excludes hostile keys of the same size +too. This small, deliberately selected sample cannot establish an optimal +cutoff, population false-positive rate, or recall rate. The large frozen +GATE-03 sweep remains a separate measurement with zero key-blob lines; these +three excerpts do not retroactively change that result. + +## Decision + +Retain the minimum and MEDIUM grade. The threshold remains a heuristic for short +elisions, now with measured sensitivity rather than an unsupported claim of +calibration. Length alone cannot separate legitimate key installation from +unauthorized persistence. The independent benign match supports keeping the +grade at MEDIUM under PATTERNS.md rule 3. The remediation now asks the reviewer +to verify explicit user authorization, acknowledging legitimate installations. + +The existing regex is not an SSH-key parser: it neither validates decoded key +structure nor guarantees that a long elided blob is rejected. The boundary +test uses synthetic 43/44/45-character blobs only to pin those mechanics, not +as evidence of valid keys. Broader sampling across key types and independent +prompt-injection documents is still needed before claiming calibration.