diff --git a/.agents/skills/docs_consistency_audit/SKILL.md b/.agents/skills/docs_consistency_audit/SKILL.md index 5f60c1e3d..9c13001c5 100644 --- a/.agents/skills/docs_consistency_audit/SKILL.md +++ b/.agents/skills/docs_consistency_audit/SKILL.md @@ -307,7 +307,43 @@ After that local transaction: If extraction, verification, validation, commit, push, or remote-SHA verification fails, leave the prior branch authoritative, write the failure to run output, and report a -blocked run. +blocked run using the failure path below. + +#### Failure reporting, including early exits + +If any required step fails, stop extraction and comparison. Create a blocked artifact +even when the normal `run` command never produced one: + +```bash +python3 .agents/skills/docs_consistency_audit/scripts/audit_consistency.py blocked-report \ + --run-id RUN_ID \ + --stage redirect-normalization \ + --code redirect-cycle \ + --state-dir STATE_WORKTREE/.agents/state/docs_consistency_audit \ + --pages 0 \ + --total-pages TOTAL_PAGES \ + --output /tmp/docs-consistency-run.json +``` + +Use the actual failed stage and matching code, not the example's values. Stages are +`preflight`, `inventory`, `redirect-normalization`, `authorities`, `extraction`, +`comparison`, and `state-persistence`. Codes are `redirect-cycle`, `source-unavailable`, +`state-unavailable`, `extraction-invalid`, `comparison-failed`, +`state-persistence-failed`, and `unknown`. Use `unknown` when the cause is unverified. +The helper maps codes to public-safe explanations and next actions; never put raw +exceptions, private evidence, or credentials in a report. + +Pass `--state-dir` only for the authoritative state fetched in preflight, not a partial +local replacement. Omit it when state cannot be read; do not fall back to default-branch +state. Omit unknown page counts rather than supplying zero. The helper reads state but +does not modify it. Write its output outside the state directory. If reading state for +this report fails, regenerate without `--state-dir`; the message then says the saved +count is unknown. + +Use this blocked artifact for step 9, including after a persistence failure. Do not post +the earlier comparison artifact as a successful result. Preserve prior findings and +receipts; no blocked run resolves a finding. Keep the blocker code, stage, known page +counts, and recommended action consistent in Slack, the run log, and final run output. ### 9. Report only actionable changes @@ -320,11 +356,23 @@ Post at most one top-level Slack message for: Unchanged findings, low-confidence candidates, successful no-change runs, and the inactive schedule guard do not post. The durable state and run log are their record. -The message includes counts for new, changed, resolved, and unchanged active findings; -up to three highest-severity titles; the standing state PR when it changed; and the -environment-correct run URL. Resolve the run URL from the current run at runtime. Never -hard-code a Warp host. Do not include all quotes or any private source path, symbol, -identifier, configuration value, or secret. +The message starts with the outcome: audit completed, partial audit, or could not check +docs. It explains that this audit detects conflicting or stale claims without editing +docs. For completed and partial audits, include counts for new, changed, resolved, and +unchanged active findings, plus up to three highest-severity changes with a brief +conflict explanation, public page links, and the action or human review needed. +Distinguish resolved findings from work still needed. + +For blocked runs, explain the failed stage, blocker, known page coverage, and next +action. State that no new results are available; label prior findings as saved history, +not reverified results. Missing failure or coverage details are unknown, not evidence +of zero findings or a successful check. Partial runs also name carried-forward findings +that were not rechecked. Do not describe extracted pages as fully audited pages. + +Link to run details and saved findings and audit history. Explain that the standing +draft PR contains audit state, not documentation fixes; do not ask readers to merge it. +Resolve the environment-correct run URL at runtime. Never hard-code a Warp host or put +the run UUID in the headline. Do not include full quotes or private source details. Generate the one decision and payload: @@ -337,7 +385,8 @@ python3 .agents/skills/docs_consistency_audit/scripts/audit_consistency.py notif ``` Call the connected Slack MCP once only when `should_post` is true, using `message` -verbatim. If posting fails, write the message to run output and report the delivery +verbatim with Slack mrkdwn formatting. Use the same outcome and next action in the final +run output. If posting fails, write the message to run output and report the delivery failure. Do not retry in a way that can duplicate the top-level message. ## Benchmark and rollout gate @@ -410,12 +459,14 @@ Prepend one entry after the file header: ## YYYY-MM-DD — complete | partial | blocked - Run: environment-correct URL or run ID - Source commits: docs SHA; warp SHA; warp-server SHA -- Coverage: complete | partial | blocked; PAGES/TOTAL pages -- Findings: NEW new; CHANGED changed; RESOLVED resolved; ACTIVE active +- Coverage: complete | partial | blocked; PAGES/TOTAL pages extracted | unknown +- Findings: NEW new; CHANGED changed; RESOLVED resolved; ACTIVE active | prior count unknown - Suppressed: COUNT; expired: COUNT - Cost: CHARACTERS characters; CANDIDATES candidates; MODEL_CALLS model calls -- Sources unavailable: none | comma-separated source classes +- Sources unavailable / blocker: none | public-safe source class, failed stage, code, and next action - State PR: URL | unchanged | unavailable ``` -Keep the entry under 10 lines. Never include secrets or private source details. +Keep the entry under 10 lines. For blocked runs, replace lifecycle counts with +`no new results; ACTIVE previously active, not rechecked` (or `prior count unknown`). +Never include secrets or private source details. diff --git a/.agents/skills/docs_consistency_audit/evals/evals.json b/.agents/skills/docs_consistency_audit/evals/evals.json index 7917de5e1..cf566673b 100644 --- a/.agents/skills/docs_consistency_audit/evals/evals.json +++ b/.agents/skills/docs_consistency_audit/evals/evals.json @@ -46,6 +46,40 @@ "A duplicate receipt or losing push race is a successful no-op.", "Exactly one invocation can mutate state or produce a notification." ] + }, + { + "id": 5, + "prompt": "Preview a blocked audit notification locally: redirect normalization failed before extraction, 0/390 pages extracted, and the authoritative saved state contains two active findings and no completed full audit. Use blocked-report and notification. Do not post to Slack or mutate saved state.", + "expected_output": "One payload explaining the purpose, redirect-checker failure, coverage, two saved findings not rechecked, next action, and links labeled by purpose. No zero-findings or success claim.", + "files": [], + "assertions": [ + "The headline states that the audit could not check docs, without a run UUID.", + "The blocker, 0/390 page coverage, and repair/rerun action are explicit.", + "Prior findings are not presented as newly verified.", + "The standing PR is described as audit history, not documentation fixes." + ] + }, + { + "id": 6, + "prompt": "Preview a partial audit notification locally with a newly verified privacy contradiction and two older active findings carried forward. Explain the conflict, link its public pages, and identify the required human review. Do not post to Slack.", + "expected_output": "A partial-audit payload with a short conflict explanation, public page links, human policy review, and a warning that carried-forward findings were not rechecked.", + "files": [], + "assertions": [ + "The message distinguishes partial coverage from a completed full audit.", + "The finding includes public page links and a human-review action.", + "No prior finding is resolved because it is absent from partial coverage." + ] + }, + { + "id": 7, + "prompt": "Preview an early preflight failure where saved audit state cannot be fetched and page coverage is unknown. Use blocked-report without --state-dir. Do not load default-branch state or post to Slack.", + "expected_output": "A failure payload stating that history could not be loaded, counts and coverage are unknown, and source/state access must be restored before rerunning.", + "files": [], + "assertions": [ + "Unknown counts are not replaced with zero.", + "No stale fallback state is read or changed.", + "The message explains the next action without exposing raw exceptions or private evidence." + ] } ] } diff --git a/.agents/skills/docs_consistency_audit/scripts/audit_consistency.py b/.agents/skills/docs_consistency_audit/scripts/audit_consistency.py index d747a9c42..e0d8bb5fd 100644 --- a/.agents/skills/docs_consistency_audit/scripts/audit_consistency.py +++ b/.agents/skills/docs_consistency_audit/scripts/audit_consistency.py @@ -26,6 +26,7 @@ from decimal import Decimal from pathlib import Path from typing import Any, Iterable +from urllib.parse import urlsplit from zoneinfo import ZoneInfo @@ -155,6 +156,10 @@ def load_verified_redirects(path: Path | None) -> dict[str, str]: continue if any(token in source for token in (":", "*", "(", "{")): continue + # Slash canonicalization is not an edge between distinct claim identities. + # Keep literal self-redirects so the cycle detector still rejects them. + if source != destination and normalize_route(source) == normalize_route(destination): + continue redirects[normalize_route(source)] = normalize_route(destination) return redirects @@ -1768,7 +1773,7 @@ def run_pipeline(args: argparse.Namespace) -> int: "changed": sum(item.get("lifecycle") == "changed" for item in lifecycle["changes"]), "resolved": sum(item.get("lifecycle") == "resolved" for item in lifecycle["changes"]), "existing": sum( - item.get("lifecycle") == "existing" + item.get("lifecycle") == "existing" and item.get("status") == "active" for item in lifecycle["findings"].values() ), "suppressed": len(suppressed), @@ -1782,6 +1787,18 @@ def run_pipeline(args: argparse.Namespace) -> int: "run_id": run_id, "source_commit": args.source_commit, "coverage": coverage, + "report_context": { + "pages_extracted": len(processed_pages), + "pages_reused": len(unchanged_pages), + "total_pages": len(inventory["pages"]) if "pages" in inventory else None, + "prior_state_available": bool(previous_manifest), + "last_completed_run": previous_manifest.get("last_completed_run"), + "carried_forward": sum( + item.get("status") == "active" + and item.get("carried_forward_due_to_incomplete_run", False) + for item in lifecycle["findings"].values() + ), + }, "source_availability": ( load_json(Path(args.source_availability), {}) if args.source_availability @@ -2179,21 +2196,183 @@ def schedule_claim_command(args: argparse.Namespace) -> int: return 0 -def notification_command(args: argparse.Namespace) -> int: - artifact = load_json(Path(args.artifact)) +FAILURE_MESSAGES = { + "redirect-cycle": ( + "The audit's redirect checker detected a loop and stopped comparison.", + "Inspect redirect normalization in the audit helper and the public redirect " + "chain, correct the cause, then rerun the audit.", + ), + "source-unavailable": ( + "A required source could not be read, so the audit could not finish.", + "Restore access to the required source, then rerun the audit.", + ), + "state-unavailable": ( + "Saved audit history could not be loaded safely.", + "Restore access to the standing audit state branch before rerunning.", + ), + "extraction-invalid": ( + "Claim extraction did not pass evidence or format validation.", + "Inspect the extraction failure in the run details, correct it, then rerun.", + ), + "comparison-failed": ( + "The audit could not finish comparing documentation claims.", + "Inspect the comparison failure in the run details, correct it, then rerun.", + ), + "state-persistence-failed": ( + "The audit could not verify that its results were safely saved.", + "Check state validation, push, and remote verification in the run details " + "before rerunning. Do not merge the standing state PR.", + ), + "unknown": ( + "The audit stopped, but no failure reason was recorded.", + "Inspect the run details to identify the blocker before rerunning.", + ), +} +AUDIT_STAGES = ( + "preflight", "inventory", "redirect-normalization", "authorities", + "extraction", "comparison", "state-persistence", "unknown", +) + + +def blocked_report_command(args: argparse.Namespace) -> int: + """Create a report without extracting claims or mutating saved audit state.""" + context = { + "pages_extracted": args.pages, + "total_pages": args.total_pages, + "prior_state_available": False, + } + for count in (args.pages, args.total_pages): + if count is not None and count < 0: + raise AuditError("Page counts cannot be negative") + if args.pages is not None and args.total_pages is not None and args.pages > args.total_pages: + raise AuditError("Extracted pages cannot exceed total pages") + summary = {} + if args.state_dir: + state_dir = Path(args.state_dir).resolve() + if Path(args.output).resolve().is_relative_to(state_dir): + raise AuditError("Blocked reports must be written outside the saved state directory") + manifest = load_json(state_dir / "manifest.json") + saved = load_json(state_dir / "findings.json") + ensure_no_private_reference(saved) + active = sum( + item.get("status") == "active" + for item in saved.get("findings", {}).values() + ) + context.update({ + "prior_state_available": True, + "last_completed_run": manifest.get("last_completed_run"), + "carried_forward": active, + }) + summary = {"active": active, "existing": active} + artifact = { + "schema_version": SCHEMA_VERSION, + "run_id": args.run_id, + "coverage": "blocked", + "failure": {"stage": args.stage, "code": args.code}, + "report_context": context, + "summary": summary, + "lifecycle_changes": [], + } ensure_no_private_reference(artifact) + write_json(Path(args.output), artifact) + print(canonical_json(artifact)) + return 0 + + +def slack_text(value: str, limit: int = 300) -> str: + """Keep evidence on one line and prevent Slack mentions/link injection.""" + value = " ".join(value.split()) + if len(value) > limit: + value = value[:limit - 1] + "…" + return value.replace("&", "&").replace("<", "<").replace(">", ">") + + +def slack_link(url: str, label: str) -> str: + parsed = urlsplit(url) + if ( + parsed.scheme != "https" or not parsed.hostname or parsed.username + or parsed.password or any(char.isspace() or char in "<>|" for char in url) + ): + raise AuditError("Report links must be HTTPS URLs without credentials or Slack markup") + ensure_no_private_reference(url) + return f"<{url}|{slack_text(label)}>" + + +def finding_message(finding: dict[str, Any]) -> str: + lifecycle = finding.get("lifecycle", "changed") + if finding.get("reopened") and lifecycle == "changed": + lifecycle = "reopened" + line = ( + f"• {slack_text(lifecycle.capitalize())} · " + f"{slack_text(finding.get('severity', 'unknown'))} severity · " + f"{slack_text(finding.get('confidence', 'unknown'))} confidence: " + f"{slack_text(finding.get('title', 'Untitled finding'), 160)}" + ) + claims = finding.get("claims", []) + public_pages = [] + for claim in claims: + validate_public_claim(claim) + source = claim["source"] + route = source.get("route", "") + if ( + source.get("repository") == "warpdotdev/docs" + and re.fullmatch(r"/[a-zA-Z0-9_./-]*", route) + and not route.startswith("//") + ): + link = slack_link("https://docs.warp.dev" + route, route) + if link not in public_pages: + public_pages.append(link) + # The title/rationale describes the conflict; full quotes belong in the artifact. + if finding.get("rationale"): + line += "\n " + slack_text(finding["rationale"]) + if public_pages: + line += "\n Pages: " + " · ".join(public_pages[:3]) + if lifecycle == "resolved": + action = "Resolved in this comparison; no documentation edit requested." + elif finding.get("requires_human_review") or any( + term in re.split(r"[\s_-]+", finding.get(key, "").lower()) + for key in ("category", "topic") + for term in ( + "security", "privacy", "billing", "retention", "training", + "telemetry", "permission", "permissions", "redaction", "data", + ) + ): + action = "Human policy review required before editing these claims." + else: + action = finding.get("suggested_resolution") or "Review the conflicting claims before editing." + return line + "\n Action: " + slack_text(action) + + +def render_notification( + artifact: dict[str, Any], + run_url: str | None = None, + state_pr_url: str | None = None, +) -> dict[str, Any]: + ensure_no_private_reference(artifact) + coverage = artifact.get("coverage") + if coverage not in {"complete", "partial", "blocked"}: + raise AuditError("Notification requires complete, partial, or blocked coverage") changes = artifact.get("lifecycle_changes", []) actionable = [ finding for finding in changes - if finding.get("confidence") == "high" - or ( - finding.get("severity") == "high" - and finding.get("confidence") == "medium" + if finding.get("lifecycle") != "existing" + and ( + finding.get("confidence") == "high" + or ( + finding.get("severity") == "high" + and finding.get("confidence") == "medium" + ) ) ] - blocked = artifact.get("coverage") == "blocked" + blocked = coverage == "blocked" + if blocked: + # Unpersisted or incomplete changes are not new audit results. + actionable = [] + elif coverage == "partial": + actionable = [item for item in actionable if item.get("lifecycle") != "resolved"] should_post = blocked or bool(actionable) summary = artifact.get("summary", {}) + context = artifact.get("report_context", {}) titles = sorted( actionable, key=lambda finding: ( @@ -2201,24 +2380,92 @@ def notification_command(args: argparse.Namespace) -> int: finding.get("title", ""), ), )[:3] + outcome = { + "complete": "audit completed", + "partial": "partial audit", + "blocked": "could not check docs", + }[coverage] lines = [ - f"{'⚠️' if blocked else '📚'} Docs consistency audit · {artifact.get('run_id', 'unknown run')}", - ( - "Run blocked; prior findings were preserved." - if blocked - else ( - f"New {summary.get('new', 0)} · changed {summary.get('changed', 0)} · " - f"resolved {summary.get('resolved', 0)} · active {summary.get('active', 0)}" - ) - ), + f"*Docs consistency audit — {outcome}*", + "Checks public documentation for conflicting or stale claims. It does not edit docs.", + "", ] - if titles: - lines.append("Highest-severity changes: " + "; ".join(item["title"] for item in titles)) - if args.state_pr_url: - lines.append(f"State PR: {args.state_pr_url}") - if args.run_url: - lines.append(f"Run: {args.run_url}") - result = { + pages, total = context.get("pages_extracted"), context.get("total_pages") + coverage_text = ( + f"{pages}/{total} pages extracted." + if pages is not None and total is not None + else f"{pages} pages extracted; total unavailable." + if pages is not None + else "Page coverage was not recorded." + ) + if not blocked and "pages_reused" in context: + coverage_text += f" {context['pages_reused']} unchanged pages reused from saved claims." + if blocked: + failure = artifact.get("failure", {}) + stage = failure.get("stage", "unknown") + if stage not in AUDIT_STAGES: + stage = "unknown" + explanation, action = FAILURE_MESSAGES.get( + failure.get("code"), FAILURE_MESSAGES["unknown"], + ) + lines.extend([ + f"*What happened:* {explanation}", + f"*Stopped at:* {stage.replace('-', ' ')}. {coverage_text}", + "*What changed:* No new results are available from this blocked audit.", + ]) + if context.get("prior_state_available") and "active" in summary: + count = summary["active"] + saved_findings = "finding remains" if count == 1 else "findings remain" + lines.append( + f"{count} previously active {saved_findings} in saved history; " + "this run did not verify their current status." + ) + else: + lines.append("Saved findings were not loaded; their current count is unknown.") + lines.append("*Next action:* " + action) + else: + lines.extend([ + f"*Coverage:* {coverage_text}", + ( + f"*What changed:* {summary.get('new', 0)} new · " + f"{summary.get('changed', 0)} changed · " + f"{summary.get('resolved', 0) if coverage == 'complete' else 0} resolved · " + f"{summary.get('existing', 0)} unchanged active · {summary.get('active', 0)} active total." + ), + ]) + if coverage == "partial": + lines.append( + "Coverage is incomplete. Prior findings are not resolved by this run; " + f"{context.get('carried_forward', 'an unknown number of')} active findings " + "were carried forward without rechecking." + ) + lines.extend(finding_message(item) for item in titles) + if len(actionable) > len(titles): + count = len(actionable) - len(titles) + remainder = "change is" if count == 1 else "changes are" + lines.append(f"{count} more {remainder} in the audit history.") + lines.append( + "*Next action:* Review the changes above." + if any(item.get("lifecycle") != "resolved" for item in actionable) + else "*Next action:* No edits requested for these resolved findings." + ) + if ( + coverage != "complete" + and context.get("prior_state_available") + and "last_completed_run" in context + and context["last_completed_run"] is None + ): + lines.append("Saved history records no completed full audit.") + links = [] + if run_url: + links.append(slack_link(run_url, "Run details")) + if state_pr_url: + links.append(slack_link(state_pr_url, "Saved findings and audit history")) + if links: + lines.extend(["", " · ".join(links)]) + if state_pr_url: + lines.append("The linked PR stores audit state; it contains no documentation fixes.") + return { "should_post": should_post, "reason": ( "blocked" @@ -2230,6 +2477,12 @@ def notification_command(args: argparse.Namespace) -> int: "message": "\n".join(lines) if should_post else None, "actionable_finding_ids": [item["finding_id"] for item in actionable], } + + +def notification_command(args: argparse.Namespace) -> int: + result = render_notification( + load_json(Path(args.artifact)), args.run_url, args.state_pr_url, + ) write_json(Path(args.output), result) print(canonical_json(result)) return 0 @@ -2301,6 +2554,18 @@ def build_parser() -> argparse.ArgumentParser: notification.add_argument("--state-pr-url") notification.add_argument("--output", required=True) notification.set_defaults(func=notification_command) + blocked_report = subparsers.add_parser( + "blocked-report", + help="Create a public-safe failure artifact without modifying prior audit state", + ) + blocked_report.add_argument("--run-id", required=True) + blocked_report.add_argument("--stage", choices=AUDIT_STAGES, required=True) + blocked_report.add_argument("--code", choices=sorted(FAILURE_MESSAGES), required=True) + blocked_report.add_argument("--state-dir", help="Previously fetched authoritative state only") + blocked_report.add_argument("--pages", type=int) + blocked_report.add_argument("--total-pages", type=int) + blocked_report.add_argument("--output", required=True) + blocked_report.set_defaults(func=blocked_report_command) inventory = subparsers.add_parser("inventory", help="Inventory public docs and plan changed shards") inventory.add_argument("--docs-repo", default="/workspace/docs") diff --git a/.agents/skills/docs_consistency_audit/scripts/test_audit_consistency.py b/.agents/skills/docs_consistency_audit/scripts/test_audit_consistency.py index c82626a16..0fbbdac10 100644 --- a/.agents/skills/docs_consistency_audit/scripts/test_audit_consistency.py +++ b/.agents/skills/docs_consistency_audit/scripts/test_audit_consistency.py @@ -450,6 +450,50 @@ def test_incomplete_context_suppresses_exact_mismatch(self): self.assertIsNone(result) +class RedirectTests(unittest.TestCase): + def load(self, entries): + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "vercel.json" + path.write_text(json.dumps({"redirects": [ + {"source": source, "destination": destination} + for source, destination in entries + ]})) + return audit.load_verified_redirects(path) + + def test_slash_canonicalization_is_not_a_cycle_in_either_direction(self): + for source, destination in (("/agents", "/agents/"), ("/agents/", "/agents")): + with self.subTest(source=source): + redirects = self.load([ + ("/old", "/older/"), ("/older", "/agents/"), + (source, destination), + ]) + self.assertNotIn("/agents", redirects) + self.assertEqual(audit.canonical_redirect_route("/old", redirects), "/agents") + old, new = claim(path="old.mdx"), claim(path="agents.mdx") + old["source"]["route"], new["source"]["route"] = "/old/", "/agents/" + self.assertEqual( + audit.normalize_claim(audit.apply_redirect_identity(old, redirects))["claim_id"], + audit.normalize_claim(audit.apply_redirect_identity(new, redirects))["claim_id"], + ) + + def test_genuine_self_and_multi_route_cycles_still_fail(self): + for entries in ( + [("/agents", "/agents")], + [("/agents/", "/agents/")], + [("/a", "/b/"), ("/b", "/a/")], + ): + with self.subTest(entries=entries): + redirects = self.load(entries) + with self.assertRaisesRegex(audit.AuditError, "Redirect cycle"): + audit.canonical_redirect_route(entries[0][0], redirects) + + def test_repository_redirects_do_not_create_false_cycles(self): + redirects = audit.load_verified_redirects(SCRIPT.parents[4] / "vercel.json") + self.assertTrue(redirects) + for route in sorted(set(redirects) | set(redirects.values())): + audit.canonical_redirect_route(route, redirects) + + class FingerprintAndLifecycleTests(unittest.TestCase): def setUp(self): self.left = claim() @@ -1141,6 +1185,192 @@ def test_blocked_run_produces_payload_without_findings(self): }) self.assertTrue(result["should_post"]) self.assertEqual(result["reason"], "blocked") + self.assertIn("could not check docs", result["message"]) + self.assertIn("no failure reason was recorded", result["message"]) + self.assertIn("count is unknown", result["message"]) + self.assertNotIn("0 new", result["message"]) + + def test_blocked_run_explains_coverage_history_and_next_action(self): + result = self.render({ + "run_id": "do-not-put-this-in-the-headline", + "coverage": "blocked", + "failure": {"stage": "redirect-normalization", "code": "redirect-cycle"}, + "summary": {"active": 2}, + "report_context": { + "pages_extracted": 0, "total_pages": 390, + "prior_state_available": True, "last_completed_run": None, + }, + "lifecycle_changes": [], + }) + message = result["message"] + for expected in ( + "conflicting or stale claims", "redirect checker detected a loop", + "0/390 pages extracted", "No new results", "2 previously active", + "did not verify their current status", "Next action", + "no completed full audit", "", + "", + "no documentation fixes", + ): + self.assertIn(expected, message) + self.assertNotIn("do-not-put-this-in-the-headline", message) + + def artifact(self, coverage="complete", **changes): + item = finding(claim(path="one.mdx"), claim(path="two.mdx", value=3000)) + item.update({ + "lifecycle": "new", + "rationale": "The overview and reference give different file limits.", + "suggested_resolution": "Confirm the limit, then align the overview.", + **changes, + }) + return { + "coverage": coverage, + "summary": {"new": 1, "changed": 0, "resolved": 0, "existing": 2, "active": 3}, + "report_context": {"pages_extracted": 8, "total_pages": 390, "carried_forward": 2}, + "lifecycle_changes": [item], + } + + def test_complete_message_has_conflict_pages_and_action(self): + message = self.render(self.artifact())["message"] + for expected in ( + "audit completed", "New · medium severity", "overview and reference", + "", "Confirm the limit", + "2 unchanged active", + ): + self.assertIn(expected, message) + + def test_partial_message_does_not_imply_complete_verification(self): + message = self.render(self.artifact("partial"))["message"] + self.assertIn("partial audit", message) + self.assertIn("8/390 pages extracted", message) + self.assertIn("2 active findings were carried forward without rechecking", message) + self.assertNotIn("audit completed", message) + + def test_resolved_finding_does_not_request_an_edit(self): + message = self.render(self.artifact(lifecycle="resolved", reopened=True))["message"] + self.assertIn("Resolved · medium severity", message) + self.assertIn("No edits requested", message) + self.assertNotIn("Confirm the limit", message) + + def test_partial_or_blocked_never_announces_resolution(self): + for coverage in ("partial", "blocked"): + artifact = self.artifact(coverage, lifecycle="resolved") + result = self.render(artifact) + self.assertEqual(result["actionable_finding_ids"], []) + if coverage == "partial": + self.assertFalse(result["should_post"]) + + def test_policy_findings_require_human_review(self): + message = self.render(self.artifact(requires_human_review=True))["message"] + self.assertIn("Human policy review required", message) + self.assertNotIn("Confirm the limit", message) + message = self.render(self.artifact(category="privacy"))["message"] + self.assertIn("Human policy review required", message) + + def test_reopened_findings_are_named(self): + message = self.render(self.artifact(lifecycle="changed", reopened=True))["message"] + self.assertIn("Reopened ·", message) + + def test_changes_are_severity_ordered_and_limited_to_three(self): + artifact = self.artifact() + artifact["lifecycle_changes"] = [ + {**artifact["lifecycle_changes"][0], "finding_id": str(i), + "severity": severity, "title": f"Finding {i}"} + for i, severity in enumerate(("low", "high", "medium", "high")) + ] + message = self.render(artifact)["message"] + self.assertLess(message.index("Finding 1"), message.index("Finding 3")) + self.assertLess(message.index("Finding 3"), message.index("Finding 2")) + self.assertNotIn("Finding 0", message) + self.assertIn("1 more change is", message) + + def test_low_confidence_and_existing_findings_remain_silent(self): + for changes in ({"confidence": "low"}, {"lifecycle": "existing"}): + result = self.render(self.artifact(**changes)) + self.assertFalse(result["should_post"]) + self.assertIsNone(result["message"]) + + def test_high_severity_medium_confidence_still_posts(self): + result = self.render(self.artifact(severity="high", confidence="medium")) + self.assertTrue(result["should_post"]) + + def test_private_evidence_is_rejected(self): + artifact = self.artifact() + artifact["lifecycle_changes"][0]["rationale"] = "warpdotdev/warp-server" + with self.assertRaises(audit.AuditError): + self.render(artifact) + artifact = self.artifact() + artifact["lifecycle_changes"][0]["claims"][0]["source"]["visibility"] = "private" + with self.assertRaises(audit.AuditError): + self.render(artifact) + + def test_slack_markup_is_escaped_and_urls_are_validated(self): + message = self.render(self.artifact(title=" "))["message"] + self.assertNotIn("", message) + self.assertIn("<!channel>", message) + for url in ("http://example.test", "https://user:password@example.test", "https://x|bad"): + with self.assertRaises(audit.AuditError): + audit.slack_link(url, "Run details") + + +class BlockedReportTests(unittest.TestCase): + def test_early_failure_without_state_records_unknown_counts(self): + with tempfile.TemporaryDirectory() as directory: + output = Path(directory) / "blocked.json" + self.assertEqual(audit.main([ + "blocked-report", "--run-id", "run-1", "--stage", "preflight", + "--code", "state-unavailable", "--output", str(output), + ]), 0) + artifact = json.loads(output.read_text()) + message = audit.render_notification(artifact)["message"] + self.assertIn("Saved audit history could not be loaded safely", message) + self.assertIn("Page coverage was not recorded", message) + self.assertNotIn("0/0", message) + + def test_blocked_report_reads_but_does_not_modify_prior_state(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + state = root / "state" + state.mkdir() + audit.write_json(state / "manifest.json", {"last_completed_run": None}) + audit.write_json(state / "findings.json", {"findings": { + "active": {"status": "active"}, "resolved": {"status": "resolved"}, + }}) + before = {path: path.read_bytes() for path in state.iterdir()} + output = root / "blocked.json" + self.assertEqual(audit.main([ + "blocked-report", "--run-id", "run-1", "--stage", "comparison", + "--code", "comparison-failed", "--state-dir", str(state), + "--pages", "8", "--total-pages", "390", "--output", str(output), + ]), 0) + artifact = json.loads(output.read_text()) + self.assertEqual(artifact["summary"]["active"], 1) + self.assertEqual(artifact["lifecycle_changes"], []) + self.assertEqual(before, {path: path.read_bytes() for path in state.iterdir()}) + + def test_invalid_counts_fail_without_writing_report(self): + with tempfile.TemporaryDirectory() as directory: + output = Path(directory) / "blocked.json" + for pages, total in (("-1", "390"), ("391", "390")): + self.assertEqual(audit.main([ + "blocked-report", "--run-id", "run-1", "--stage", "inventory", + "--code", "unknown", "--pages", pages, "--total-pages", total, + "--output", str(output), + ]), 2) + self.assertFalse(output.exists()) + + + def test_report_cannot_overwrite_saved_state(self): + with tempfile.TemporaryDirectory() as directory: + state = Path(directory) + target = state / "manifest.json" + target.write_text('{"last_completed_run": null}') + before = target.read_bytes() + self.assertEqual(audit.main([ + "blocked-report", "--run-id", "run-1", "--stage", "comparison", + "--code", "comparison-failed", "--state-dir", str(state), + "--output", str(target), + ]), 2) + self.assertEqual(target.read_bytes(), before) if __name__ == "__main__":