diff --git a/src/adjudication.rs b/src/adjudication.rs index d6e762f..c04f479 100644 --- a/src/adjudication.rs +++ b/src/adjudication.rs @@ -118,6 +118,7 @@ pub(crate) struct AdjudicationApplication { pub kept: Vec, pub kept_indices: Vec, pub unresolved_indices: Vec, + pub invalid_refutation_indices: Vec, pub resolved_indices: Vec, pub suppressed: Vec, } @@ -127,6 +128,7 @@ enum DeterministicDemotionReason { RepositoryReceipt, CitationFragment, InvalidConfirmation, + InvalidRefutation, } #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -1165,27 +1167,14 @@ pub(crate) fn validate_results( let claim_verdict = finding.repository_claim.as_ref().map(|claim| { crate::repository_search::claim_verdict(claim, repository_receipt, snapshot_id) }); - let direct_refutation_grounded = evidence_is_refutation_grounded( - &result.evidence, - &result.candidate_id, - corpus, - diff_receipt, - ) || evidence_is_complete_matching_window_refutation( - &result.evidence, + let refutation_grounded = result_has_grounded_refutation( + snapshot_id, finding, - &result.candidate_id, + result, corpus, diff_receipt, + repository_receipt, ); - let repository_refutation_grounded = - finding.repository_claim.as_ref().is_some_and(|claim| { - crate::repository_search::refutation_evidence_is_grounded( - claim, - repository_receipt, - snapshot_id, - &result.evidence, - ) - }); match result.status { AdjudicationStatus::Confirmed => { ensure!( @@ -1228,15 +1217,9 @@ pub(crate) fn validate_results( "refuted adjudication cannot publish revised finding text" ); ensure!( - direct_refutation_grounded || repository_refutation_grounded, + refutation_grounded, "refuted adjudication must cite candidate-specific contradictory evidence" ); - if claim_verdict.is_some() { - ensure!( - direct_refutation_grounded || repository_refutation_grounded, - "repository-dependent finding lacks exact candidate-specific refutation evidence" - ); - } } AdjudicationStatus::Unresolved => ensure!( result.revised_title.is_empty() @@ -1290,12 +1273,20 @@ pub(crate) fn apply_results( let mut kept = Vec::new(); let mut kept_indices = Vec::new(); let mut unresolved_indices = Vec::new(); + let mut invalid_refutation_indices = Vec::new(); let mut resolved_indices = Vec::new(); let mut suppressed = Vec::new(); for (index, (mut finding, id)) in findings.into_iter().zip(candidate_ids).enumerate() { let outcome = by_id .get(&id) .ok_or_else(|| anyhow!("validated adjudication result disappeared"))?; + if outcome.provenance + == AdjudicationProvenance::DeterministicEvidenceReceipt( + DeterministicDemotionReason::InvalidRefutation, + ) + { + invalid_refutation_indices.push(index); + } match (outcome.provenance, outcome.disposition) { (AdjudicationProvenance::Model, AdjudicationDisposition::RetainConfirmed) => { finding @@ -1342,6 +1333,7 @@ pub(crate) fn apply_results( kept, kept_indices, unresolved_indices, + invalid_refutation_indices, resolved_indices, suppressed, }) @@ -1509,11 +1501,50 @@ fn deterministic_demotion_reason( }) { Some(DeterministicDemotionReason::InvalidConfirmation) + } else if matches!(result.status, AdjudicationStatus::Refuted) + && result.revised_title.is_empty() + && result.revised_body.is_empty() + && !result_has_grounded_refutation( + snapshot_id, + finding, + result, + corpus, + receipt, + repository_receipt, + ) + { + Some(DeterministicDemotionReason::InvalidRefutation) } else { None } } +fn result_has_grounded_refutation( + snapshot_id: &str, + finding: &Finding, + result: &AdjudicationResult, + corpus: &str, + receipt: &DiffCorpusReceipt, + repository_receipt: &RepositorySearchReceipt, +) -> bool { + evidence_is_refutation_grounded(&result.evidence, &result.candidate_id, corpus, receipt) + || evidence_is_complete_matching_window_refutation( + &result.evidence, + finding, + &result.candidate_id, + corpus, + receipt, + ) + || finding.repository_claim.as_ref().is_some_and(|claim| { + crate::repository_search::refutation_evidence_is_grounded( + claim, + repository_receipt, + snapshot_id, + &result.evidence, + ) + }) +} + fn result_is_bounded_citation_fragment(result: &AdjudicationResult, finding: &Finding) -> bool { let Some(citation) = finding.evidence.as_deref() else { return false; @@ -1757,7 +1788,7 @@ mod tests { let corpus = "@@ -3 +3 @@\n- uses: action@old\n@@ -69 +74 @@\n+ uses: action@new\n"; let direct = direct_receipt(&snapshot, corpus, &findings, &ids); assert!(direct.rendered_evidence.contains("uses: action@new")); - let error = apply_results( + let applied = apply_results( &snapshot, findings, ids, @@ -1766,12 +1797,10 @@ mod tests { &direct, &unavailable_receipt(), ) - .unwrap_err(); - assert!( - error - .to_string() - .contains("candidate-specific contradictory evidence") - ); + .unwrap(); + assert_eq!(applied.kept.len(), 2); + assert_eq!(applied.unresolved_indices, vec![0, 1]); + assert!(applied.suppressed.is_empty()); } #[test] @@ -2023,6 +2052,137 @@ mod tests { assert_eq!(applied.suppressed.len(), 1); } + #[test] + fn ungrounded_refutation_preserves_the_original_candidate_for_scoring() { + let snapshot = "a".repeat(40); + let mut candidate = finding( + Kind::Risk, + "Clean change breaks runtime behavior", + "This change removes required runtime behavior and will break callers after merge.", + ); + candidate.path = "src/lib/logger.ts".into(); + candidate.line = 11; + candidate.confidence = 0.95; + candidate.evidence = Some(" // keep the log prefix stable for tests".into()); + let findings = vec![candidate]; + let ids = stable_candidate_ids(&snapshot, &findings); + let corpus = "--- a/src/lib/logger.ts\n+++ b/src/lib/logger.ts\n@@ -11 +11 @@\n- // keep the log prefix stable\n+ // keep the log prefix stable for tests\n"; + let receipt = direct_receipt(&snapshot, corpus, &findings, &ids); + for evidence in [ + "+ // keep the log prefix stable for tests", + " // keep the log prefix stable for tests", + "unsupported evidence", + "", + ] { + let result = AdjudicationResult { + candidate_id: ids[0].clone(), + status: AdjudicationStatus::Refuted, + revised_title: String::new(), + revised_body: String::new(), + evidence: evidence.into(), + duplicate_of: None, + }; + assert!( + validate_results( + &snapshot, + &findings, + &ids, + std::slice::from_ref(&result), + corpus, + &receipt, + &unavailable_receipt(), + ) + .is_err() + ); + let applied = apply_results( + &snapshot, + findings.clone(), + ids.clone(), + vec![result], + corpus, + &receipt, + &unavailable_receipt(), + ) + .unwrap(); + assert_eq!( + serde_json::to_value(&applied.kept).unwrap(), + serde_json::to_value(&findings).unwrap() + ); + assert_eq!(applied.kept_indices, vec![0]); + assert_eq!(applied.unresolved_indices, vec![0]); + assert!(applied.resolved_indices.is_empty()); + assert!(applied.suppressed.is_empty()); + } + } + + #[test] + fn ungrounded_refutation_does_not_repair_invalid_structure_or_snapshot() { + let snapshot = "a".repeat(40); + let findings = vec![finding( + Kind::Risk, + "Preserve validation", + "The change removes validation.", + )]; + let ids = stable_candidate_ids(&snapshot, &findings); + let corpus = "+ perform_write(input);\n"; + let receipt = direct_receipt(&snapshot, corpus, &findings, &ids); + let result = AdjudicationResult { + candidate_id: ids[0].clone(), + status: AdjudicationStatus::Refuted, + revised_title: String::new(), + revised_body: String::new(), + evidence: "unsupported evidence".into(), + duplicate_of: None, + }; + for results in [ + vec![], + vec![result.clone(), result.clone()], + vec![AdjudicationResult { + candidate_id: "unknown".into(), + ..result.clone() + }], + vec![AdjudicationResult { + duplicate_of: Some(ids[0].clone()), + ..result.clone() + }], + vec![AdjudicationResult { + revised_title: "invalid publication".into(), + ..result.clone() + }], + vec![AdjudicationResult { + revised_body: " ".into(), + ..result.clone() + }], + ] { + assert!( + apply_results( + &snapshot, + findings.clone(), + ids.clone(), + results, + corpus, + &receipt, + &unavailable_receipt(), + ) + .is_err() + ); + } + assert!( + apply_results( + &"b".repeat(40), + findings, + ids, + vec![result], + corpus, + &receipt, + &unavailable_receipt(), + ) + .unwrap_err() + .to_string() + .contains("snapshot mismatch") + ); + } + #[test] fn candidate_location_cannot_refute_its_own_finding() { let snapshot = "a".repeat(40); @@ -2055,18 +2215,19 @@ mod tests { duplicate_of: None, }; - assert!( - apply_results( - &snapshot, - findings, - ids, - vec![result], - corpus, - &receipt, - &unavailable_receipt(), - ) - .is_err() - ); + let applied = apply_results( + &snapshot, + findings, + ids, + vec![result], + corpus, + &receipt, + &unavailable_receipt(), + ) + .unwrap(); + assert_eq!(applied.kept.len(), 1); + assert_eq!(applied.unresolved_indices, vec![0]); + assert!(applied.suppressed.is_empty()); } #[test] @@ -2099,18 +2260,19 @@ mod tests { &ids[0], &receipt, )); - assert!( - apply_results( - &snapshot, - findings, - ids, - vec![result], - corpus, - &receipt, - &unavailable_receipt(), - ) - .is_err() - ); + let applied = apply_results( + &snapshot, + findings, + ids, + vec![result], + corpus, + &receipt, + &unavailable_receipt(), + ) + .unwrap(); + assert_eq!(applied.kept.len(), 1); + assert_eq!(applied.unresolved_indices, vec![0]); + assert!(applied.suppressed.is_empty()); } #[test] @@ -3078,17 +3240,30 @@ mod tests { ..result }; assert!( - apply_results( + validate_results( &snapshot, - findings, - ids, - vec![lexical_refutation], + &findings, + &ids, + std::slice::from_ref(&lexical_refutation), corpus, &direct, &lexical_match, ) .is_err() ); + let applied = apply_results( + &snapshot, + findings, + ids, + vec![lexical_refutation], + corpus, + &direct, + &lexical_match, + ) + .unwrap(); + assert_eq!(applied.kept.len(), 1); + assert_eq!(applied.unresolved_indices, vec![0]); + assert!(applied.suppressed.is_empty()); } #[test] @@ -3168,7 +3343,7 @@ mod tests { let corpus = "+ image: old-image\n"; let direct = direct_receipt(&snapshot, corpus, &findings, &ids); - let error = apply_results( + let applied = apply_results( &snapshot, findings, ids, @@ -3177,12 +3352,10 @@ mod tests { &direct, &receipt, ) - .unwrap_err(); - assert!( - error - .to_string() - .contains("candidate-specific contradictory evidence") - ); + .unwrap(); + assert_eq!(applied.kept.len(), 1); + assert_eq!(applied.unresolved_indices, vec![0]); + assert!(applied.suppressed.is_empty()); } #[test] @@ -3465,12 +3638,11 @@ mod tests { &receipt, &unavailable_receipt(), ); - if confirmed { - let applied = applied.unwrap(); - assert_eq!(applied.kept.len(), 1); - assert!(applied.suppressed.is_empty()); - } else { - assert!(applied.is_err()); + let applied = applied.unwrap(); + assert_eq!(applied.kept.len(), 1); + assert!(applied.suppressed.is_empty()); + if !confirmed { + assert_eq!(applied.unresolved_indices, vec![0]); } } } diff --git a/src/review.rs b/src/review.rs index 87499ba..7136007 100644 --- a/src/review.rs +++ b/src/review.rs @@ -2540,7 +2540,13 @@ async fn review_diff_at( diff_snapshot.as_str(), &diff_receipt, receipt, - ) { + ).and_then(|application| { + anyhow::ensure!( + application.invalid_refutation_indices.is_empty(), + "refuted adjudication must cite candidate-specific contradictory evidence" + ); + Ok(application) + }) { Ok(application) => { lockfile_platform_policy_allowed = true; application @@ -3504,6 +3510,7 @@ fn preserve_unadjudicated_findings( kept_indices: (0..findings.len()).collect(), kept: findings, unresolved_indices: Vec::new(), + invalid_refutation_indices: Vec::new(), resolved_indices: Vec::new(), suppressed: Vec::new(), } @@ -4777,6 +4784,7 @@ mod tests { ], kept_indices: vec![0, 1, 2], unresolved_indices: vec![0, 1, 2], + invalid_refutation_indices: Vec::new(), resolved_indices: vec![], suppressed: vec![], }; @@ -4847,6 +4855,7 @@ mod tests { ], kept_indices: vec![0, 1, 2, 3], unresolved_indices: vec![0, 1, 2, 3], + invalid_refutation_indices: Vec::new(), resolved_indices: Vec::new(), suppressed: Vec::new(), }; @@ -5008,6 +5017,7 @@ mod tests { kept: vec![ordinary.clone()], kept_indices: vec![1], unresolved_indices: Vec::new(), + invalid_refutation_indices: Vec::new(), resolved_indices: vec![0], suppressed: vec![SuppressedFinding { finding: rejected.clone(), diff --git a/tests/e2e.rs b/tests/e2e.rs index 7e13a00..f945971 100644 --- a/tests/e2e.rs +++ b/tests/e2e.rs @@ -6444,15 +6444,28 @@ async fn compact_pnpm_lockfile_ia32_platform_claim_is_suppressed_but_dependency_ #[tokio::test] async fn compact_lockfile_platform_claim_survives_failed_adjudication() { - let server = MockServer::start().await; - Mock::given(method("POST")) - .and(path("/chat/completions")) - .and(body_string_contains("single finding adjudicator")) - .respond_with(ResponseTemplate::new(503)) - .with_priority(1) - .mount(&server) - .await; - Mock::given(method("POST")) + for refuted in [false, true] { + let server = MockServer::start().await; + if !refuted { + Mock::given(method("POST")) + .and(path("/chat/completions")) + .and(body_string_contains("single finding adjudicator")) + .respond_with(ResponseTemplate::new(503)) + .with_priority(1) + .mount(&server) + .await; + } + if refuted { + Mock::given(method("POST")) + .and(path("/chat/completions")) + .and(body_string_contains("single finding adjudicator")) + .respond_with(AllRefutedAdjudicator) + .with_priority(1) + .expect(1) + .mount(&server) + .await; + } + Mock::given(method("POST")) .and(path("/chat/completions")) .respond_with(|request: &Request| { let evidence = prompt_evidence( @@ -6476,48 +6489,63 @@ async fn compact_lockfile_platform_claim_survives_failed_adjudication() { .mount(&server) .await; - let dir = tempfile::tempdir().unwrap(); - let diff = dir.path().join("pnpm-ia32-adjudication-failure.diff"); - std::fs::write( + let dir = tempfile::tempdir().unwrap(); + let diff = dir.path().join("pnpm-ia32-adjudication-failure.diff"); + std::fs::write( &diff, "diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml\n--- a/pnpm-lock.yaml\n+++ b/pnpm-lock.yaml\n@@ -1,3 +1,3 @@\n lockfileVersion: '9.0'\n packages:\n- '@rollup/rollup-win32-ia32-msvc@4.34.8':\n+ '@rollup/rollup-win32-x64-msvc@4.34.8':\n", ) .unwrap(); - let out = postil() - .current_dir(dir.path()) - .env("POSTIL_API_BASE", server.uri()) - .env("POSTIL_DISABLE_SCORER", "1") - .args(["review", "--diff-file"]) - .arg(&diff) - .args(["--output", "json"]) - .assert() - .code(1); - let envelope: Value = serde_json::from_slice(&out.get_output().stdout).unwrap(); - assert!( - envelope["findings"] - .as_array() - .unwrap() - .iter() - .any(|finding| { finding["title"] == "Preserve Windows IA32 support" }) - ); - assert!( - envelope["findings"] - .as_array() - .unwrap() - .iter() - .any(|finding| { finding["path"] == ".postil/provider" }) - ); - assert!( - envelope["suppressedFindings"] - .as_array() - .is_none_or(|findings| { - !findings - .iter() - .any(|finding| finding["reason"] == "lockfilePlatformEvidenceInsufficient") - }) - ); - assert_eq!(envelope["gate"]["failing"], true); + let out = postil() + .current_dir(dir.path()) + .env("POSTIL_API_BASE", server.uri()) + .env("REVIEW_SCORER_MODEL", "scorer-model") + .args(["review", "--diff-file"]) + .arg(&diff) + .args(["--output", "json"]) + .assert() + .code(1); + let envelope: Value = serde_json::from_slice(&out.get_output().stdout).unwrap(); + assert!( + envelope["findings"] + .as_array() + .unwrap() + .iter() + .any(|finding| { finding["title"] == "Preserve Windows IA32 support" }) + ); + assert!( + envelope["findings"] + .as_array() + .unwrap() + .iter() + .any(|finding| { + finding["path"] + == if refuted { + ".postil/model-output" + } else { + ".postil/provider" + } + }) + ); + assert!( + envelope["suppressedFindings"] + .as_array() + .is_none_or(|findings| { + !findings + .iter() + .any(|finding| finding["reason"] == "lockfilePlatformEvidenceInsufficient") + }) + ); + assert_eq!(envelope["gate"]["failing"], true); + let requests = server.received_requests().await.unwrap(); + assert!( + requests.iter().all(|request| !request_system_contains( + request, + "independent second-model scorer" + )) + ); + } } async fn mock_review_model(server: &MockServer, model: &str, findings: Value) { @@ -13562,50 +13590,97 @@ async fn cross_file_package_existence_refutes_false_absence_claim() { #[tokio::test] async fn fresh_unresolved_repository_claims_are_suppressed() { - for (name, repository, resources, state) in [ - ( - "unavailable", - false, - vec!["widget".to_string()], - "unavailable", - ), - ("exhausted", true, vec!["widget".to_string()], "exhausted"), - ] { - let server = MockServer::start().await; - mock_review( - &server, - json!([{ - "path": "src/auth.rs", "line": 42, "severity": "error", "kind": "risk", - "confidence": 0.99, "title": "Widget dependency is absent", - "body": "The repository does not contain the required widget dependency.", - "evidence": "exec_query(&token);", - "repositoryContext": {"claim": "absence", "resources": resources} - }]), - ) - .await; - let dir = tempfile::tempdir().unwrap(); - if repository { - initialize_staged_repository(dir.path()); - } - let diff = write_diff(dir.path()); - let mut command = postil(); - command - .current_dir(dir.path()) - .env("POSTIL_API_BASE", server.uri()) - .env("POSTIL_DISABLE_SCORER", "1") - .arg("review"); - if repository { - command.arg("--staged"); - } else { - command.arg("--diff-file").arg(&diff); + for refuted in [false, true] { + for (name, repository, resources, state) in [ + ( + "unavailable", + false, + vec!["widget".to_string()], + "unavailable", + ), + ("exhausted", true, vec!["widget".to_string()], "exhausted"), + ] { + let server = MockServer::start().await; + mock_review( + &server, + json!([{ + "path": "src/auth.rs", "line": 42, "severity": "error", "kind": "risk", + "confidence": 0.99, "title": "Widget dependency is absent", + "body": "The repository does not contain the required widget dependency.", + "evidence": "exec_query(&token);", + "repositoryContext": {"claim": "absence", "resources": resources} + }]), + ) + .await; + if refuted { + Mock::given(method("POST")) + .and(path("/chat/completions")) + .and(body_string_contains("single finding adjudicator")) + .respond_with(AllRefutedAdjudicator) + .with_priority(1) + .expect(1) + .mount(&server) + .await; + } + let dir = tempfile::tempdir().unwrap(); + if repository { + initialize_staged_repository(dir.path()); + } + let diff = write_diff(dir.path()); + let mut command = postil(); + command + .current_dir(dir.path()) + .env("POSTIL_API_BASE", server.uri()) + .env("REVIEW_SCORER_MODEL", "scorer-model") + .arg("review"); + if repository { + command.arg("--staged"); + } else { + command.arg("--diff-file").arg(&diff); + } + command.args(["--output", "json"]); + let out = command.assert().code(if refuted { 1 } else { 0 }); + let envelope: Value = serde_json::from_slice(&out.get_output().stdout).unwrap(); + assert_eq!(envelope["repositorySearch"]["state"], state, "{name}"); + if refuted { + let findings = envelope["findings"].as_array().unwrap(); + assert!( + findings + .iter() + .any(|finding| finding["path"] == ".postil/model-output") + ); + assert!( + findings + .iter() + .any(|finding| finding["title"] == "Widget dependency is absent") + ); + assert_eq!(envelope["counts"]["suppressed"], 0); + assert_eq!(envelope["gate"]["failing"], true); + assert!( + envelope["modelIncidents"] + .as_array() + .unwrap() + .iter() + .any(|incident| incident["category"] == "invalidOutput" + && incident["recovered"] == false) + ); + } else { + assert_eq!(envelope["counts"]["suppressed"], 1, "{name}"); + assert_eq!(envelope["findings"], json!([]), "{name}"); + assert_eq!(envelope["gate"]["failing"], false, "{name}"); + assert!( + envelope["modelIncidents"] + .as_array() + .is_none_or(Vec::is_empty) + ); + } + let requests = server.received_requests().await.unwrap(); + assert!(requests.iter().all(|request| !request_system_contains( + request, + "independent second-model scorer" + ))); + assert_eq!(envelope["resolved"], json!([])); } - command.args(["--output", "json"]); - let out = command.assert().code(0); - let envelope: Value = serde_json::from_slice(&out.get_output().stdout).unwrap(); - assert_eq!(envelope["repositorySearch"]["state"], state, "{name}"); - assert_eq!(envelope["counts"]["suppressed"], 1, "{name}"); - assert_eq!(envelope["findings"], json!([]), "{name}"); - assert_eq!(envelope["gate"]["failing"], false, "{name}"); } } @@ -13801,50 +13876,114 @@ async fn oversized_adjudication_payload_preserves_findings_without_aborting_revi #[tokio::test] async fn scorer_cannot_suppress_an_unresolved_full_rereview_baseline() { - let server = MockServer::start().await; - mock_review(&server, json!([])).await; - let directory = tempfile::tempdir().unwrap(); - let diff = write_diff(directory.path()); - let baseline = json!({ - "version": 1, "summary": "", "silent": false, - "findings": [{ - "path": "src/auth.rs", "line": 42, "severity": "error", "kind": "risk", - "confidence": 0.9, "title": "Authorization guard remains bypassed", - "body": "The authorization guard remains bypassed before query execution.", - "evidence": "exec_query(&token);" - }], - "resolved": [], "counts": {"info": 0, "warn": 0, "error": 1, "suppressed": 0}, - "confidenceBuckets": [0,0,0,0,1], - "gate": {"failOn": "error", "failing": true}, - "modelUsed": "model", "usage": {"promptTokens": 0, "completionTokens": 0}, - "baseSha": null, "headSha": null, "sinceSha": null - }); - let baseline_path = directory.path().join("scorer-baseline.json"); - std::fs::write(&baseline_path, baseline.to_string()).unwrap(); - - let output = postil() - .current_dir(directory.path()) - .env("POSTIL_API_BASE", server.uri()) - .env("REVIEW_SCORER_MODEL", "scorer-model") - .args(["review", "--diff-file"]) - .arg(&diff) - .arg("--baseline") - .arg(&baseline_path) - .args(["--output", "json"]) - .assert() - .code(1); - let envelope: Value = serde_json::from_slice(&output.get_output().stdout).unwrap(); - assert_eq!( - envelope["findings"][0]["title"], - "Authorization guard remains bypassed" - ); - assert_eq!(envelope["findings"][0]["confidence"], 0.9); - assert_eq!(envelope["resolved"], json!([])); - assert_eq!(envelope["gate"]["failing"], true); - let requests = server.received_requests().await.unwrap(); - assert!(requests.iter().all(|request| { - !String::from_utf8_lossy(&request.body).contains("independent second-model scorer") - })); + for status in ["unresolved", "refuted"] { + for claim in ["none", "absence"] { + let server = MockServer::start().await; + mock_review(&server, json!([])).await; + Mock::given(method("POST")) + .and(path("/chat/completions")) + .and(body_string_contains("single finding adjudicator")) + .respond_with(move |request: &Request| { + let request: Value = request.body_json().unwrap(); + let payload: Value = serde_json::from_str( + request["messages"].as_array().unwrap().last().unwrap()["content"] + .as_str() + .unwrap(), + ) + .unwrap(); + let results = payload["candidates"] + .as_array() + .unwrap() + .iter() + .map(|candidate| { + json!({ + "candidateId": candidate["candidateId"], "status": status, + "revisedTitle": "", "revisedBody": "", + "evidence": if status == "refuted" { "+exec_query(&token);" } else { "" }, "duplicateOf": null + }) + }) + .collect::>(); + ResponseTemplate::new(200).set_body_json(scorer_content(json!(results))) + }) + .with_priority(1) + .expect(1) + .mount(&server) + .await; + let directory = tempfile::tempdir().unwrap(); + let diff = write_diff(directory.path()); + let baseline = json!({ + "version": 1, "summary": "", "silent": false, + "findings": [{ + "path": "src/auth.rs", "line": 42, "severity": "error", "kind": "risk", + "confidence": 0.9, "title": "Authorization guard remains bypassed", + "body": "The authorization guard remains bypassed before query execution.", + "evidence": "exec_query(&token);", + "repositoryContext": if claim == "none" { Value::Null } else { json!({"claim": claim, "identifiers": ["validate_token"]}) } + }], + "resolved": [], "counts": {"info": 0, "warn": 0, "error": 1, "suppressed": 0}, + "confidenceBuckets": [0,0,0,0,1], + "gate": {"failOn": "error", "failing": true}, + "modelUsed": "model", "usage": {"promptTokens": 0, "completionTokens": 0}, + "baseSha": null, "headSha": null, "sinceSha": null + }); + let baseline_path = directory.path().join("scorer-baseline.json"); + std::fs::write(&baseline_path, baseline.to_string()).unwrap(); + + let output = postil() + .current_dir(directory.path()) + .env("POSTIL_API_BASE", server.uri()) + .env("REVIEW_SCORER_MODEL", "scorer-model") + .args(["review", "--diff-file"]) + .arg(&diff) + .arg("--baseline") + .arg(&baseline_path) + .args(["--output", "json"]) + .assert() + .code(1); + let envelope: Value = serde_json::from_slice(&output.get_output().stdout).unwrap(); + assert_eq!( + envelope["findings"][0]["title"], + "Authorization guard remains bypassed" + ); + assert_eq!(envelope["findings"][0]["confidence"], 0.9); + assert_eq!(envelope["resolved"], json!([])); + assert_eq!(envelope["gate"]["failing"], true); + assert_eq!( + envelope["findings"].as_array().unwrap().len(), + if status == "refuted" { 2 } else { 1 } + ); + assert_eq!(envelope["counts"]["suppressed"], 0); + if status == "refuted" { + assert!( + envelope["findings"] + .as_array() + .unwrap() + .iter() + .any(|finding| finding["path"] == ".postil/model-output") + ); + assert!( + envelope["modelIncidents"] + .as_array() + .unwrap() + .iter() + .any(|incident| incident["category"] == "invalidOutput" + && incident["recovered"] == false) + ); + } else { + assert!( + envelope["modelIncidents"] + .as_array() + .is_none_or(Vec::is_empty) + ); + } + assert_eq!(envelope["usageAccountingComplete"], true); + assert_model_usage_matches_aggregate(&envelope); + let requests = server.received_requests().await.unwrap(); + assert!(requests.iter().all(|request| { + !String::from_utf8_lossy(&request.body).contains("independent second-model scorer") + })); + } + } } #[tokio::test]