From b4503a70a5c5b93748ea149bc2f0fdf96df89c6f Mon Sep 17 00:00:00 2001 From: Sergey Ivochkin Date: Mon, 5 Oct 2026 14:00:53 +1100 Subject: [PATCH 1/3] fix(ci): honour config failUnder under --ci; print the gate's source (#108) `--ci` resolved the exit threshold as `flag ?? (ci ? 60 : config ?? 0)`, so a `failUnder` in .vcqa.json / package.json#vcqa was ignored in CI: a project asking for 80 passed at 61, one asking for 40 failed at 59. Precedence is now `--fail-under` -> config `failUnder` -> `--ci` default 60 -> 0, resolved in a pure `resolveFailUnder` (src/fail-under.ts). The CI output names the effective threshold and where it came from, e.g. "Passing: score 72 >= 80 (config)" / "Failing: score 70 < 80 (config)". Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01SmtSK7qDTyXhimJ9grap88 --- src/cli.test.ts | 43 ++++++++++++++++++++++++++++++++++++++++++ src/cli.ts | 18 +++++++++++++----- src/fail-under.test.ts | 27 ++++++++++++++++++++++++++ src/fail-under.ts | 22 +++++++++++++++++++++ 4 files changed, 105 insertions(+), 5 deletions(-) create mode 100644 src/fail-under.test.ts create mode 100644 src/fail-under.ts diff --git a/src/cli.test.ts b/src/cli.test.ts index 1289d35..bb61dde 100644 --- a/src/cli.test.ts +++ b/src/cli.test.ts @@ -249,6 +249,49 @@ describe("config file", () => { }, 30_000); }); +describe("--ci honours config failUnder (#108)", () => { + function scan(args: string): { status: number; out: string } { + try { + const out = execSync(`node ${CLI} ${args}`, { + encoding: "utf-8", + timeout: 30_000, + cwd: TMP, + env: { ...process.env, VCQA_NO_UPDATE_CHECK: "1" }, + }); + return { status: 0, out }; + } catch (e: any) { + return { status: e.status ?? -1, out: `${e.stdout ?? ""}${e.stderr ?? ""}` }; + } + } + + it("fails under --ci when the score is below a config failUnder above 60", () => { + writeFileSync(join(TMP, ".vcqa.json"), JSON.stringify({ failUnder: 100 })); + const { status, out } = scan("--ci --skip-tests ."); + const score = Number(/Failing: score (\d+) < 100 \(config\)/.exec(out)?.[1]); + expect(status).toBe(1); + expect(score).toBeGreaterThan(60); // the old CI default of 60 would have passed it + }, 30_000); + + it("passes under --ci when the score clears a config failUnder below 60", () => { + writeFileSync(join(TMP, ".vcqa.json"), JSON.stringify({ failUnder: 1 })); + const { status, out } = scan("--ci --skip-tests ."); + expect(status).toBe(0); + expect(out).toMatch(/Passing: score \d+ \u2265 1 \(config\)/); + }, 30_000); + + it("--fail-under overrides config under --ci", () => { + writeFileSync(join(TMP, ".vcqa.json"), JSON.stringify({ failUnder: 100 })); + const { status, out } = scan("--ci --skip-tests --fail-under 1 ."); + expect(status).toBe(0); + expect(out).toMatch(/Passing: score \d+ \u2265 1 \(flag\)/); + }, 30_000); + + it("uses the CI default of 60 when neither flag nor config sets one", () => { + const { out } = scan("--ci --skip-tests ."); + expect(out).toMatch(/(Passing: score \d+ \u2265 60|Failing: score \d+ < 60) \(ci default\)/); + }, 30_000); +}); + describe("init command", () => { it("creates workflow file", () => { const out = run("init ."); diff --git a/src/cli.ts b/src/cli.ts index ab01e3a..c4ab330 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -13,6 +13,7 @@ import { loadConfig } from "./config.js"; import { scan } from "./core.js"; import { computeDelta } from "./delta.js"; import { detectStack, detectWorkspace } from "./detect.js"; +import { resolveFailUnder } from "./fail-under.js"; import { postPRComment } from "./pr-comment.js"; import { generatePages } from "./report/html.js"; import { buildReportHistorySnapshot, withFreshAnalyzerSnapshots } from "./report-contract.js"; @@ -122,8 +123,8 @@ function printHelp(): void { \x1b[1mFlags:\x1b[0m --skip-tests Skip test execution (faster scan) - --ci CI mode (exit 1 if score < 60) - --fail-under N Exit 1 if score below N (e.g. --fail-under 80) + --ci CI mode (exit 1 if score < failUnder from config, else 60) + --fail-under N Exit 1 if score below N; overrides config and --ci (e.g. --fail-under 80) --json Output JSON only (no terminal UI) --badge Generate SVG badge --sarif Generate SARIF for GitHub Code Scanning @@ -696,11 +697,18 @@ async function main() { } } - const failUnder = flags.failUnder ?? (ciMode ? 60 : (config.failUnder ?? 0)); - if (failUnder > 0 && score < failUnder && !watchMode) { - if (!quietMode) console.log(` \x1b[31mFailing: score ${score} < ${failUnder}\x1b[0m\n`); + const gate = resolveFailUnder(flags.failUnder, config.failUnder, ciMode); + if (gate.threshold > 0 && score < gate.threshold && !watchMode) { + if (!quietMode) console.log(` \x1b[31mFailing: score ${score} < ${gate.threshold} (${gate.source})\x1b[0m\n`); process.exit(1); } + if (ciMode && !quietMode && !watchMode) { + console.log( + gate.threshold > 0 + ? ` \x1b[32mPassing: score ${score} \u2265 ${gate.threshold} (${gate.source})\x1b[0m\n` + : ` \x1b[2mNo score gate: failUnder is 0 (${gate.source})\x1b[0m\n`, + ); + } if (!quietMode && !ciMode && !watchMode && !process.env.VCQA_NO_UPDATE_CHECK) { checkForUpdate(VERSION).catch(() => {}); // ok — non-blocking, best-effort diff --git a/src/fail-under.test.ts b/src/fail-under.test.ts new file mode 100644 index 0000000..4fdd32b --- /dev/null +++ b/src/fail-under.test.ts @@ -0,0 +1,27 @@ +import { describe, expect, it } from "vitest"; +import { resolveFailUnder } from "./fail-under.js"; + +describe("resolveFailUnder (#108)", () => { + it("honours config failUnder under --ci instead of the CI default", () => { + expect(resolveFailUnder(null, 80, true)).toEqual({ threshold: 80, source: "config" }); + expect(resolveFailUnder(null, 40, true)).toEqual({ threshold: 40, source: "config" }); + }); + + it("lets --fail-under override both config and the CI default", () => { + expect(resolveFailUnder(90, 40, true)).toEqual({ threshold: 90, source: "flag" }); + expect(resolveFailUnder(30, 80, false)).toEqual({ threshold: 30, source: "flag" }); + expect(resolveFailUnder(0, 80, true)).toEqual({ threshold: 0, source: "flag" }); + }); + + it("falls back to 60 under --ci only when config sets nothing", () => { + expect(resolveFailUnder(null, undefined, true)).toEqual({ threshold: 60, source: "ci default" }); + }); + + it("does not gate outside CI with no flag and no config", () => { + expect(resolveFailUnder(null, undefined, false)).toEqual({ threshold: 0, source: "none" }); + }); + + it("treats a config failUnder of 0 as an explicit choice", () => { + expect(resolveFailUnder(null, 0, true)).toEqual({ threshold: 0, source: "config" }); + }); +}); diff --git a/src/fail-under.ts b/src/fail-under.ts new file mode 100644 index 0000000..cdb2f4f --- /dev/null +++ b/src/fail-under.ts @@ -0,0 +1,22 @@ +/** Resolve the exit threshold for a scan and say where it came from. + * + * Precedence: `--fail-under` flag → config `failUnder` (.vcqa.json / + * package.json#vcqa) → `--ci` default of 60 → 0 (no gate). A config value + * is honoured under `--ci`: the CI default only applies when the project has + * not chosen its own threshold (#108). */ + +export type FailUnderSource = "flag" | "config" | "ci default" | "none"; + +export interface FailUnder { + threshold: number; + source: FailUnderSource; +} + +export const CI_DEFAULT_FAIL_UNDER = 60; + +export function resolveFailUnder(flag: number | null | undefined, configValue: number | undefined, ciMode: boolean): FailUnder { + if (flag != null) return { threshold: flag, source: "flag" }; + if (configValue != null) return { threshold: configValue, source: "config" }; + if (ciMode) return { threshold: CI_DEFAULT_FAIL_UNDER, source: "ci default" }; + return { threshold: 0, source: "none" }; +} From 5bd4219baf1eeaf86d1092ed1884955805a5955a Mon Sep 17 00:00:00 2001 From: Sergey Ivochkin Date: Mon, 5 Oct 2026 19:22:42 +1100 Subject: [PATCH 2/3] test(ci): prove config failUnder clears a sub-60 score under --ci (#108) The previous low-threshold test scanned a fixture that scores above 60, so it passed under the old CI default too. The fixture now scores below 60 and the test asserts it, so only the new precedence exits 0. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01SmtSK7qDTyXhimJ9grap88 --- src/cli.test.ts | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/src/cli.test.ts b/src/cli.test.ts index bb61dde..90d407f 100644 --- a/src/cli.test.ts +++ b/src/cli.test.ts @@ -272,11 +272,17 @@ describe("--ci honours config failUnder (#108)", () => { expect(score).toBeGreaterThan(60); // the old CI default of 60 would have passed it }, 30_000); - it("passes under --ci when the score clears a config failUnder below 60", () => { + it("passes under --ci when a config failUnder below 60 clears a score below 60", () => { + // A project that scores under 60: the old CI default would fail it (exit 1). + const fn = (i: number) => + `export function f${i}(a: any, b: any): any {\n try { eval("x" + a); } catch (e) {}\n // @ts-ignore\n if (a) { if (b) { for (const x of a) { if (x) { while (b) { if ((a && b) || x) { return a as any; } } } } } }\n return JSON.parse(b);\n}\n`; + const body = Array.from({ length: 40 }, (_, i) => fn(i)).join(""); + for (let j = 0; j < 6; j++) writeFileSync(join(TMP, "src", `m${j}.ts`), body); writeFileSync(join(TMP, ".vcqa.json"), JSON.stringify({ failUnder: 1 })); const { status, out } = scan("--ci --skip-tests ."); + const score = Number(/Passing: score (\d+) \u2265 1 \(config\)/.exec(out)?.[1]); expect(status).toBe(0); - expect(out).toMatch(/Passing: score \d+ \u2265 1 \(config\)/); + expect(score).toBeLessThan(60); // the old CI default of 60 would have failed it }, 30_000); it("--fail-under overrides config under --ci", () => { @@ -299,7 +305,8 @@ describe("init command", () => { expect(existsSync(join(TMP, ".github", "workflows", "vibecodeqa.yml"))).toBe(true); const workflow = readFileSync(join(TMP, ".github", "workflows", "vibecodeqa.yml"), "utf-8"); expect(workflow).toContain("pull_request"); - expect(workflow).toContain("--fail-under"); + expect(workflow).toContain("--ci"); + expect(workflow).not.toContain("--fail-under"); // the gate lives in .vcqa.json (#108) }); it("does not overwrite existing workflow", () => { @@ -339,11 +346,12 @@ describe("init command", () => { expect(Object.keys(config.checks)).toContain("confusion"); expect(Object.keys(config.checks)).toContain("context"); expect(Object.keys(config.checks).length).toBe(Object.keys(CHECK_META).length); + // The CI gate the scaffolded workflow enforces lives here, not in a flag (#108) + expect(config.failUnder).toBe(70); // Should have help fields expect(config._comment).toContain("vibecodeqa.online"); expect(config._checks_help).toContain("enabled"); expect(config._ignore_help).toContain("vendor"); - expect(config.failUnder).toBe(60); }); it("does not overwrite existing .vcqa.json", () => { From 30162186f788b0238ea322acc3ede1e6efcbbca4 Mon Sep 17 00:00:00 2001 From: Sergey Ivochkin Date: Mon, 5 Oct 2026 19:22:42 +1100 Subject: [PATCH 3/3] fix(init): put the scaffolded CI gate in .vcqa.json, not a flag (#108) `vcqa init` wrote a workflow running `--ci --fail-under 70` next to a .vcqa.json with `failUnder: 60`. With the flag winning, the config value was dead and the two disagreed. The workflow now runs `--ci` and the config carries `failUnder: 70`, the gate the workflow used to enforce. README's --ci / --fail-under rows match the new --help text. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01SmtSK7qDTyXhimJ9grap88 --- README.md | 4 ++-- src/commands/init.ts | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 4357a9a..78e03d6 100644 --- a/README.md +++ b/README.md @@ -254,8 +254,8 @@ Auto-detects: TypeScript/JavaScript/Dart, React/Vue/Svelte/Flutter, Vite/Webpack | Flag | Description | |------|-------------| | `--skip-tests` | Skip test execution (fast mode) | -| `--ci` | CI mode (exit 1 if score < 60) | -| `--fail-under N` | Exit 1 if score < N | +| `--ci` | CI mode (exit 1 if score < `failUnder` from config, else 60) | +| `--fail-under N` | Exit 1 if score < N; overrides config and `--ci` | | `--json` | JSON output | | `--badge` | Generate SVG badge | | `--sarif` | SARIF for GitHub Code Scanning | diff --git a/src/commands/init.ts b/src/commands/init.ts index 72d0575..1273730 100644 --- a/src/commands/init.ts +++ b/src/commands/init.ts @@ -32,7 +32,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - run: npx @vibecodeqa/cli --ci --fail-under 70 --sarif --badge + - run: npx @vibecodeqa/cli --ci --sarif --badge - uses: github/codeql-action/upload-sarif@v3 if: always() with: @@ -85,8 +85,8 @@ jobs: _checks_help: 'Set { "enabled": false } to disable. Add "ignore": ["generated/**"] to skip files per-check.', ignore: [], _ignore_help: 'Global file patterns to skip: ["vendor/**", "*.generated.ts", "proto/**"]', - failUnder: 60, - _failUnder_help: "Exit with code 1 if score below this. Overridden by --fail-under flag.", + failUnder: 70, + _failUnder_help: "Exit with code 1 if score below this, including under --ci. Overridden by --fail-under flag.", }; writeFileSync(vcqaConfigPath, `${JSON.stringify(config, null, 2)}\n`); console.log(` \x1b[32m+\x1b[0m .vcqa.json`);