diff --git a/README.md b/README.md index 4357a9a..78e03d6 100644 --- a/README.md +++ b/README.md @@ -254,8 +254,8 @@ Auto-detects: TypeScript/JavaScript/Dart, React/Vue/Svelte/Flutter, Vite/Webpack | Flag | Description | |------|-------------| | `--skip-tests` | Skip test execution (fast mode) | -| `--ci` | CI mode (exit 1 if score < 60) | -| `--fail-under N` | Exit 1 if score < N | +| `--ci` | CI mode (exit 1 if score < `failUnder` from config, else 60) | +| `--fail-under N` | Exit 1 if score < N; overrides config and `--ci` | | `--json` | JSON output | | `--badge` | Generate SVG badge | | `--sarif` | SARIF for GitHub Code Scanning | diff --git a/src/cli.test.ts b/src/cli.test.ts index 7382fbd..66df006 100644 --- a/src/cli.test.ts +++ b/src/cli.test.ts @@ -286,6 +286,55 @@ describe("config file", () => { }, 30_000); }); +describe("--ci honours config failUnder (#108)", () => { + function scan(args: string): { status: number; out: string } { + try { + const out = execSync(`node ${CLI} ${args}`, { + encoding: "utf-8", + timeout: 30_000, + cwd: TMP, + env: { ...process.env, VCQA_NO_UPDATE_CHECK: "1" }, + }); + return { status: 0, out }; + } catch (e: any) { + return { status: e.status ?? -1, out: `${e.stdout ?? ""}${e.stderr ?? ""}` }; + } + } + + it("fails under --ci when the score is below a config failUnder above 60", () => { + writeFileSync(join(TMP, ".vcqa.json"), JSON.stringify({ failUnder: 100 })); + const { status, out } = scan("--ci --skip-tests ."); + const score = Number(/Failing: score (\d+) < 100 \(config\)/.exec(out)?.[1]); + expect(status).toBe(1); + expect(score).toBeGreaterThan(60); // the old CI default of 60 would have passed it + }, 30_000); + + it("passes under --ci when a config failUnder below 60 clears a score below 60", () => { + // A project that scores under 60: the old CI default would fail it (exit 1). + const fn = (i: number) => + `export function f${i}(a: any, b: any): any {\n try { eval("x" + a); } catch (e) {}\n // @ts-ignore\n if (a) { if (b) { for (const x of a) { if (x) { while (b) { if ((a && b) || x) { return a as any; } } } } } }\n return JSON.parse(b);\n}\n`; + const body = Array.from({ length: 40 }, (_, i) => fn(i)).join(""); + for (let j = 0; j < 6; j++) writeFileSync(join(TMP, "src", `m${j}.ts`), body); + writeFileSync(join(TMP, ".vcqa.json"), JSON.stringify({ failUnder: 1 })); + const { status, out } = scan("--ci --skip-tests ."); + const score = Number(/Passing: score (\d+) \u2265 1 \(config\)/.exec(out)?.[1]); + expect(status).toBe(0); + expect(score).toBeLessThan(60); // the old CI default of 60 would have failed it + }, 30_000); + + it("--fail-under overrides config under --ci", () => { + writeFileSync(join(TMP, ".vcqa.json"), JSON.stringify({ failUnder: 100 })); + const { status, out } = scan("--ci --skip-tests --fail-under 1 ."); + expect(status).toBe(0); + expect(out).toMatch(/Passing: score \d+ \u2265 1 \(flag\)/); + }, 30_000); + + it("uses the CI default of 60 when neither flag nor config sets one", () => { + const { out } = scan("--ci --skip-tests ."); + expect(out).toMatch(/(Passing: score \d+ \u2265 60|Failing: score \d+ < 60) \(ci default\)/); + }, 30_000); +}); + describe("init command", () => { it("creates workflow file", () => { const out = run("init ."); @@ -293,7 +342,8 @@ describe("init command", () => { expect(existsSync(join(TMP, ".github", "workflows", "vibecodeqa.yml"))).toBe(true); const workflow = readFileSync(join(TMP, ".github", "workflows", "vibecodeqa.yml"), "utf-8"); expect(workflow).toContain("pull_request"); - expect(workflow).toContain("--fail-under"); + expect(workflow).toContain("--ci"); + expect(workflow).not.toContain("--fail-under"); // the gate lives in .vcqa.json (#108) }); it("does not overwrite existing workflow", () => { @@ -333,11 +383,12 @@ describe("init command", () => { expect(Object.keys(config.checks)).toContain("confusion"); expect(Object.keys(config.checks)).toContain("context"); expect(Object.keys(config.checks).length).toBe(Object.keys(CHECK_META).length); + // The CI gate the scaffolded workflow enforces lives here, not in a flag (#108) + expect(config.failUnder).toBe(70); // Should have help fields expect(config._comment).toContain("vibecodeqa.online"); expect(config._checks_help).toContain("enabled"); expect(config._ignore_help).toContain("vendor"); - expect(config.failUnder).toBe(60); }); it("does not overwrite existing .vcqa.json", () => { diff --git a/src/cli.ts b/src/cli.ts index f43e493..f309e7c 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -14,6 +14,7 @@ import { loadConfig } from "./config.js"; import { scan } from "./core.js"; import { computeDelta } from "./delta.js"; import { detectStack, detectWorkspace } from "./detect.js"; +import { resolveFailUnder } from "./fail-under.js"; import { isPartialScan } from "./history.js"; import { postPRComment } from "./pr-comment.js"; import { generatePages } from "./report/html.js"; @@ -124,8 +125,8 @@ function printHelp(): void { \x1b[1mFlags:\x1b[0m --skip-tests Skip test execution (faster scan) - --ci CI mode (exit 1 if score < 60) - --fail-under N Exit 1 if score below N (e.g. --fail-under 80) + --ci CI mode (exit 1 if score < failUnder from config, else 60) + --fail-under N Exit 1 if score below N; overrides config and --ci (e.g. --fail-under 80) --json Output JSON only (no terminal UI) --badge Generate SVG badge --sarif Generate SARIF for GitHub Code Scanning @@ -706,11 +707,18 @@ async function main() { } } - const failUnder = flags.failUnder ?? (ciMode ? 60 : (config.failUnder ?? 0)); - if (failUnder > 0 && score < failUnder && !watchMode) { - if (!quietMode) console.log(` \x1b[31mFailing: score ${score} < ${failUnder}\x1b[0m\n`); + const gate = resolveFailUnder(flags.failUnder, config.failUnder, ciMode); + if (gate.threshold > 0 && score < gate.threshold && !watchMode) { + if (!quietMode) console.log(` \x1b[31mFailing: score ${score} < ${gate.threshold} (${gate.source})\x1b[0m\n`); process.exit(1); } + if (ciMode && !quietMode && !watchMode) { + console.log( + gate.threshold > 0 + ? ` \x1b[32mPassing: score ${score} \u2265 ${gate.threshold} (${gate.source})\x1b[0m\n` + : ` \x1b[2mNo score gate: failUnder is 0 (${gate.source})\x1b[0m\n`, + ); + } if (!quietMode && !ciMode && !watchMode && !process.env.VCQA_NO_UPDATE_CHECK) { checkForUpdate(VERSION).catch(() => {}); // ok — non-blocking, best-effort diff --git a/src/commands/init.ts b/src/commands/init.ts index 72d0575..1273730 100644 --- a/src/commands/init.ts +++ b/src/commands/init.ts @@ -32,7 +32,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - run: npx @vibecodeqa/cli --ci --fail-under 70 --sarif --badge + - run: npx @vibecodeqa/cli --ci --sarif --badge - uses: github/codeql-action/upload-sarif@v3 if: always() with: @@ -85,8 +85,8 @@ jobs: _checks_help: 'Set { "enabled": false } to disable. Add "ignore": ["generated/**"] to skip files per-check.', ignore: [], _ignore_help: 'Global file patterns to skip: ["vendor/**", "*.generated.ts", "proto/**"]', - failUnder: 60, - _failUnder_help: "Exit with code 1 if score below this. Overridden by --fail-under flag.", + failUnder: 70, + _failUnder_help: "Exit with code 1 if score below this, including under --ci. Overridden by --fail-under flag.", }; writeFileSync(vcqaConfigPath, `${JSON.stringify(config, null, 2)}\n`); console.log(` \x1b[32m+\x1b[0m .vcqa.json`); diff --git a/src/fail-under.test.ts b/src/fail-under.test.ts new file mode 100644 index 0000000..4fdd32b --- /dev/null +++ b/src/fail-under.test.ts @@ -0,0 +1,27 @@ +import { describe, expect, it } from "vitest"; +import { resolveFailUnder } from "./fail-under.js"; + +describe("resolveFailUnder (#108)", () => { + it("honours config failUnder under --ci instead of the CI default", () => { + expect(resolveFailUnder(null, 80, true)).toEqual({ threshold: 80, source: "config" }); + expect(resolveFailUnder(null, 40, true)).toEqual({ threshold: 40, source: "config" }); + }); + + it("lets --fail-under override both config and the CI default", () => { + expect(resolveFailUnder(90, 40, true)).toEqual({ threshold: 90, source: "flag" }); + expect(resolveFailUnder(30, 80, false)).toEqual({ threshold: 30, source: "flag" }); + expect(resolveFailUnder(0, 80, true)).toEqual({ threshold: 0, source: "flag" }); + }); + + it("falls back to 60 under --ci only when config sets nothing", () => { + expect(resolveFailUnder(null, undefined, true)).toEqual({ threshold: 60, source: "ci default" }); + }); + + it("does not gate outside CI with no flag and no config", () => { + expect(resolveFailUnder(null, undefined, false)).toEqual({ threshold: 0, source: "none" }); + }); + + it("treats a config failUnder of 0 as an explicit choice", () => { + expect(resolveFailUnder(null, 0, true)).toEqual({ threshold: 0, source: "config" }); + }); +}); diff --git a/src/fail-under.ts b/src/fail-under.ts new file mode 100644 index 0000000..cdb2f4f --- /dev/null +++ b/src/fail-under.ts @@ -0,0 +1,22 @@ +/** Resolve the exit threshold for a scan and say where it came from. + * + * Precedence: `--fail-under` flag → config `failUnder` (.vcqa.json / + * package.json#vcqa) → `--ci` default of 60 → 0 (no gate). A config value + * is honoured under `--ci`: the CI default only applies when the project has + * not chosen its own threshold (#108). */ + +export type FailUnderSource = "flag" | "config" | "ci default" | "none"; + +export interface FailUnder { + threshold: number; + source: FailUnderSource; +} + +export const CI_DEFAULT_FAIL_UNDER = 60; + +export function resolveFailUnder(flag: number | null | undefined, configValue: number | undefined, ciMode: boolean): FailUnder { + if (flag != null) return { threshold: flag, source: "flag" }; + if (configValue != null) return { threshold: configValue, source: "config" }; + if (ciMode) return { threshold: CI_DEFAULT_FAIL_UNDER, source: "ci default" }; + return { threshold: 0, source: "none" }; +}