Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -254,8 +254,8 @@ Auto-detects: TypeScript/JavaScript/Dart, React/Vue/Svelte/Flutter, Vite/Webpack
| Flag | Description |
|------|-------------|
| `--skip-tests` | Skip test execution (fast mode) |
| `--ci` | CI mode (exit 1 if score < 60) |
| `--fail-under N` | Exit 1 if score < N |
| `--ci` | CI mode (exit 1 if score < `failUnder` from config, else 60) |
| `--fail-under N` | Exit 1 if score < N; overrides config and `--ci` |
| `--json` | JSON output |
| `--badge` | Generate SVG badge |
| `--sarif` | SARIF for GitHub Code Scanning |
Expand Down
55 changes: 53 additions & 2 deletions src/cli.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -286,14 +286,64 @@ describe("config file", () => {
}, 30_000);
});

describe("--ci honours config failUnder (#108)", () => {
function scan(args: string): { status: number; out: string } {
try {
const out = execSync(`node ${CLI} ${args}`, {
encoding: "utf-8",
timeout: 30_000,
cwd: TMP,
env: { ...process.env, VCQA_NO_UPDATE_CHECK: "1" },
});
return { status: 0, out };
} catch (e: any) {
return { status: e.status ?? -1, out: `${e.stdout ?? ""}${e.stderr ?? ""}` };
}
}

it("fails under --ci when the score is below a config failUnder above 60", () => {
writeFileSync(join(TMP, ".vcqa.json"), JSON.stringify({ failUnder: 100 }));
const { status, out } = scan("--ci --skip-tests .");
const score = Number(/Failing: score (\d+) < 100 \(config\)/.exec(out)?.[1]);
expect(status).toBe(1);
expect(score).toBeGreaterThan(60); // the old CI default of 60 would have passed it
}, 30_000);

it("passes under --ci when a config failUnder below 60 clears a score below 60", () => {
// A project that scores under 60: the old CI default would fail it (exit 1).
const fn = (i: number) =>
`export function f${i}(a: any, b: any): any {\n try { eval("x" + a); } catch (e) {}\n // @ts-ignore\n if (a) { if (b) { for (const x of a) { if (x) { while (b) { if ((a && b) || x) { return a as any; } } } } } }\n return JSON.parse(b);\n}\n`;
const body = Array.from({ length: 40 }, (_, i) => fn(i)).join("");
for (let j = 0; j < 6; j++) writeFileSync(join(TMP, "src", `m${j}.ts`), body);
writeFileSync(join(TMP, ".vcqa.json"), JSON.stringify({ failUnder: 1 }));
const { status, out } = scan("--ci --skip-tests .");
const score = Number(/Passing: score (\d+) \u2265 1 \(config\)/.exec(out)?.[1]);
expect(status).toBe(0);
expect(score).toBeLessThan(60); // the old CI default of 60 would have failed it
}, 30_000);

it("--fail-under overrides config under --ci", () => {
writeFileSync(join(TMP, ".vcqa.json"), JSON.stringify({ failUnder: 100 }));
const { status, out } = scan("--ci --skip-tests --fail-under 1 .");
expect(status).toBe(0);
expect(out).toMatch(/Passing: score \d+ \u2265 1 \(flag\)/);
}, 30_000);

it("uses the CI default of 60 when neither flag nor config sets one", () => {
const { out } = scan("--ci --skip-tests .");
expect(out).toMatch(/(Passing: score \d+ \u2265 60|Failing: score \d+ < 60) \(ci default\)/);
}, 30_000);
});

describe("init command", () => {
it("creates workflow file", () => {
const out = run("init .");
expect(out).toContain("vibecodeqa.yml");
expect(existsSync(join(TMP, ".github", "workflows", "vibecodeqa.yml"))).toBe(true);
const workflow = readFileSync(join(TMP, ".github", "workflows", "vibecodeqa.yml"), "utf-8");
expect(workflow).toContain("pull_request");
expect(workflow).toContain("--fail-under");
expect(workflow).toContain("--ci");
expect(workflow).not.toContain("--fail-under"); // the gate lives in .vcqa.json (#108)
});

it("does not overwrite existing workflow", () => {
Expand Down Expand Up @@ -333,11 +383,12 @@ describe("init command", () => {
expect(Object.keys(config.checks)).toContain("confusion");
expect(Object.keys(config.checks)).toContain("context");
expect(Object.keys(config.checks).length).toBe(Object.keys(CHECK_META).length);
// The CI gate the scaffolded workflow enforces lives here, not in a flag (#108)
expect(config.failUnder).toBe(70);
// Should have help fields
expect(config._comment).toContain("vibecodeqa.online");
expect(config._checks_help).toContain("enabled");
expect(config._ignore_help).toContain("vendor");
expect(config.failUnder).toBe(60);
});

it("does not overwrite existing .vcqa.json", () => {
Expand Down
18 changes: 13 additions & 5 deletions src/cli.ts
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ import { loadConfig } from "./config.js";
import { scan } from "./core.js";
import { computeDelta } from "./delta.js";
import { detectStack, detectWorkspace } from "./detect.js";
import { resolveFailUnder } from "./fail-under.js";
import { isPartialScan } from "./history.js";
import { postPRComment } from "./pr-comment.js";
import { generatePages } from "./report/html.js";
Expand Down Expand Up @@ -124,8 +125,8 @@ function printHelp(): void {

\x1b[1mFlags:\x1b[0m
--skip-tests Skip test execution (faster scan)
--ci CI mode (exit 1 if score < 60)
--fail-under N Exit 1 if score below N (e.g. --fail-under 80)
--ci CI mode (exit 1 if score < failUnder from config, else 60)
--fail-under N Exit 1 if score below N; overrides config and --ci (e.g. --fail-under 80)
--json Output JSON only (no terminal UI)
--badge Generate SVG badge
--sarif Generate SARIF for GitHub Code Scanning
Expand Down Expand Up @@ -706,11 +707,18 @@ async function main() {
}
}

const failUnder = flags.failUnder ?? (ciMode ? 60 : (config.failUnder ?? 0));
if (failUnder > 0 && score < failUnder && !watchMode) {
if (!quietMode) console.log(` \x1b[31mFailing: score ${score} < ${failUnder}\x1b[0m\n`);
const gate = resolveFailUnder(flags.failUnder, config.failUnder, ciMode);
if (gate.threshold > 0 && score < gate.threshold && !watchMode) {
if (!quietMode) console.log(` \x1b[31mFailing: score ${score} < ${gate.threshold} (${gate.source})\x1b[0m\n`);
process.exit(1);
}
if (ciMode && !quietMode && !watchMode) {
console.log(
gate.threshold > 0
? ` \x1b[32mPassing: score ${score} \u2265 ${gate.threshold} (${gate.source})\x1b[0m\n`
: ` \x1b[2mNo score gate: failUnder is 0 (${gate.source})\x1b[0m\n`,
);
}

if (!quietMode && !ciMode && !watchMode && !process.env.VCQA_NO_UPDATE_CHECK) {
checkForUpdate(VERSION).catch(() => {}); // ok — non-blocking, best-effort
Expand Down
6 changes: 3 additions & 3 deletions src/commands/init.ts
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ jobs:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- run: npx @vibecodeqa/cli --ci --fail-under 70 --sarif --badge
- run: npx @vibecodeqa/cli --ci --sarif --badge
- uses: github/codeql-action/upload-sarif@v3
if: always()
with:
Expand Down Expand Up @@ -85,8 +85,8 @@ jobs:
_checks_help: 'Set { "enabled": false } to disable. Add "ignore": ["generated/**"] to skip files per-check.',
ignore: [],
_ignore_help: 'Global file patterns to skip: ["vendor/**", "*.generated.ts", "proto/**"]',
failUnder: 60,
_failUnder_help: "Exit with code 1 if score below this. Overridden by --fail-under flag.",
failUnder: 70,
_failUnder_help: "Exit with code 1 if score below this, including under --ci. Overridden by --fail-under flag.",
};
writeFileSync(vcqaConfigPath, `${JSON.stringify(config, null, 2)}\n`);
console.log(` \x1b[32m+\x1b[0m .vcqa.json`);
Expand Down
27 changes: 27 additions & 0 deletions src/fail-under.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
import { describe, expect, it } from "vitest";
import { resolveFailUnder } from "./fail-under.js";

describe("resolveFailUnder (#108)", () => {
it("honours config failUnder under --ci instead of the CI default", () => {
expect(resolveFailUnder(null, 80, true)).toEqual({ threshold: 80, source: "config" });
expect(resolveFailUnder(null, 40, true)).toEqual({ threshold: 40, source: "config" });
});

it("lets --fail-under override both config and the CI default", () => {
expect(resolveFailUnder(90, 40, true)).toEqual({ threshold: 90, source: "flag" });
expect(resolveFailUnder(30, 80, false)).toEqual({ threshold: 30, source: "flag" });
expect(resolveFailUnder(0, 80, true)).toEqual({ threshold: 0, source: "flag" });
});

it("falls back to 60 under --ci only when config sets nothing", () => {
expect(resolveFailUnder(null, undefined, true)).toEqual({ threshold: 60, source: "ci default" });
});

it("does not gate outside CI with no flag and no config", () => {
expect(resolveFailUnder(null, undefined, false)).toEqual({ threshold: 0, source: "none" });
});

it("treats a config failUnder of 0 as an explicit choice", () => {
expect(resolveFailUnder(null, 0, true)).toEqual({ threshold: 0, source: "config" });
});
});
22 changes: 22 additions & 0 deletions src/fail-under.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
/** Resolve the exit threshold for a scan and say where it came from.
*
* Precedence: `--fail-under` flag → config `failUnder` (.vcqa.json /
* package.json#vcqa) → `--ci` default of 60 → 0 (no gate). A config value
* is honoured under `--ci`: the CI default only applies when the project has
* not chosen its own threshold (#108). */

export type FailUnderSource = "flag" | "config" | "ci default" | "none";

export interface FailUnder {
threshold: number;
source: FailUnderSource;
}

export const CI_DEFAULT_FAIL_UNDER = 60;

export function resolveFailUnder(flag: number | null | undefined, configValue: number | undefined, ciMode: boolean): FailUnder {
if (flag != null) return { threshold: flag, source: "flag" };
if (configValue != null) return { threshold: configValue, source: "config" };
if (ciMode) return { threshold: CI_DEFAULT_FAIL_UNDER, source: "ci default" };
return { threshold: 0, source: "none" };
}