From 1aa953fe8f707889517d4db27993157464d007c8 Mon Sep 17 00:00:00 2001 From: Lloyd Evans Date: Thu, 8 Oct 2026 23:21:02 +0100 Subject: [PATCH] Add changed tests and a shardable gate --- .github/workflows/ci.yml | 19 ++- README.md | 32 +++-- package.json | 3 +- scripts/testing/test-selection.test.mts | 179 ++++++++++++++++++++++++ vitest.config.mts | 10 +- 5 files changed, 224 insertions(+), 19 deletions(-) create mode 100644 scripts/testing/test-selection.test.mts diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index cb92ce2..3968635 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -9,8 +9,13 @@ permissions: contents: read jobs: - verify: + gate: runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + # Add entries here for more shards, e.g. [1, 2, 3, 4]. Indices and total come from the matrix jobs. + shard: [1] steps: - uses: actions/checkout@v7 - uses: actions/setup-node@v7 @@ -18,4 +23,14 @@ jobs: node-version: 24 cache: npm - run: npm ci - - run: npm run verify-gate + - run: npm run verify-gate -- --shard=$((${{ strategy.job-index }} + 1))/${{ strategy.job-total }} + + verify: + needs: gate + if: ${{ always() }} + runs-on: ubuntu-latest + steps: + - name: Require every shard to succeed + env: + GATE_RESULT: ${{ needs.gate.result }} + run: test "$GATE_RESULT" = success diff --git a/README.md b/README.md index 6b301b5..705107f 100644 --- a/README.md +++ b/README.md @@ -12,24 +12,26 @@ See [CONTRIBUTING.md](CONTRIBUTING.md) for contributor conventions. Run `npm install` at the root to install dependencies and link the packages declared in [package.json](package.json). -| Command | Description | -| ---------------------- | ------------------------------------------------------------- | -| `npm start` | Dev server with hot reload (localhost:5173 by default) | -| `npm run build` | Minified production build to `dist/` | -| `npm run preview` | Serve the built site (localhost:4173 by default) | -| `npm run typecheck` | Type-check application, packages and tooling without emitting | -| `npm run lint` | Lint with ESLint (warnings fail) | -| `npm run lint-fix` | Apply ESLint's automatic fixes | -| `npm run format` | Format the repo with Prettier | -| `npm run format-check` | Check formatting without writing | -| `npm test` | Run unit tests once | -| `npm run mutation` | Incremental mutation testing with Stryker | -| `npm run verify-gate` | Run the CI gate locally, without mutation testing | -| `npm run test-watch` | Run unit tests in watch mode | +| Command | Description | +| ---------------------- | ----------------------------------------------------------------------------- | +| `npm start` | Dev server with hot reload (localhost:5173 by default) | +| `npm run build` | Minified production build to `dist/` | +| `npm run preview` | Serve the built site (localhost:4173 by default) | +| `npm run typecheck` | Type-check application, packages and tooling without emitting | +| `npm run lint` | Lint with ESLint (warnings fail) | +| `npm run lint-fix` | Apply ESLint's automatic fixes | +| `npm run format` | Format the repo with Prettier | +| `npm run format-check` | Check formatting without writing | +| `npm test` | Run unit tests once | +| `npm run test-changed` | Run tests affected by changes since `origin/HEAD`, including uncommitted work | +| `npm run mutation` | Incremental mutation testing with Stryker | +| `npm run verify-gate` | Run the CI gate locally, without mutation testing | +| `npm run test-watch` | Run unit tests in watch mode | ## Notes - [index.html](index.html) is the entry page. Vite uses its defaults without a configuration file, including its browser target and no production source maps. -- Vite transpiles TypeScript without type-checking. Run `npm run typecheck` to check types; `npm run verify-gate` includes it before the tests and build. +- Vite transpiles TypeScript without type-checking. Run `npm run typecheck` to check types. `npm run verify-gate` includes it before the build and tests. +- `test-changed` is feedback while working, not a gate. Fetch a current `origin/HEAD` first. An unavailable comparison fails. Selection can miss deleted or renamed files, files read from disk and differences between staged and working content. The gate runs all tests. `npm run verify-gate -- --shard=1/4` runs its other stages in full and only narrows the final test stage. - TypeScript is pinned to the 6.x line. - Mutation testing uses [Stryker's incremental mode](https://stryker-mutator.io/docs/stryker-js/incremental/). Changes outside mutated source and test files can leave reused results stale; run `npm run mutation -- --force` for a fresh full run. CI does this on pushes to the default branch. diff --git a/package.json b/package.json index 98a7609..c831f22 100644 --- a/package.json +++ b/package.json @@ -13,9 +13,10 @@ "format": "npx --no-install prettier . --write", "format-check": "npx --no-install prettier . --check", "test": "npx --no-install vitest run", + "test-changed": "git diff --name-only origin/HEAD...HEAD -- && npm test -- --changed=origin/HEAD", "test-watch": "npx --no-install vitest", "mutation": "npx --no-install stryker run stryker.config.mts --incremental", - "verify-gate": "npm audit --audit-level=critical && npm run lint && npm run format-check && npm run typecheck && npm test && npm run build" + "verify-gate": "npm audit --audit-level=critical && npm run lint && npm run format-check && npm run typecheck && npm run build && npm test --" }, "dependencies": { "@example/greeter": "*" diff --git a/scripts/testing/test-selection.test.mts b/scripts/testing/test-selection.test.mts new file mode 100644 index 0000000..83a7e45 --- /dev/null +++ b/scripts/testing/test-selection.test.mts @@ -0,0 +1,179 @@ +import { spawnSync } from "node:child_process"; +import { copyFileSync, mkdirSync, mkdtempDisposableSync, readFileSync, symlinkSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { dirname, join, relative } from "node:path"; +import { fileURLToPath } from "node:url"; +import { afterAll, beforeAll, expect, test } from "vitest"; +import { configDefaults } from "vitest/config"; +import { createVitest, type Vitest } from "vitest/node"; +import manifest from "../../package.json" with { type: "json" }; + +const project = fileURLToPath(new URL("../../", import.meta.url)); +const directory = mkdtempDisposableSync(join(tmpdir(), "test-selection-")); +const root = directory.path; +const testFiles = [ + "packages/example/src/value.test.ts", + "scripts/tools/tool.test.mts", + "src/first.test.ts", + "src/second.test.ts", +]; +let runner: Vitest; + +function write(path: string, content: string) { + mkdirSync(dirname(join(root, path)), { recursive: true }); + writeFileSync(join(root, path), content); +} + +beforeAll(async () => { + for (const path of ["vitest.config.mts", "scripts/workspace/workspace-packages.mts"]) { + mkdirSync(dirname(join(root, path)), { recursive: true }); + copyFileSync(join(project, path), join(root, path)); + } + symlinkSync(join(project, "node_modules"), join(root, "node_modules"), "junction"); + write("package.json", JSON.stringify({ workspaces: ["packages/*"] })); + write("packages/example/package.json", JSON.stringify({ name: "@fixture/example", exports: { ".": "./src/index.ts" } })); + write("packages/example/src/index.ts", "export const value = 1;"); + write("src/value.ts", "export const value = 1;"); + write("scripts/tools/tool.mts", "export const value = 1;"); + write("README.md", "Unrelated documentation."); + + const imports = ["./index.ts", "./tool.mts", "./value.ts", "@fixture/example"]; + testFiles.forEach((path, index) => + write( + path, + `import { test, expect } from "vitest"; +import { appendFileSync } from "node:fs"; +import { value } from ${JSON.stringify(imports[index])}; +test("uses its dependency", () => { + expect(value).toBe(1); + appendFileSync("executed.log", ${JSON.stringify(`${path}\n`)}); +});`, + ), + ); + + runner = await createVitest("test", { root, watch: false }, { server: { watch: null } }); +}, 15000); + +afterAll(async () => { + try { + await runner.close(); + } finally { + directory.remove(); + } +}); + +test("retains the default full-run triggers", () => + expect(runner.config.forceRerunTriggers).toEqual(expect.arrayContaining(configDefaults.forceRerunTriggers))); + +test.each([ + "package-lock.json", + "eslint.config.mts", + "vitest.config.mts", + "vite.config.ts", + "tsconfig.base.json", + "src/tsconfig.json", + "packages/example/tsconfig.json", + "scripts/tools/tool.mts", + "scripts/workspace/workspace-packages.mts", +])("selects every test when %s changes", async (path) => { + runner.config.related = [join(root, path).replaceAll("\\", "/")]; + const selected = await runner.getRelevantTestSpecifications(); + + expect(selected.map((spec) => relative(root, spec.moduleId).replaceAll("\\", "/")).sort()).toEqual(testFiles); +}); + +test.each([ + { path: "src/value.ts", expected: ["src/first.test.ts"] }, + { path: "packages/example/src/index.ts", expected: ["packages/example/src/value.test.ts", "src/second.test.ts"] }, + { path: "scripts/tools/tool.test.mts", expected: ["scripts/tools/tool.test.mts"] }, + { path: "README.md", expected: [] }, +])("selects only dependants of $path", async ({ path, expected }) => { + runner.config.related = [join(root, path).replaceAll("\\", "/")]; + const selected = await runner.getRelevantTestSpecifications(); + + expect(selected.map((spec) => relative(root, spec.moduleId).replaceAll("\\", "/")).sort()).toEqual(expected); +}); + +function runScript(name: string, args: string[] = []) { + return spawnSync(`npm run ${name} -- ${args.join(" ")}`, { + cwd: root, + shell: true, + encoding: "utf8", + timeout: 30000, + }); +} + +function readLines(path: string) { + return readFileSync(join(root, path), "utf8").trim().split("\n").filter(Boolean); +} + +test("refuses an unresolved changed reference before starting tests", () => { + const missing = "refs/heads/missing-test-selection-reference"; + const script = manifest.scripts["test-changed"].replaceAll("origin/HEAD", missing); + write("test-sentinel.mts", 'console.log("TESTS_STARTED");'); + write("package.json", JSON.stringify({ scripts: { "test-changed": script, test: "node test-sentinel.mts" } })); + expect(runScript("test").stdout).toContain("TESTS_STARTED"); + + const gitDirectory = spawnSync("git", ["rev-parse", "--absolute-git-dir"], { cwd: project, encoding: "utf8" }); + expect(gitDirectory.status, gitDirectory.stderr).toBe(0); + const result = spawnSync("npm run test-changed", { + cwd: root, + shell: true, + encoding: "utf8", + timeout: 15000, + env: { ...process.env, GIT_DIR: gitDirectory.stdout.trim(), GIT_WORK_TREE: project }, + }); + + expect(result.error).toBeUndefined(); + expect(result.status).not.toBe(0); + expect(result.stderr).toContain(missing); + expect(result.stdout).not.toContain("TESTS_STARTED"); +}, 30000); + +test("keeps the gate stages in order and forwards arguments only to tests, covering each file once across shards", () => { + // Record the npm stage boundaries. Execute the real test script in a small fixture. + write( + "record.mts", + `import { appendFileSync } from "node:fs"; +import { spawnSync } from "node:child_process"; +const args = process.argv.slice(2); +appendFileSync("stages.log", args.join(" ") + "\\n"); +if (args[0] === "test") { + const result = spawnSync("npm " + args.join(" "), { shell: true, stdio: "inherit" }); + process.exitCode = result.status ?? 1; +}`, + ); + write( + "package.json", + JSON.stringify({ + workspaces: ["packages/*"], + scripts: { + "verify-gate": manifest.scripts["verify-gate"].replaceAll("npm ", "node record.mts "), + test: manifest.scripts.test, + }, + }), + ); + const stages = ["audit --audit-level=critical", "run lint", "run format-check", "run typecheck", "run build"]; + const run = (args: string[]) => { + write("stages.log", ""); + write("executed.log", ""); + const result = runScript("verify-gate", args); + expect(result.error).toBeUndefined(); + expect(result.status, result.stdout + result.stderr).toBe(0); + expect( + readLines("stages.log").map((line) => + line + .split(" ") + .filter((arg) => arg !== "--") + .join(" "), + ), + ).toEqual([...stages, ["test", ...args].join(" ")]); + + return readLines("executed.log").sort(); + }; + + expect(run([])).toEqual(testFiles); + const shards = [1, 2, 3].map((index) => run([`--shard=${index}/3`])); + expect(shards.every((files) => files.length > 0 && files.length < testFiles.length)).toBe(true); + expect(shards.flat().sort()).toEqual(testFiles); +}, 60000); diff --git a/vitest.config.mts b/vitest.config.mts index a6d47a3..67758e4 100644 --- a/vitest.config.mts +++ b/vitest.config.mts @@ -1,4 +1,4 @@ -import { defineConfig } from "vitest/config"; +import { configDefaults, defineConfig } from "vitest/config"; import { fileURLToPath } from "node:url"; import { readWorkspacePackages } from "./scripts/workspace/workspace-packages.mts"; @@ -15,5 +15,13 @@ export default defineConfig({ }, test: { include: ["src/**/*.test.ts", "packages/*/src/**/*.test.ts", "scripts/**/*.test.mts"], + forceRerunTriggers: [ + ...configDefaults.forceRerunTriggers, + "**/{vitest,vite}.config.*", + "**/package-lock.json", + "**/eslint.config.*", + "**/tsconfig*.json", + "**/scripts/**/!(*.test).mts", + ], }, });