diff --git a/README.md b/README.md index 662af71..846f2ad 100644 --- a/README.md +++ b/README.md @@ -99,6 +99,10 @@ These switches are in no preset: - `modules.automerge: true` adds a workflow that turns on auto-merge for Dependabot's patch and minor updates, so they merge once the checks and reviews the default branch requires have passed; majors stay manual (GitHub only; see [CI and releases](docs/ci-and-releases.md#dependabot-auto-merge)). +- `modules.publish: true` adds a job to the release workflow that publishes each release of a node + package to npm, or of a python package to PyPI, through trusted publishing: no token is stored, and + npm adds provenance. The registry needs a one-time trusted publisher entry (GitHub only; see + [CI and releases](docs/ci-and-releases.md#publishing-to-npm-and-pypi)). You can also start from the file instead of from detection: put a `.repokeeper.yml` you wrote yourself in the repository and run `repokeeper init`. When the file is there and repokeeper has not @@ -128,7 +132,7 @@ Requires Node.js 22.12 or newer. | Guide | What it covers | | --- | --- | -| [CI and releases](docs/ci-and-releases.md) | The workflows repokeeper writes, pinning or mirroring the reusable workflows, `repokeeper eject`, script and Dockerfile linting, release-please, the drift check, update pull requests, code scanning, cache cleanup, Dependabot auto-merge | +| [CI and releases](docs/ci-and-releases.md) | The workflows repokeeper writes, pinning or mirroring the reusable workflows, `repokeeper eject`, script and Dockerfile linting, release-please, publishing to npm and PyPI, the drift check, update pull requests, code scanning, cache cleanup, Dependabot auto-merge | | [Monorepos](docs/monorepos.md) | Stacks in folders: `stack_options..directory`, what runs where, the limits | | [GitHub settings](docs/github-settings.md) | `repokeeper github apply`: description, topics, merge settings and auto-merge, security, branch protection | | [GitLab](docs/gitlab.md) | What differs on GitLab, the image each stack's job runs in, how releases write the version, the two jobs that need a token, and `repokeeper gitlab apply`: merge settings, branch protection, the Renovate schedule | diff --git a/docs/ci-and-releases.md b/docs/ci-and-releases.md index 4e6a08c..46e18ac 100644 --- a/docs/ci-and-releases.md +++ b/docs/ci-and-releases.md @@ -130,6 +130,11 @@ Set `modules.drift: true` to add a `repokeeper` job to `ci.yml` that runs `repok the version that wrote the standard, so a pull request that edits a managed file fails until the edit is resolved. +One edit is not drift: a step of a managed job that uses the same action at another commit or tag, as +a Dependabot pull request leaves it. `check` passes, and `update` keeps the newer version. Any other +change to the job still counts, and so does the ref a job calls a reusable workflow at +(`uses: …/.github/workflows/…@v0`), which repokeeper pins on purpose. + ## Update pull requests Set `modules.autoupdate: true` and repokeeper writes `.github/workflows/repokeeper-autoupdate.yml`: every @@ -266,6 +271,71 @@ merge click and nothing more. On GitLab the switch writes nothing: Renovate opens the updates there and has its own `automerge` setting. It needs repokeeper 0.11.0 or newer. +## Publishing to npm and PyPI + +Set `modules.publish: true` and repokeeper adds a `publish` job to `.github/workflows/release.yml`. When +the release job has created a release (the moment release-please's pull request is merged), the job +checks out the release and publishes it: + +| Released stack | Registry | What the job runs | +| --- | --- | --- | +| `node` | npm | Node.js 24, the stack's install command (`npm ci`, `pnpm install`, ...), then `npm publish --access public` | +| `python` | PyPI | `python -m build`, then [`pypa/gh-action-pypi-publish`](https://github.com/pypa/gh-action-pypi-publish) | + +It uses **trusted publishing**: the job has `id-token: write`, and the registry accepts the short-lived +OIDC token that GitHub gives the run. No npm or PyPI token is stored anywhere. On npm, a package +published this way from a public repository gets a +[provenance statement](https://docs.npmjs.com/generating-provenance-statements) by itself. That is why +the job does not pass `--provenance`: npm makes no provenance for a private repository and would refuse +the flag there. + +The job is part of `release.yml` itself, not a reusable workflow. PyPI does not accept a reusable +workflow as a trusted publisher, and npm checks the name of the calling workflow. So on both registries +the trusted publisher names `release.yml`. + +**Set up once, before the first release** (until then the job fails): + +- **npm:** on npmjs.com, open the package → *Settings* → *Trusted publishing* → *GitHub Actions*, and + enter the owner, the repository and the workflow filename `release.yml`. Leave the environment empty. + Under the allowed actions, allow `npm publish`. A package that is not on npm yet has to be published + once by hand first, because the setting belongs to an existing package. The `repository.url` in + `package.json` has to point at this repository too. +- **PyPI:** on pypi.org, under *Your projects* → *Publishing* (or, for a new project, *Add a pending + publisher*), add a GitHub publisher with the owner, the repository and the workflow `release.yml`, + and leave the environment empty. + +**The trap.** Every field is matched exactly and is case-sensitive, and the registry does not check the +entry when you save it. When the owner, the repository or the workflow filename differs in any way (for +example `release.yaml`, `publish.yml`, or an owner spelled with different capitals), the token is +refused. npm then answers the publish with **404 Not Found** or **403 Forbidden**, which reads like a +missing package or a missing permission rather than a mismatch. If the first publish fails that way, +compare the entry with the repository's URL and `release.yml` letter by letter. PyPI states the reason +more plainly (`invalid-publisher`), but the fix is the same. + +What else to know: + +- The job builds what the package itself builds: npm runs the package's `prepack` and `prepublishOnly` + scripts during `npm publish`, so put the build there. For Python, `python -m build` uses the build + backend in `pyproject.toml`. +- It publishes the stack that is released, which is the first stack with a release type of its own (see + [Releases](#releases)). When that stack is neither `node` nor `python`, the switch writes nothing, and + `init`, `update` and `check` say so. The same happens when `modules.release` is off. +- **Folders are not supported.** release-please names the outputs of a package in a folder after the + folder, and the release workflow does not pass those on. A stack under + `stack_options..directory` is therefore not published, and the switch says so. +- The actions the job uses are pinned to commits, the same ones repokeeper's own workflows use. + Dependabot keeps them up to date like any other action: a newer commit or tag of the same action is + not drift (see [Drift check](#drift-check)). While the job is still as repokeeper wrote it, + `repokeeper update` moves the pins itself. +- No preset turns the module on: not every package is meant to be public, and the registry has to be + set up first. +- A `publish` job you wrote yourself, as repokeeper's own `release.yml` has, is reported as unmanaged. + Adopt it with `--adopt`, or keep it by listing `.github/workflows/release.yml#jobs.publish` under + `owned`. +- On GitLab the switch writes nothing: releases there are semantic-release's. + +It needs repokeeper 0.11.0 or newer. + --- Back to the [README](../README.md). diff --git a/playground/config.d.ts b/playground/config.d.ts index e34b9b8..4676150 100644 --- a/playground/config.d.ts +++ b/playground/config.d.ts @@ -16,6 +16,7 @@ export const AUTOUPDATE_NEEDS_VERSION: string; export const SECURITY_NEEDS_VERSION: string; export const CACHE_CLEANUP_NEEDS_VERSION: string; export const AUTOMERGE_NEEDS_VERSION: string; +export const PUBLISH_NEEDS_VERSION: string; export const STACK_NEEDS_VERSION: Record; export const STACKS: Array<{ id: string; label: string }>; export const PLATFORM_STACKS: Record<"github" | "gitlab", string[]>; diff --git a/playground/config.js b/playground/config.js index fa510bc..00ff66f 100644 --- a/playground/config.js +++ b/playground/config.js @@ -20,6 +20,8 @@ export const SECURITY_NEEDS_VERSION = "0.9.0"; export const CACHE_CLEANUP_NEEDS_VERSION = "0.11.0"; /** The same for Dependabot auto-merge. */ export const AUTOMERGE_NEEDS_VERSION = "0.11.0"; +/** The same for publishing. */ +export const PUBLISH_NEEDS_VERSION = "0.11.0"; /** The same for the stacks added after the standard the file names: an earlier repokeeper does not know them. */ export const STACK_NEEDS_VERSION = { docker: "0.9.0" }; @@ -86,6 +88,11 @@ export const MODULES = [ label: "Dependabot auto-merge", hint: "auto-merge for patch and minor updates, after the required checks and reviews (GitHub)", }, + { + id: "publish", + label: "Publishing", + hint: "a release job that publishes to npm or PyPI with trusted publishing, no token (GitHub)", + }, ]; /** What stands in for a detail the form was left empty on; `problems` reports each one still in use. */ @@ -106,10 +113,11 @@ const ESSENTIAL_OFF = [ "security", "cache_cleanup", "automerge", + "publish", ]; /** The switches no preset turns on. */ -const IN_NO_PRESET = ["drift", "autoupdate", "security", "cache_cleanup", "automerge"]; +const IN_NO_PRESET = ["drift", "autoupdate", "security", "cache_cleanup", "automerge", "publish"]; /** The module switches of a preset, `health` as a plain on or off. */ export function presetModules(preset) { @@ -185,6 +193,7 @@ export function buildConfig(input) { ...(modules.security ? { security: true } : {}), ...(modules.cache_cleanup ? { cache_cleanup: true } : {}), ...(modules.automerge ? { automerge: true } : {}), + ...(modules.publish ? { publish: true } : {}), health: modules.health ? { license: health.license ? "MIT" : false, @@ -272,6 +281,13 @@ export function problems(input) { : "Dependabot auto-merge is written for GitHub only: on GitLab the switch writes nothing.", ); } + if (input.modules.publish) { + found.push( + input.platform === "github" + ? `Publishing needs repokeeper ${PUBLISH_NEEDS_VERSION} or newer, a node or python stack at the root, and a trusted publisher on npmjs.com or pypi.org.` + : "Publishing is written for GitHub only: on GitLab the switch writes nothing.", + ); + } if (input.platform === "github") { const { mode, ref, source } = input.workflows; if ((mode === "ref" || (mode === "mirror" && ref.trim() !== "")) && !/^[A-Za-z0-9_./-]+$/.test(ref.trim())) { diff --git a/src/commands/presets.ts b/src/commands/presets.ts index 092bac0..f0789d3 100644 --- a/src/commands/presets.ts +++ b/src/commands/presets.ts @@ -28,6 +28,7 @@ export const MODULE_IDS: ModuleId[] = [ "security", "cache_cleanup", "automerge", + "publish", ]; /** @@ -39,6 +40,8 @@ export const MODULE_IDS: ModuleId[] = [ * none either: it deletes things, so it is yours to ask for. Nor is Dependabot auto-merge * (`modules.automerge`): merging without a person is a choice the repository's owner makes. Under * `strict` it still waits for the review the ruleset requires, since the workflow never approves. + * Nor is publishing (`modules.publish`): it needs a trusted publisher set up on the registry first, and + * not every package is meant to be public. */ export function applyPreset(config: RepokeeperConfig, preset: Preset): RepokeeperConfig { if (preset === "standard") return config; @@ -56,6 +59,7 @@ export function applyPreset(config: RepokeeperConfig, preset: Preset): Repokeepe security: false, cache_cleanup: false, automerge: false, + publish: false, health: false, }, }; @@ -127,6 +131,7 @@ export async function chooseModules( security: true, cache_cleanup: true, automerge: true, + publish: true, health, }, }; diff --git a/src/commands/report.ts b/src/commands/report.ts index 014fec4..35e3edd 100644 --- a/src/commands/report.ts +++ b/src/commands/report.ts @@ -5,6 +5,7 @@ import type { Io } from "../cli.js"; import { duplicateTestRuns } from "../duplicates.js"; import { crlfTrackedFiles } from "../git.js"; import { describeOutput, type ModuleContext, nodeAtRoot, type Output } from "../model.js"; +import { publishPlan, type Registry } from "../modules/publish.js"; import { overriddenAttributes } from "../sync/attributes.js"; import type { RemovalAction } from "../sync/decide.js"; import type { SyncResult } from "../sync/sync.js"; @@ -12,6 +13,13 @@ import { TOOL_VERSIONS } from "../version.js"; const AUTOMERGE_WORKFLOW = ".github/workflows/automerge.yml"; +/** What to enter on the registry once, so that it takes the release workflow's word for who publishes. */ +function trustedPublisherStep(ctx: ModuleContext, registry: Registry): string { + const repo = ctx.repo.owner ? `${ctx.repo.owner}/${ctx.repo.name}` : "this repository"; + const where = registry === "npm" ? "npmjs.com (allow npm publish)" : "pypi.org (environment left empty)"; + return `add a trusted publisher on ${where}: repository ${repo}, workflow release.yml, exactly; until then the publish job fails`; +} + export interface CommandOptions { dryRun: boolean; force: boolean; @@ -104,6 +112,13 @@ export async function nextSteps(root: string, ctx: ModuleContext, result: SyncRe "run `repokeeper github apply` to allow auto-merge; Dependabot's updates then merge once the checks and reviews the default branch requires have passed, and not at all while it requires none", ); } + const publish = result.decisions.find( + (d) => d.output.module === "publish" && d.output.kind === "yaml" && d.action === "create", + ); + if (publish && ctx.platform.id === "github") { + const plan = publishPlan(ctx); + if (!("problem" in plan)) steps.push(trustedPublisherStep(ctx, plan.registry)); + } if (wrote(".gitattributes")) { const crlf = await crlfTrackedFiles(root); if (crlf.length > 0) { @@ -143,6 +158,16 @@ export async function printWarnings(root: string, ctx: ModuleContext, io: Io): P if (ctx.platform.id === "gitlab" && ctx.config.modules.automerge) { io.out("note: modules.automerge writes nothing on gitlab; Renovate's own automerge setting does that work there"); } + if (ctx.config.modules.publish) { + const plan = publishPlan(ctx); + if (ctx.platform.id === "gitlab") { + io.out( + "note: modules.publish writes nothing on gitlab; publishing is written for GitHub's release workflow only", + ); + } else if ("problem" in plan) { + io.out(`note: modules.publish writes nothing: ${plan.problem}`); + } + } if (ctx.platform.id === "gitlab") { for (const stack of ctx.stacks) { const os = stack.ci?.with.os; diff --git a/src/config/load.ts b/src/config/load.ts index e521b1b..a1cfe8d 100644 --- a/src/config/load.ts +++ b/src/config/load.ts @@ -69,12 +69,13 @@ export function parseConfig(text: string): RepokeeperConfig { export function renderConfig(config: RepokeeperConfig): string { // written only when on: a file without the line is one every earlier repokeeper reads as well - const { autoupdate, security, cache_cleanup, automerge, ...modules } = config.modules; + const { autoupdate, security, cache_cleanup, automerge, publish, ...modules } = config.modules; const asked = { ...(autoupdate ? { autoupdate } : {}), ...(security ? { security } : {}), ...(cache_cleanup ? { cache_cleanup } : {}), ...(automerge ? { automerge } : {}), + ...(publish ? { publish } : {}), }; const written = { ...config, @@ -123,6 +124,7 @@ function withDefaults( security: modules.security ?? false, cache_cleanup: modules.cache_cleanup ?? false, automerge: modules.automerge ?? false, + publish: modules.publish ?? false, health: modules.health, }, owned: data.owned ?? [], diff --git a/src/config/schema.ts b/src/config/schema.ts index e59ca57..8d22e70 100644 --- a/src/config/schema.ts +++ b/src/config/schema.ts @@ -29,6 +29,7 @@ export const configSchema = { security: flag, cache_cleanup: flag, automerge: flag, + publish: flag, health: { anyOf: [ { const: false }, diff --git a/src/config/types.ts b/src/config/types.ts index 6d0b8c6..a00e66a 100644 --- a/src/config/types.ts +++ b/src/config/types.ts @@ -45,6 +45,8 @@ export interface ModulesConfig { cache_cleanup: boolean; /** A workflow that turns on auto-merge for Dependabot's patch and minor updates; off unless asked for. */ automerge: boolean; + /** A job in the release workflow that publishes each release to npm or PyPI; off unless asked for. */ + publish: boolean; health: HealthConfig | false; } @@ -92,6 +94,7 @@ export function defaultConfig(input: { security: false, cache_cleanup: false, automerge: false, + publish: false, health: { license: "MIT", copyright: input.copyright, contact: input.contact, codeowners: input.codeowners }, }, owned: [], diff --git a/src/model.ts b/src/model.ts index 765a7fe..c6ae1db 100644 --- a/src/model.ts +++ b/src/model.ts @@ -166,6 +166,8 @@ export interface PlatformAdapter { cacheCleanup(ctx: ModuleContext): Output[]; /** A workflow that turns on auto-merge for patch and minor dependency updates; empty where the platform has none. */ dependencyAutomerge(ctx: ModuleContext): Output[]; + /** A job that publishes each release of `stack` to `registry`; empty where the platform has none written. */ + registryPublish(ctx: ModuleContext, stack: ResolvedStack, registry: "npm" | "pypi"): Output[]; /** release-please configuration, its manifest and the caller release workflow. `directory` is the released stack's folder. */ releaseAutomation(ctx: ModuleContext, release: ReleaseInfo, directory?: string): Output[]; } diff --git a/src/modules/index.ts b/src/modules/index.ts index 72626cc..6982a50 100644 --- a/src/modules/index.ts +++ b/src/modules/index.ts @@ -9,6 +9,7 @@ import { editorconfigModule } from "./editorconfig.js"; import { gitignoreModule } from "./gitignore.js"; import { healthModule } from "./health.js"; import { hooksModule } from "./hooks.js"; +import { publishModule } from "./publish.js"; import { releaseModule } from "./release.js"; import { securityModule } from "./security.js"; @@ -25,4 +26,5 @@ export const MODULES: Module[] = [ securityModule, cacheCleanupModule, automergeModule, + publishModule, ]; diff --git a/src/modules/publish.ts b/src/modules/publish.ts new file mode 100644 index 0000000..cc976c2 --- /dev/null +++ b/src/modules/publish.ts @@ -0,0 +1,34 @@ +import type { Module, ModuleContext, ReleaseType, ResolvedStack } from "../model.js"; +import { pickReleaseStack } from "./release.js"; + +export type Registry = "npm" | "pypi"; + +/** The registry each release type publishes to; a type with no line here is not published. */ +export const REGISTRIES: Partial> = { node: "npm", python: "pypi" }; + +export type PublishPlan = { stack: ResolvedStack; registry: Registry } | { problem: string }; + +/** + * What `modules.publish` publishes, or why it writes nothing: the released stack has to be one with a + * registry, at the root. A stack in a folder is released by release-please under outputs named after + * the folder, which the release workflow does not pass on. + */ +export function publishPlan(ctx: ModuleContext): PublishPlan { + if (!ctx.config.modules.release) return { problem: "it publishes what modules.release releases, which is off" }; + const stack = pickReleaseStack(ctx.stacks); + const registry = stack ? REGISTRIES[stack.release.type] : undefined; + if (!stack || !registry) { + return { problem: `it publishes node packages to npm and python packages to PyPI, and ${stack?.id} is neither` }; + } + if (stack.directory) return { problem: `publishing a stack in a folder (${stack.directory}) is not supported` }; + return { stack, registry }; +} + +export const publishModule: Module = { + id: "publish", + enabled: (config) => config.modules.publish, + outputs: (ctx) => { + const plan = publishPlan(ctx); + return "problem" in plan ? [] : ctx.platform.registryPublish(ctx, plan.stack, plan.registry); + }, +}; diff --git a/src/platforms/github.ts b/src/platforms/github.ts index 0f9863a..b83e954 100644 --- a/src/platforms/github.ts +++ b/src/platforms/github.ts @@ -10,9 +10,20 @@ import { type Output, type PlatformAdapter, type ReleaseInfo, + type ResolvedStack, } from "../model.js"; +import type { Registry } from "../modules/publish.js"; import { readWorkflow } from "../templates.js"; -import { EXACT_WORKFLOW_REF, PACKAGE_VERSION, REUSABLE_REPO, WORKFLOW_REF } from "../version.js"; +import { + ACTION_PINS, + EXACT_WORKFLOW_REF, + PACKAGE_VERSION, + REUSABLE_REPO, + TOOL_VERSIONS, + WORKFLOW_REF, +} from "../version.js"; + +const pinned = (pin: (typeof ACTION_PINS)[keyof typeof ACTION_PINS]) => `${pin.action}@${pin.sha}`; const yamlFile = (module: string, path: string, data: unknown): Output => ({ kind: "file", @@ -463,6 +474,12 @@ export const githubPlatform: PlatformAdapter = { ]; }, + registryPublish(_ctx: ModuleContext, stack: ResolvedStack, registry: Registry): Output[] { + // A job of release.yml itself, not a reusable workflow: PyPI does not take a reusable workflow as a + // trusted publisher, and npm checks the name of the calling workflow. Both then name release.yml. + return [workflowKey("publish", ".github/workflows/release.yml", ["jobs", "publish"], publishJob(stack, registry))]; + }, + releaseAutomation(ctx: ModuleContext, release: ReleaseInfo, directory?: string): Output[] { const path = ".github/workflows/release.yml"; const seed = release.version ?? ctx.repo.releasedVersion ?? "0.0.0"; @@ -542,6 +559,49 @@ function mergeMethod(ctx: ModuleContext): "squash" | "merge" | "rebase" { return "rebase"; } +/** + * The job that publishes a release once release-please has made it. The registry trusts the run through + * OIDC (`id-token: write`), so no token is stored; npm adds provenance by itself for a public repository. + */ +function publishJob(stack: ResolvedStack, registry: Registry): Record { + const checkout = { uses: pinned(ACTION_PINS.checkout), with: { "persist-credentials": false } }; + const common = { + needs: "release", + // biome-ignore lint/suspicious/noTemplateCurlyInString: a GitHub Actions expression, not a JS template + if: "${{ needs.release.outputs.release_created == 'true' }}", + "runs-on": "ubuntu-latest", + permissions: { contents: "read", "id-token": "write" }, + }; + if (registry === "pypi") { + return { + ...common, + steps: [ + checkout, + { uses: pinned(ACTION_PINS.setupPython), with: { "python-version": "3.x" } }, + { name: "Build", run: `python -m pip install build==${TOOL_VERSIONS.pypaBuild} && python -m build` }, + { name: "Publish to PyPI", uses: pinned(ACTION_PINS.pypiPublish) }, + ], + }; + } + const manager = stack.ci?.with["package-manager"] ?? "npm"; + return { + ...common, + ...(manager !== "npm" ? { env: { COREPACK_ENABLE_DOWNLOAD_PROMPT: "0" } } : {}), + steps: [ + checkout, + // Node.js 24 comes with an npm new enough (11.5.1 or later) to publish through OIDC + { + uses: pinned(ACTION_PINS.setupNode), + with: { "node-version": "24", "registry-url": "https://registry.npmjs.org" }, + }, + ...(manager !== "npm" ? [{ name: "Enable corepack", run: "corepack enable" }] : []), + { name: "Install dependencies", run: stack.ci?.with["install-command"] ?? "npm ci" }, + // npm itself, whatever the package manager: it is the client that signs in through OIDC + { name: "Publish to npm", run: "npm publish --access public" }, + ], + }; +} + // Runs the stack job unless the changes job said "only listed paths". A changes job that failed // gives no answer, and then the stack job runs: an error never skips the tests. // biome-ignore lint/suspicious/noTemplateCurlyInString: a GitHub Actions expression, not a JS template diff --git a/src/platforms/gitlab.ts b/src/platforms/gitlab.ts index cbeee90..aa25d84 100644 --- a/src/platforms/gitlab.ts +++ b/src/platforms/gitlab.ts @@ -226,6 +226,11 @@ export const gitlabPlatform: PlatformAdapter = { return []; }, + // releases on GitLab are semantic-release's, whose own plugins publish; repokeeper does not write them yet + registryPublish(): Output[] { + return []; + }, + ciWorkflow(ctx: ModuleContext): Output[] { const jobs: Output[] = []; const node = ctx.stacks.find((stack) => stack.id === "node")?.ci; diff --git a/src/sync/decide.ts b/src/sync/decide.ts index 5801338..1779882 100644 --- a/src/sync/decide.ts +++ b/src/sync/decide.ts @@ -7,6 +7,46 @@ export type Action = "create" | "write" | "adopt" | "unchanged" | "conflict" | " /** `left`: a seed that still exists; `release`: output the user now owns, left as it is and dropped from the lock. */ export type RemovalAction = "delete" | "orphan-edited" | "gone" | "left" | "release"; +/** + * `owner/repo[/path]@ref` of an action. A local action (`./…`), a Docker image (`docker://…`) and a + * reusable workflow (`…/.github/workflows/…`) do not match or are left out below. + */ +const ACTION = /^([A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+(?:\/[^@\s]+)?)@[^@\s]+$/; + +/** The value with the version of every action a step uses taken out, so only the action's name is left. */ +function withoutActionVersions(value: unknown): unknown { + if (Array.isArray(value)) return value.map(withoutActionVersions); + if (value === null || typeof value !== "object") return value; + return Object.fromEntries( + Object.entries(value).map(([key, item]) => { + if (key !== "steps" || !Array.isArray(item)) return [key, withoutActionVersions(item)]; + return [ + key, + item.map((step) => { + const uses = (step as { uses?: unknown } | null)?.uses; + const action = typeof uses === "string" && !uses.includes("/.github/workflows/") ? ACTION.exec(uses) : null; + const rest = withoutActionVersions(step) as Record; + return action ? { ...rest, uses: `${action[1]}@` } : rest; + }), + ]; + }), + ); +} + +/** + * Whether two YAML values (as JSON text) differ only in the versions of the actions their steps use: the + * same action at another commit or tag, as a Dependabot pull request leaves it. The call of a reusable + * workflow is not a step, so the ref it is pinned to still counts. + */ +export function sameIgnoringActionVersions(currentText: string, desiredJson: string): boolean { + try { + const loose = (json: string) => JSON.stringify(withoutActionVersions(JSON.parse(json))); + return loose(currentText) === loose(desiredJson); + } catch { + return false; + } +} + /** The update decision table from the spec, section 7. */ export function decide( output: Output, @@ -19,7 +59,13 @@ export function decide( if (current === hashText(desiredText(output))) return "unchanged"; // adding the two marker lines is always safe, so they are written without --adopt or --accept if (output.kind === "marker") return "write"; - if (entry) return current === entry.hash ? "write" : "conflict"; + if (entry) { + // what repokeeper wrote and nobody touched is moved, pinned actions included + if (current === entry.hash) return "write"; + // a newer version of the same action is the repository's to keep, not an edit to report + if (output.kind === "yaml" && sameIgnoringActionVersions(currentText, desiredText(output))) return "unchanged"; + return "conflict"; + } return adopt ? "adopt" : "unmanaged"; } diff --git a/src/version.ts b/src/version.ts index d016536..bda7f79 100644 --- a/src/version.ts +++ b/src/version.ts @@ -19,6 +19,25 @@ export const TOOL_VERSIONS = { // 10.x needs conventional-changelog-writer 9, and semantic-release 25's notes generator bundles 8 conventionalCommitsPreset: "9.3.1", renovate: "44.149.2", + // PyPA's `build`, which the publish job builds a Python package with + pypaBuild: "1.6.1", +} as const; + +/** + * Actions used by jobs written straight into a repository's own workflows (the publish job), pinned by + * commit. `repokeeper update` moves them while the key is as it wrote it; a newer commit of the same + * action, from Dependabot, is not drift. They move with the same actions in the reusable workflows + * here, which a test holds them to. + */ +export const ACTION_PINS = { + checkout: { action: "actions/checkout", sha: "3d3c42e5aac5ba805825da76410c181273ba90b1", version: "v7.0.1" }, + setupNode: { action: "actions/setup-node", sha: "949feb2413d6458794dcd2491c4babbbce0c15c1", version: "v7.1.0" }, + setupPython: { action: "actions/setup-python", sha: "5fda3b95a4ea91299a34e894583c3862153e4b97", version: "v7.0.0" }, + pypiPublish: { + action: "pypa/gh-action-pypi-publish", + sha: "dc37677b2e1c63e2034f94d8a5b11f265b73ba33", + version: "v1.14.2", + }, } as const; /** Repository hosting the reusable workflows that generated CI files call. */ diff --git a/test/config.test.ts b/test/config.test.ts index a4265da..c9e4c47 100644 --- a/test/config.test.ts +++ b/test/config.test.ts @@ -26,7 +26,7 @@ describe("parseConfig", () => { expect(parseConfig(renderConfig(base))).toEqual(base); }); - it("defaults boolean modules to true (drift, autoupdate, security, cache_cleanup and automerge to false) and optional lists to empty", () => { + it("defaults boolean modules to true (drift, autoupdate, security, cache_cleanup, automerge and publish to false) and optional lists to empty", () => { const config = parseConfig( "schema: 1\nstandard: 1.0.0\nplatform: github\nstacks: [node]\nmodules:\n health: false\n", ); @@ -43,6 +43,7 @@ describe("parseConfig", () => { security: false, cache_cleanup: false, automerge: false, + publish: false, health: false, }); expect(config.owned).toEqual([]); diff --git a/test/modules-publish.test.ts b/test/modules-publish.test.ts new file mode 100644 index 0000000..62c064e --- /dev/null +++ b/test/modules-publish.test.ts @@ -0,0 +1,184 @@ +import { readFileSync } from "node:fs"; +import { describe, expect, it } from "vitest"; +import { parse } from "yaml"; +import { applyPreset } from "../src/commands/presets.js"; +import { nextSteps, printWarnings } from "../src/commands/report.js"; +import { parseConfig, renderConfig } from "../src/config/load.js"; +import type { Output } from "../src/model.js"; +import { MODULES } from "../src/modules/index.js"; +import { publishModule } from "../src/modules/publish.js"; +import { planOutputs } from "../src/plan.js"; +import type { SyncResult } from "../src/sync/sync.js"; +import { ACTION_PINS, TOOL_VERSIONS } from "../src/version.js"; +import { capture, gitlabContext, makeContext, nodeResolved, tempDir } from "./helpers.js"; + +const PATH = ".github/workflows/release.yml"; +const on = (overrides: Parameters[0] = {}) => + makeContext({ ...overrides, modules: { publish: true, ...overrides.modules } }); +const publishJob = (ctx = on()) => { + const outputs = publishModule.outputs(ctx); + expect(outputs).toHaveLength(1); + const [output] = outputs as [Output]; + expect(output).toMatchObject({ kind: "yaml", module: "publish", path: PATH, keyPath: ["jobs", "publish"] }); + return (output as { value: Record }).value; +}; +const python = nodeResolved({ + id: "python", + dependabot: ["pip"], + ci: { workflow: "stack-python.yml", with: { "install-command": "python -m pip install -e ." } }, + release: { type: "python", version: "1.0.0" }, +}); +const COMMON = { + needs: "release", + // biome-ignore lint/suspicious/noTemplateCurlyInString: a GitHub Actions expression + if: "${{ needs.release.outputs.release_created == 'true' }}", + "runs-on": "ubuntu-latest", + // trusted publishing: the registry takes the run's OIDC token instead of a stored one + permissions: { contents: "read", "id-token": "write" }, +}; +const CHECKOUT = { uses: `actions/checkout@${ACTION_PINS.checkout.sha}`, with: { "persist-credentials": false } }; +const notes = async (ctx: ReturnType) => { + const noted = capture(); + await printWarnings(await tempDir(), ctx, noted.io); + return noted.out; +}; +const dependabotIgnore = (ctx: ReturnType) => { + const file = planOutputs(ctx).find((output) => output.path === ".github/dependabot.yml"); + const config = parse(file?.kind === "file" ? file.content : "") as { + updates: { "package-ecosystem": string; ignore?: { "dependency-name": string }[] }[]; + }; + return config.updates.find((update) => update["package-ecosystem"] === "github-actions")?.ignore; +}; + +describe("publish module", () => { + it("is off unless asked for, in no preset, and round-trips through the configuration file", () => { + expect(makeContext().config.modules.publish).toBe(false); + expect(publishModule.enabled(makeContext().config)).toBe(false); + expect(renderConfig(makeContext().config)).not.toContain("publish"); + for (const preset of ["essential", "standard", "strict"] as const) { + expect(applyPreset(makeContext().config, preset).modules.publish).toBe(false); + } + expect(publishModule.enabled(on().config)).toBe(true); + expect(parseConfig(renderConfig(on().config)).modules.publish).toBe(true); + expect(MODULES).toContain(publishModule); + }); + + it("adds a publish job to the release workflow, next to the release job", () => { + const outputs = planOutputs(on()).filter((output) => output.path === PATH && output.kind === "yaml"); + expect(outputs.map((output) => (output.kind === "yaml" ? output.keyPath.join(".") : ""))).toEqual( + expect.arrayContaining(["jobs.release", "jobs.publish"]), + ); + }); + + it("publishes a node package to npm through trusted publishing, with no token", () => { + expect(publishJob()).toEqual({ + ...COMMON, + steps: [ + CHECKOUT, + { + uses: `actions/setup-node@${ACTION_PINS.setupNode.sha}`, + with: { "node-version": "24", "registry-url": "https://registry.npmjs.org" }, + }, + { name: "Install dependencies", run: "npm ci" }, + // no --provenance: npm adds it by itself for a public repository, and refuses it for a private one + { name: "Publish to npm", run: "npm publish --access public" }, + ], + }); + expect(JSON.stringify(publishJob())).not.toMatch(/NODE_AUTH_TOKEN|NPM_TOKEN|secrets\./); + }); + + it("installs with the stack's package manager, and still publishes with npm", () => { + const pnpm = nodeResolved(); + pnpm.ci = { workflow: "stack-node.yml", with: { "package-manager": "pnpm", "install-command": "pnpm install" } }; + const job = publishJob(on({ stacks: [pnpm] })) as { env: unknown; steps: { name?: string; run?: string }[] }; + expect(job.env).toEqual({ COREPACK_ENABLE_DOWNLOAD_PROMPT: "0" }); + expect(job.steps.map((step) => step.run).filter(Boolean)).toEqual([ + "corepack enable", + "pnpm install", + "npm publish --access public", + ]); + }); + + it("publishes a python package to PyPI through trusted publishing", () => { + expect(publishJob(on({ config: { stacks: ["python"] }, stacks: [python] }))).toEqual({ + ...COMMON, + steps: [ + CHECKOUT, + { uses: `actions/setup-python@${ACTION_PINS.setupPython.sha}`, with: { "python-version": "3.x" } }, + { name: "Build", run: `python -m pip install build==${TOOL_VERSIONS.pypaBuild} && python -m build` }, + { name: "Publish to PyPI", uses: `pypa/gh-action-pypi-publish@${ACTION_PINS.pypiPublish.sha}` }, + ], + }); + }); + + it("publishes the stack that is released, and only that one", () => { + const go = nodeResolved({ id: "go", release: { type: "go", version: null } }); + const job = publishJob(on({ config: { stacks: ["python", "go"] }, stacks: [python, go] })) as { + steps: { name?: string }[]; + }; + expect(job.steps.at(-1)?.name).toBe("Publish to PyPI"); + // release-please releases go here, so there is no python release to publish + expect(publishModule.outputs(on({ config: { stacks: ["go", "python"] }, stacks: [go, python] }))).toEqual([]); + }); + + it("writes nothing, and says why, where it cannot publish", async () => { + const go = on({ + config: { stacks: ["go"] }, + stacks: [nodeResolved({ id: "go", release: { type: "go", version: null } })], + }); + expect(publishModule.outputs(go)).toEqual([]); + expect(await notes(go)).toContain( + "note: modules.publish writes nothing: it publishes node packages to npm and python packages to PyPI, and go is neither", + ); + + const folder = on({ stacks: [nodeResolved({ directory: "web" })] }); + expect(publishModule.outputs(folder)).toEqual([]); + expect(await notes(folder)).toContain( + "note: modules.publish writes nothing: publishing a stack in a folder (web) is not supported", + ); + + const unreleased = on({ modules: { release: false } }); + expect(publishModule.outputs(unreleased)).toEqual([]); + expect(await notes(unreleased)).toContain( + "note: modules.publish writes nothing: it publishes what modules.release releases, which is off", + ); + expect(await notes(on())).toEqual([]); + }); + + it("leaves Dependabot free to bump the actions it writes, here and in the repository's own workflows", () => { + // a version-only bump of them is not drift (see sync-action-versions.test.ts) + expect(dependabotIgnore(on())).toBeUndefined(); + expect(dependabotIgnore(on({ config: { stacks: ["python"] }, stacks: [python] }))).toBeUndefined(); + }); + + it("pins the actions it writes to the commits the reusable workflows here use", () => { + const reusable = + readFileSync(".github/workflows/stack-node.yml", "utf8") + + readFileSync(".github/workflows/stack-python.yml", "utf8"); + for (const pin of [ACTION_PINS.checkout, ACTION_PINS.setupNode, ACTION_PINS.setupPython]) { + expect(reusable).toContain(`${pin.action}@${pin.sha} # ${pin.version}`); + } + for (const pin of Object.values(ACTION_PINS)) expect(pin.sha).toMatch(/^[0-9a-f]{40}$/); + }); + + it("names the trusted publisher to set up, once: when the job is new", async () => { + const job: Output = { kind: "yaml", module: "publish", path: PATH, keyPath: ["jobs", "publish"], value: {} }; + const result = (action: string): SyncResult => + ({ decisions: [{ output: job, action }], removals: [] }) as unknown as SyncResult; + expect(await nextSteps(".", on(), result("create"))).toEqual([ + "add a trusted publisher on npmjs.com (allow npm publish): repository vannt-dev/example, workflow release.yml, exactly; until then the publish job fails", + ]); + expect(await nextSteps(".", on({ config: { stacks: ["python"] }, stacks: [python] }), result("create"))).toEqual([ + "add a trusted publisher on pypi.org (environment left empty): repository vannt-dev/example, workflow release.yml, exactly; until then the publish job fails", + ]); + expect(await nextSteps(".", on(), result("unchanged"))).toEqual([]); + }); + + it("writes nothing on GitLab, and says so", async () => { + const gitlab = gitlabContext({ modules: { publish: true } }); + expect(publishModule.outputs(gitlab)).toEqual([]); + expect(await notes(gitlab)).toContain( + "note: modules.publish writes nothing on gitlab; publishing is written for GitHub's release workflow only", + ); + }); +}); diff --git a/test/presets.test.ts b/test/presets.test.ts index e0d7874..1d854ed 100644 --- a/test/presets.test.ts +++ b/test/presets.test.ts @@ -74,6 +74,7 @@ describe("presets", () => { security: false, cache_cleanup: false, automerge: false, + publish: false, health: false, }); expect(applyPreset(base(), "essential").github).toBeUndefined(); @@ -213,12 +214,13 @@ describe("init --interactive", () => { "Apply security?", "Apply cache_cleanup?", "Apply automerge?", + "Apply publish?", "Write these files?", ]; it("names the files of each part, takes the answers, and writes only what was chosen", async () => { const dir = await nodeRepo(); - // editor, ignore, commits, hooks, ci, release, deps, health, drift, update, security, caches, automerge, write + // editor, ignore, commits, hooks, ci, release, deps, health, drift, update, security, caches, automerge, publish, write const init = await repokeeper( dir, ["init", "--interactive"], @@ -236,6 +238,7 @@ describe("init --interactive", () => { undefined, undefined, undefined, + undefined, true, ], ); @@ -250,6 +253,7 @@ describe("init --interactive", () => { expect(init.out).toContain("security:\n .github/workflows/security.yml"); expect(init.out).toContain("cache_cleanup:\n .github/workflows/cache-cleanup.yml"); expect(init.out).toContain("automerge:\n .github/workflows/automerge.yml"); + expect(init.out).toContain("publish:\n .github/workflows/release.yml"); const config = parse(await readFile(join(dir, ".repokeeper.yml"), "utf8")); expect(config.modules).toMatchObject({ commits: true, hooks: false, ci: true, release: false, deps: true }); @@ -267,6 +271,9 @@ describe("init --interactive", () => { // and for Dependabot auto-merge expect(config.modules).not.toHaveProperty("automerge"); expect(existsSync(join(dir, ".github/workflows/automerge.yml"))).toBe(false); + // and for publishing + expect(config.modules).not.toHaveProperty("publish"); + expect(existsSync(join(dir, ".github/workflows/release.yml"))).toBe(false); expect(existsSync(join(dir, "lefthook.yml"))).toBe(false); expect(existsSync(join(dir, "release-please-config.json"))).toBe(false); expect(existsSync(join(dir, "commitlint.config.mjs"))).toBe(true); @@ -296,7 +303,12 @@ describe("init --interactive", () => { answers[11] = true; // and Dependabot auto-merge answers[12] = true; - await repokeeper(other, ["init", "-i", "--preset", "essential"], answers); + // and publishing, which publishes what the release module releases, off in essential + answers[13] = true; + const taken = await repokeeper(other, ["init", "-i", "--preset", "essential"], answers); + expect(taken.out).toContain( + "note: modules.publish writes nothing: it publishes what modules.release releases, which is off", + ); expect(parse(await readFile(join(other, ".repokeeper.yml"), "utf8")).modules.autoupdate).toBe(true); expect(existsSync(join(other, ".github/workflows/repokeeper-autoupdate.yml"))).toBe(true); expect(parse(await readFile(join(other, ".repokeeper.yml"), "utf8")).modules.security).toBe(true); @@ -305,6 +317,8 @@ describe("init --interactive", () => { expect(existsSync(join(other, ".github/workflows/cache-cleanup.yml"))).toBe(true); expect(parse(await readFile(join(other, ".repokeeper.yml"), "utf8")).modules.automerge).toBe(true); expect(existsSync(join(other, ".github/workflows/automerge.yml"))).toBe(true); + expect(parse(await readFile(join(other, ".repokeeper.yml"), "utf8")).modules.publish).toBe(true); + expect(existsSync(join(other, ".github/workflows/release.yml"))).toBe(false); expect(parse(await readFile(join(other, ".repokeeper.yml"), "utf8")).modules.health.license).toBe("MIT"); expect(existsSync(join(other, "CONTRIBUTING.md"))).toBe(true); }); diff --git a/test/sync-action-versions.test.ts b/test/sync-action-versions.test.ts new file mode 100644 index 0000000..37dacfd --- /dev/null +++ b/test/sync-action-versions.test.ts @@ -0,0 +1,127 @@ +import { execFileSync } from "node:child_process"; +import { readFile, writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { describe, expect, it } from "vitest"; +import { run } from "../src/cli.js"; +import type { Output } from "../src/model.js"; +import { decide, sameIgnoringActionVersions } from "../src/sync/decide.js"; +import { hashText } from "../src/sync/hash.js"; +import { type LockEntry, targetOf } from "../src/sync/lock.js"; +import { desiredText } from "../src/sync/state.js"; +import { ACTION_PINS } from "../src/version.js"; +import { capture, tempDir } from "./helpers.js"; + +const OURS = "3d3c42e5aac5ba805825da76410c181273ba90b1"; +const NEWER = "1111111111111111111111111111111111111111"; +const job = (checkout: string, run = "npm publish --access public", call?: string) => ({ + needs: "release", + ...(call ? { uses: call } : {}), + steps: [ + { uses: checkout, with: { "persist-credentials": false } }, + { name: "Publish", run }, + ], +}); +const output: Output = { + kind: "yaml", + module: "publish", + path: ".github/workflows/release.yml", + keyPath: ["jobs", "publish"], + value: job(`actions/checkout@${OURS}`), +}; +const text = (value: unknown) => JSON.stringify(value); +const lockEntry = (written: unknown): LockEntry => ({ + id: "yaml:x", + module: "publish", + hash: hashText(text(written)), + target: targetOf(output), +}); +const ours = lockEntry(output.value); + +describe("a version-only change of an action a step uses", () => { + it("is not an edit: Dependabot moving the commit or the tag leaves the key unchanged", () => { + expect(decide(output, text(job(`actions/checkout@${NEWER}`)), ours, false)).toBe("unchanged"); + expect(decide(output, text(job("actions/checkout@v8.0.0")), ours, false)).toBe("unchanged"); + }); + + it("is moved by repokeeper while the key is still what it wrote", () => { + // repokeeper wrote NEWER before; the standard now pins OURS, and nobody touched the file + const before = job(`actions/checkout@${NEWER}`); + expect(decide(output, text(before), lockEntry(before), false)).toBe("write"); + }); + + it("is still an edit when the action itself or anything else in the key changed", () => { + expect(decide(output, text(job(`someone/checkout@${OURS}`)), ours, false)).toBe("conflict"); + expect(decide(output, text(job(`actions/checkout@${NEWER}`, "npm publish")), ours, false)).toBe("conflict"); + expect(decide(output, text({ ...job(`actions/checkout@${NEWER}`), "runs-on": "self-hosted" }), ours, false)).toBe( + "conflict", + ); + }); + + it("does not cover the call of a reusable workflow, whose ref repokeeper pins on purpose", () => { + const call = (ref: string) => ({ uses: `vannt-dev/repokeeper/.github/workflows/release-please.yml@${ref}` }); + const caller: Output = { ...output, keyPath: ["jobs", "release"], value: call("v0") }; + expect(decide(caller, text(call("main")), lockEntry(call("v0")), false)).toBe("conflict"); + expect(sameIgnoringActionVersions(text(call("main")), text(call("v0")))).toBe(false); + }); + + it("does not take over a key repokeeper never wrote", () => { + expect(decide(output, text(job(`actions/checkout@${NEWER}`)), undefined, false)).toBe("unmanaged"); + }); + + it("leaves local actions and docker images alone", () => { + const local = (path: string) => ({ steps: [{ uses: path }] }); + expect(sameIgnoringActionVersions(text(local("./a")), text(local("./b")))).toBe(false); + expect(sameIgnoringActionVersions(text(local("docker://alpine:3.20")), text(local("docker://alpine:3.21")))).toBe( + false, + ); + // an action in a subfolder of its repository is still the same action + const sub = (ref: string) => ({ steps: [{ uses: `github/codeql-action/init@${ref}` }] }); + expect(sameIgnoringActionVersions(text(sub("v3")), text(sub(OURS)))).toBe(true); + }); + + it("only applies to YAML keys", () => { + const file: Output = { kind: "file", module: "m", path: "a.yml", content: `uses: actions/checkout@${OURS}\n` }; + const written = { ...ours, hash: hashText(desiredText(file)) }; + expect(decide(file, `uses: actions/checkout@${NEWER}\n`, written, false)).toBe("conflict"); + }); +}); + +describe("check after Dependabot bumps the publish job", () => { + const git = (cwd: string, ...args: string[]) => execFileSync("git", args, { cwd, stdio: "pipe" }); + + it("reports no drift for a new commit of the same action, and drift for any other edit", async () => { + const dir = await tempDir(); + await writeFile(join(dir, "package.json"), `${JSON.stringify({ name: "demo", version: "1.0.0" })}\n`); + await writeFile( + join(dir, ".repokeeper.yml"), + "schema: 1\nstandard: 1.0.0\nplatform: github\nstacks: [node]\nmodules:\n publish: true\n health: false\n", + ); + git(dir, "init", "-q", "-b", "main"); + git(dir, "config", "user.name", "Demo"); + git(dir, "config", "user.email", "demo@example.com"); + git(dir, "config", "core.autocrlf", "false"); + git(dir, "remote", "add", "origin", "https://github.com/demo-owner/demo.git"); + git(dir, "add", "-A"); + git(dir, "commit", "-qm", "chore: initial"); + const cli = async (...args: string[]) => { + const c = capture(dir); + return { code: await run(args, c.io), out: c.out.join("\n") }; + }; + expect((await cli("init")).code).toBe(0); + const path = join(dir, ".github/workflows/release.yml"); + const written = await readFile(path, "utf8"); + const pin = `${ACTION_PINS.setupNode.action}@${ACTION_PINS.setupNode.sha}`; + expect(written).toContain(pin); + + // what a Dependabot pull request does: a new commit, and the comment it keeps beside it + await writeFile(path, written.replace(pin, `${ACTION_PINS.setupNode.action}@${NEWER} # v7.2.0`)); + expect(await cli("check")).toMatchObject({ code: 0 }); + // nor is the bump undone, or reported, by update + const update = await cli("update"); + expect(update.code).toBe(0); + expect(await readFile(path, "utf8")).toContain(`@${NEWER} # v7.2.0`); + + await writeFile(path, written.replace("npm publish --access public", "npm publish")); + expect((await cli("check")).code).toBe(1); + }); +});