From fdf460b7ce051df20302e88fb842cb401edff559 Mon Sep 17 00:00:00 2001 From: Kit <344842+roodboi@users.noreply.github.com> Date: Fri, 13 Nov 2026 20:07:16 +0000 Subject: [PATCH 01/32] feat(env): prepare resumable migration and separate cleanup (HACK-2191) --- src/commands/doctor.ts | 451 ++--------- src/commands/env-migrate.ts | 145 ++++ src/commands/env.ts | 12 +- src/commands/project.ts | 38 +- src/commands/session.ts | 10 +- src/lib/env-migration-files.ts | 83 ++ src/lib/env-migration-snapshot.ts | 229 ++++++ src/lib/env-migration.ts | 1023 ++++++++++++++++++++++++ src/lib/hack-env.ts | 4 + src/lib/project-env-config.ts | 683 +++++++++++++++- tests/env-migration-journal.test.ts | 467 +++++++++++ tests/helpers/env-migration-child.ts | 31 + tests/helpers/env-migration-fixture.ts | 115 +++ 13 files changed, 2839 insertions(+), 452 deletions(-) create mode 100644 src/commands/env-migrate.ts create mode 100644 src/lib/env-migration-files.ts create mode 100644 src/lib/env-migration-snapshot.ts create mode 100644 src/lib/env-migration.ts create mode 100644 tests/env-migration-journal.test.ts create mode 100644 tests/helpers/env-migration-child.ts create mode 100644 tests/helpers/env-migration-fixture.ts diff --git a/src/commands/doctor.ts b/src/commands/doctor.ts index 25917cf69..36d807b8e 100644 --- a/src/commands/doctor.ts +++ b/src/commands/doctor.ts @@ -49,7 +49,6 @@ import { HACK_PROJECT_DIR_PRIMARY, LEGACY_GRAFANA_HOST, PROJECT_ENV_CONTRACT_FILENAME, - PROJECT_ENV_FILENAME, PROJECT_ENV_KEY_FILENAME, PROJECT_ENV_STATE_FILENAME, } from "../constants.ts"; @@ -145,6 +144,11 @@ import { inspectWorktreeSecretKeys, } from "../lib/doctor-worktree.ts"; import { parseDotEnv } from "../lib/env.ts"; +import { + applyEnvMigration, + previewEnvMigration, +} from "../lib/env-migration.ts"; +import { envMigrationSnapshots } from "../lib/env-migration-snapshot.ts"; import { ensureDir, ensureGitignoreEntry, @@ -226,28 +230,20 @@ import { import { runProjectDomainCommand } from "../lib/project-domain-command.ts"; import { canMigrateCheckoutLegacyDotenv, - describeKeptSecretStorePointer, - describeRetainedLegacyEnvFile, discoverComposeServiceNames, dropRemovedLegacySecretStorePointer, ensureHackDirGitignore, HACK_DIR_GITIGNORE_READ_ONLY_WARNING, HACK_DIR_GITIGNORE_REFUSAL, HackDirGitignoreRefusalError, - inspectLegacyComposeEnvFileReferences, inspectProjectEnvMaterialization, - isGitTrackedOrUnknownProjectPath, - type LegacyComposeEnvFileReference, LegacyDotenvMigrationRefusalError, LegacyEnvCleanupError, type LegacyProjectEnvMigration, listProjectEnvPlaintextEntries, - migrateLegacyProjectEnv, ProjectEnvLayerPathConflictError, - planTrackedEnvMigrationChanges, projectEnvConfigExistsForMutation, removeLegacyProjectEnvArtifacts, - repairLegacyComposeEnvFileReferences, resolveProjectEnvConfig, type TrackedEnvMigrationChange, } from "../lib/project-env-config.ts"; @@ -4173,11 +4169,18 @@ export async function assertTrackedEnvMigrationConsent(opts: { if (!project) { return; } - const changes = await planTrackedEnvMigrationChanges({ + const config = await readProjectConfig(project); + const preview = await previewEnvMigration({ projectRoot: project.projectRoot, projectDir: project.projectDir, - composeFile: project.composeFile, + projectName: config.name ?? "project", + serviceNames: await discoverComposeServiceNames({ + composeFile: project.composeFile, + }), }); + const changes: TrackedEnvMigrationChange[] = preview.files + .filter((file) => file.tracked && file.action !== "removable") + .map((file) => ({ path: file.path, change: "edit" })); if (changes.length === 0) { return; } @@ -4203,85 +4206,6 @@ async function confirmTrackedEnvChange(opts: { } /** Confirms the Compose env_file repair, as a tracked change when git tracks the file. */ -async function confirmComposeEnvRepair(opts: { - readonly projectRoot: string; - readonly composeFile: string; - readonly acceptTrackedChanges: boolean; -}): Promise { - const message = - "Remove legacy .hack/.env* env_file references from compose now?"; - if ( - await isGitTrackedOrUnknownProjectPath({ - projectRoot: opts.projectRoot, - path: opts.composeFile, - }) - ) { - return await confirmTrackedEnvChange({ - message: `${message} (git tracks ${relative(opts.projectRoot, opts.composeFile)}; commit the change for teammates)`, - accept: opts.acceptTrackedChanges, - }); - } - return await doctorConfirm({ message, initialValue: true }); -} - -function buildEnvMigrationNote(opts: { - readonly migrated: LegacyProjectEnvMigration; - readonly projectRoot: string; - /** Only the migration without a contract makes every value global. */ - readonly globalDelivery: boolean; -}): string { - const { migrated } = opts; - const notes = [`Wrote ${migrated.wroteFiles.join(", ")}`]; - for (const moved of migrated.movedPlainValues) { - notes.push( - `Moved ${moved.key} to ${relative(opts.projectRoot, moved.path)} (values omitted).` - ); - } - if (migrated.updatedProjectConfig) { - notes.push("Updated .hack/hack.config.json for the modern env format."); - } - if (opts.globalDelivery) { - notes.push( - "Migrated values are global: every service, including one added later, and every host lifecycle hook now receives them." - ); - } - return notes.join("\n"); -} - -/** Describes the env migration and asks whether to run it. */ -async function confirmProjectEnvMigration(opts: { - readonly projectRoot: string; - readonly projectDir: string; - readonly contractPath: string; - readonly hasContract: boolean; - readonly trackedChanges: readonly TrackedEnvMigrationChange[]; -}): Promise { - const { projectDir, contractPath, hasContract } = opts; - note( - [ - "Detected legacy project env files.", - hasContract - ? `- source: ${contractPath} and current local env state` - : `- source: ${resolve(projectDir, PROJECT_ENV_FILENAME)} and its overlays (no ${PROJECT_ENV_CONTRACT_FILENAME}; secret-backend values are not migrated)`, - hasContract - ? `- target: ${resolve(projectDir, "hack.env.default.yaml")}` - : `- target: values go to the gitignored ${resolve(projectDir, "hack.env.local.yaml")} (and hack.env..local.yaml); ${resolve(projectDir, "hack.env.default.yaml")} is written with no values`, - "- runtime: direct env injection by default; .hack/.env becomes compatibility-only", - ...(opts.trackedChanges.length > 0 - ? [ - `- git-tracked files that may change (deletes and Compose edits ask first): ${describeTrackedEnvMigrationChanges({ projectRoot: opts.projectRoot, changes: opts.trackedChanges })}`, - ] - : []), - ].join("\n"), - "env migration" - ); - return await doctorConfirm({ - message: - "Migrate project env from .hack/hack.env.json/.hack/.env to hack.env.*.yaml now?", - initialValue: true, - }); -} - async function maybeMigrateProjectEnvConfig(opts: { readonly startDir: string; readonly acceptTrackedChanges: boolean; @@ -4290,171 +4214,44 @@ async function maybeMigrateProjectEnvConfig(opts: { if (!project) { return; } - - // The migration writes env layers, so it decides the format like - // `hack env set`: a symlinked layer counts as an env layer (never written - // through), and a dangling or non-file layer entry refuses closed with - // E_PATH_CONFLICT instead of being replaced. - if ( - (await projectEnvConfigExistsForMutation({ - projectDir: project.projectDir, - })) && - // A checkout whose legacy files hold values its layers don't deliver still - // needs migrating, for example after a teammate's migration was committed. - !(await canMigrateCheckoutLegacyDotenv({ - projectRoot: project.projectRoot, - projectDir: project.projectDir, - serviceNames: await discoverComposeServiceNames({ - composeFile: project.composeFile, - }), - })) - ) { - note("Project already uses the new env config files.", "doctor"); - await maybeRepairModernProjectComposeEnvReferences({ - projectRoot: project.projectRoot, + const config = await readProjectConfig(project); + const migrationProject = { + projectRoot: project.projectRoot, + projectDir: project.projectDir, + projectName: config.name ?? "project", + serviceNames: await discoverComposeServiceNames({ composeFile: project.composeFile, - projectDir: project.projectDir, - acceptTrackedChanges: opts.acceptTrackedChanges, - }); - await printMigrationProtectionHint(project); - return; - } - - // A project without the hack.env.json contract keeps its values only in - // .hack/.env and .hack/.env.; it is migrated from those files. - // `hack env set`/`unset --service` refusals in such a project point here. - const contractPath = resolve( - project.projectDir, - PROJECT_ENV_CONTRACT_FILENAME + }), + }; + const preview = await previewEnvMigration(migrationProject); + const tracked = preview.files.filter( + (file) => file.tracked && file.action !== "removable" ); - const contract = await readHackEnvContract({ - projectDir: project.projectDir, - }); - const hasContract = contract.exists; - - const cfg = await readProjectConfig(project); - const projectName = ( - cfg.name ?? - project.projectRoot.split("/").at(-1) ?? - "project" - ).trim(); - const serviceNames = await discoverComposeServiceNames({ - composeFile: project.composeFile, - }); - // The migration refuses this contract, so don't describe it or ask first. - if ( - contract.parseError === undefined && - !(await confirmProjectEnvMigration({ - projectRoot: project.projectRoot, - projectDir: project.projectDir, - contractPath, - hasContract, - trackedChanges: await planTrackedEnvMigrationChanges({ - projectRoot: project.projectRoot, - projectDir: project.projectDir, - composeFile: project.composeFile, - }), - })) - ) { + const accept = + opts.acceptTrackedChanges || + tracked.length === 0 || + (await confirmTrackedEnvChange({ + message: "Apply the previewed env migration changes to tracked files?", + accept: false, + })); + if (!accept) { return; } - - const migrated = await migrateLegacyProjectEnv({ - projectRoot: project.projectRoot, - projectDir: project.projectDir, - projectName, - serviceNames, - materialize: false, - migrateUndeclaredDotenv: true, - composeRepairFollows: true, + const result = await applyEnvMigration({ + project: migrationProject, + acceptTrackedChanges: accept, + snapshots: await envMigrationSnapshots(), }); - if (migrated.wroteFiles.length === 0) { - note("Legacy env migration found nothing to write.", "doctor"); - await printMigrationProtectionHint(project); - return; - } note( - buildEnvMigrationNote({ - migrated, - projectRoot: project.projectRoot, - globalDelivery: !hasContract, - }), - "doctor" + result.status === "already_complete" + ? "Env migration already completed; no files were changed." + : "Env migration completed. Legacy files were kept for a separate cleanup step.", + "env migration" + ); + note( + `Journal: ${result.journalPath}\nRollback snapshot: ${result.manifestPath}\nAfter validating the project, preview cleanup with: hack env migrate --cleanup --path ${JSON.stringify(project.projectRoot)}`, + "env migration" ); - - for (const retained of migrated.retainedEnvFiles) { - note( - describeRetainedLegacyEnvFile({ shownPath: retained.path, retained }), - "env cleanup" - ); - } - if (migrated.keptSecretStorePointer) { - note( - describeKeptSecretStorePointer({ - shownConfigPath: migrated.keptSecretStorePointer.configPath, - shownStorePath: migrated.keptSecretStorePointer.storePath, - pointer: migrated.keptSecretStorePointer, - }), - "env cleanup" - ); - } - - let composeRepaired = false; - if (migrated.composeEnvFileReferences.length > 0) { - note( - buildComposeEnvRepairNote({ - references: migrated.composeEnvFileReferences, - }), - "compose env repair" - ); - const shouldRepairCompose = await confirmComposeEnvRepair({ - projectRoot: project.projectRoot, - composeFile: project.composeFile, - acceptTrackedChanges: opts.acceptTrackedChanges, - }); - if (shouldRepairCompose) { - const repaired = await repairLegacyComposeEnvFileReferences({ - composeFile: project.composeFile, - projectDir: project.projectDir, - }); - if (repaired.changed) { - note( - [ - `Updated ${project.composeFile}.`, - ...repaired.removed.map( - (reference) => - `- ${reference.service}: removed env_file ${reference.configuredPath}` - ), - ].join("\n"), - "compose env repair" - ); - composeRepaired = true; - } else { - note( - "No compose env_file updates were needed; legacy compatibility files remain blocked from cleanup.", - "compose env repair" - ); - } - } else { - note( - "Leaving legacy .hack/.env* files in place because compose still references them.", - "compose env repair" - ); - } - } - - const cleanupCandidates = doctorEnvCleanupCandidates({ - migrated, - composeRepaired, - }); - await offerDoctorEnvCleanup({ - projectRoot: project.projectRoot, - projectDir: project.projectDir, - projectName, - paths: cleanupCandidates, - context: migrated.cleanupContext, - acceptTrackedChanges: opts.acceptTrackedChanges, - }); await printMigrationProtectionHint(project); } @@ -4489,97 +4286,6 @@ async function printMigrationProtectionHint(project: { ); } -/** - * Offers the legacy env files a migration left for removal. Untracked files - * keep doctor's usual prompt; git-tracked ones get their own prompt and, - * without one, are removed only with `--accept-tracked-env-changes`. - */ -async function offerDoctorEnvCleanup(opts: { - readonly projectRoot: string; - readonly projectDir: string; - readonly projectName: string; - readonly paths: readonly string[]; - readonly context?: LegacyProjectEnvMigration["cleanupContext"]; - readonly acceptTrackedChanges: boolean; -}): Promise { - const untracked: string[] = []; - const tracked: string[] = []; - for (const path of opts.paths) { - ((await isGitTrackedOrUnknownProjectPath({ - projectRoot: opts.projectRoot, - path, - })) - ? tracked - : untracked - ).push(path); - } - - // A key cannot be removed while its tracked store still exists. Include an - // untracked key in that store's later consent batch, keeping the dependency - // intact whether tracked cleanup is accepted, declined or refused. - const store = opts.context?.store; - if (store && tracked.includes(store.path)) { - const keyIndex = untracked.indexOf(store.keyPath); - if (keyIndex !== -1) { - tracked.push(...untracked.splice(keyIndex, 1)); - } - } - - if (untracked.length > 0) { - note( - [ - "Legacy env artifacts still exist and can now be removed:", - ...untracked.map((path) => `- ${path}`), - ].join("\n"), - "env cleanup" - ); - if ( - await doctorConfirm({ - message: - "Remove legacy env files and obsolete project env config entries now?", - initialValue: true, - }) - ) { - await removeDoctorEnvCleanup({ - projectRoot: opts.projectRoot, - projectDir: opts.projectDir, - projectName: opts.projectName, - paths: untracked, - context: opts.context, - }); - } - } - - if (tracked.length > 0) { - note( - [ - "These legacy env files are tracked by git, or are keys kept with a tracked store, and can now be removed. Teammates share the tracked files, so commit their removal together with the new env layers:", - ...tracked.map((path) => `- ${path}`), - ].join("\n"), - "env cleanup" - ); - if ( - await confirmTrackedEnvChange({ - message: "Remove these git-tracked legacy env files now?", - accept: opts.acceptTrackedChanges, - }) - ) { - await removeDoctorEnvCleanup({ - projectRoot: opts.projectRoot, - projectDir: opts.projectDir, - projectName: opts.projectName, - paths: tracked, - context: opts.context, - }); - } else { - note( - "Kept the git-tracked legacy env files. Remove them with `git rm` when teammates have migrated.", - "env cleanup" - ); - } - } -} - /** * Removes confirmed legacy env files. Once the encrypted secret store itself is * gone, its pointer leaves `hack.config.json` too; the migration kept it so a @@ -4655,73 +4361,6 @@ export function doctorEnvCleanupCandidates(opts: { ].sort((left, right) => left.localeCompare(right)); } -function buildComposeEnvRepairNote(opts: { - readonly references: readonly LegacyComposeEnvFileReference[]; -}): string { - return [ - "Compose still references legacy Hack compatibility env files:", - ...opts.references.map( - (reference) => - `- ${reference.service}: ${reference.configuredPath} (${reference.resolvedPath})` - ), - "Hack now injects env directly at runtime by default, so these env_file entries should usually be removed.", - ].join("\n"); -} - -async function maybeRepairModernProjectComposeEnvReferences(opts: { - readonly projectRoot: string; - readonly composeFile: string; - readonly projectDir: string; - readonly acceptTrackedChanges: boolean; -}): Promise { - const references = await inspectLegacyComposeEnvFileReferences({ - composeFile: opts.composeFile, - projectDir: opts.projectDir, - }); - if (references.length === 0) { - return; - } - - note( - buildComposeEnvRepairNote({ - references, - }), - "compose env repair" - ); - const shouldRepairCompose = await confirmComposeEnvRepair({ - projectRoot: opts.projectRoot, - composeFile: opts.composeFile, - acceptTrackedChanges: opts.acceptTrackedChanges, - }); - if (!shouldRepairCompose) { - note( - "Leaving legacy .hack/.env* files in place because compose still references them.", - "compose env repair" - ); - return; - } - - const repaired = await repairLegacyComposeEnvFileReferences({ - composeFile: opts.composeFile, - projectDir: opts.projectDir, - }); - if (!repaired.changed) { - note("No compose env_file updates were needed.", "compose env repair"); - return; - } - - note( - [ - `Updated ${opts.composeFile}.`, - ...repaired.removed.map( - (reference) => - `- ${reference.service}: removed env_file ${reference.configuredPath}` - ), - ].join("\n"), - "compose env repair" - ); -} - export async function buildDoctorRemediationPlanLines(opts: { readonly startDir: string; readonly migrateEnvConfig: boolean; diff --git a/src/commands/env-migrate.ts b/src/commands/env-migrate.ts new file mode 100644 index 000000000..343ad3dca --- /dev/null +++ b/src/commands/env-migrate.ts @@ -0,0 +1,145 @@ +import { resolve } from "node:path"; +import { + type CliContext, + CliUsageError, + type CommandArgs, + defineCommand, + defineOption, + withHandler, +} from "../cli/command.ts"; +import { optJson, optPath } from "../cli/options.ts"; +import { emitCliResult, okResult } from "../lib/cli-result.ts"; +import { + applyEnvMigration, + cleanupEnvMigration, + previewEnvMigration, + rollbackEnvMigration, +} from "../lib/env-migration.ts"; +import { envMigrationSnapshots } from "../lib/env-migration-snapshot.ts"; +import { MissingProjectContextError } from "../lib/missing-project-error.ts"; +import { findProjectContext, readProjectConfig } from "../lib/project.ts"; +import { discoverComposeServiceNames } from "../lib/project-env-config.ts"; + +const optApply = defineOption({ + name: "apply", + type: "boolean", + long: "--apply", + description: + "Apply the previewed migration with a snapshot and resumable journal", +} as const); +const optCleanup = defineOption({ + name: "cleanup", + type: "boolean", + long: "--cleanup", + description: + "Preview later legacy cleanup; combine with --apply only after migration has completed", +} as const); +const optRollback = defineOption({ + name: "rollbackManifest", + type: "string", + long: "--rollback-manifest", + valueHint: "", + description: + "Restore the migration snapshot, refusing changes made afterward", +} as const); +const optAccept = defineOption({ + name: "acceptTrackedChanges", + type: "boolean", + long: "--accept-tracked-env-changes", + description: + "Consent to the displayed edits or removals of Git-tracked legacy env files", +} as const); + +const options = [ + optPath, + optJson, + optApply, + optCleanup, + optRollback, + optAccept, +] as const; +const spec = defineCommand({ + name: "migrate", + summary: + "Preview, resume or undo project env migration; cleanup is a separate later step", + group: "Integrations", + options, + positionals: [], + subcommands: [], +} as const); + +export async function handleEnvMigrate({ + ctx, + args, +}: { + readonly ctx: CliContext; + readonly args: CommandArgs; +}): Promise { + const opts = args.options; + if ( + opts.rollbackManifest !== undefined && + (opts.apply || opts.cleanup || opts.acceptTrackedChanges) + ) { + throw new CliUsageError( + "--rollback-manifest cannot be combined with --apply, --cleanup or --accept-tracked-env-changes." + ); + } + if (opts.acceptTrackedChanges && !opts.apply) { + throw new CliUsageError("--accept-tracked-env-changes requires --apply."); + } + const context = await findProjectContext(resolve(ctx.cwd, opts.path ?? ".")); + if (!context) { + throw new MissingProjectContextError(); + } + const config = await readProjectConfig(context); + const project = { + projectRoot: context.projectRoot, + projectDir: context.projectDir, + projectName: config.name ?? "project", + serviceNames: await discoverComposeServiceNames({ + composeFile: context.composeFile, + }), + }; + let action = "preview"; + if (opts.apply) { + action = "apply"; + } + if (opts.cleanup) { + action = "cleanup"; + } + if (opts.rollbackManifest) { + action = "rollback"; + } + let data: unknown; + if (opts.rollbackManifest) { + data = await rollbackEnvMigration({ + project, + manifestPath: resolve(ctx.cwd, opts.rollbackManifest), + snapshots: await envMigrationSnapshots(), + }); + } else if (opts.apply) { + data = await (opts.cleanup ? cleanupEnvMigration : applyEnvMigration)({ + project, + acceptTrackedChanges: opts.acceptTrackedChanges === true, + snapshots: await envMigrationSnapshots(), + }); + } else { + const preview = await previewEnvMigration(project); + data = { + ...preview, + action, + dryRun: true, + files: opts.cleanup + ? preview.files.filter((entry) => entry.action === "removable") + : preview.files, + }; + } + if (opts.json) { + emitCliResult({ result: okResult({ data }) }); + return 0; + } + process.stdout.write(`${JSON.stringify(data, null, 2)}\n`); + return 0; +} + +export const envMigrateCommand = withHandler(spec, handleEnvMigrate); diff --git a/src/commands/env.ts b/src/commands/env.ts index e6d82a697..46dde3a4c 100644 --- a/src/commands/env.ts +++ b/src/commands/env.ts @@ -28,6 +28,10 @@ import { parseEnvHydrateInputFormat, readEnvHydrateInput, } from "../lib/env-hydrate-input.ts"; +import { + hasPendingEnvMigration, + migrateLegacyEnvForCommand as migrateLegacyProjectEnv, +} from "../lib/env-migration.ts"; import { assertEnvValueStdinUsage, readEnvValueFromStdin, @@ -88,7 +92,6 @@ import { hydrateProjectEnvLocalLayer, listProjectEnvPlaintextEntries, materializeProjectEnv, - migrateLegacyProjectEnv, type ProjectEnvResolvedMetadata, parseProjectEnvTarget, projectEnvConfigExists, @@ -112,6 +115,7 @@ import { import { run } from "../lib/shell.ts"; import { display } from "../ui/display.ts"; import { logger } from "../ui/logger.ts"; +import { envMigrateCommand } from "./env-migrate.ts"; import { hostPsCommand } from "./host-ps.ts"; const optShowSecrets = defineOption({ @@ -992,9 +996,10 @@ async function maybeMigrateLegacyProjectEnv(input: { readonly reason: "mutation" | "runtime"; }): Promise { if ( - await projectEnvConfigExists({ + !(await hasPendingEnvMigration(input.project)) && + (await projectEnvConfigExists({ projectDir: input.project.projectDir, - }) + })) ) { return false; } @@ -2879,6 +2884,7 @@ export const envCommand = defineCommand({ options: [], positionals: [], subcommands: [ + envMigrateCommand, withHandler(listSpec, handleEnvList), withHandler(protectSpec, handleEnvProtect), withHandler(getSpec, handleEnvGet), diff --git a/src/commands/project.ts b/src/commands/project.ts index bb288ed57..297e07fbb 100644 --- a/src/commands/project.ts +++ b/src/commands/project.ts @@ -1,3 +1,8 @@ +import { + hasPendingEnvMigration, + migrateLegacyEnvForCommand as migrateLegacyProjectEnv, + recordEnvMigrationProjectRun, +} from "../lib/env-migration.ts"; import { createLifecycleProcessController, installLifecycleSignalCleanup, @@ -320,7 +325,6 @@ import { finishImplicitLegacyEnvMigration, HACK_DIR_GITIGNORE_READ_ONLY_WARNING, hasUnmigratedLegacyDotenv, - migrateLegacyProjectEnv, projectEnvConfigExists, resolveProjectEnvConfig, selectProjectEnvValuesForExecutionTarget, @@ -1380,9 +1384,10 @@ async function maybePromptLegacyProjectEnvMigration(opts: { readonly serviceNames: readonly string[]; }): Promise { if ( - await projectEnvConfigExists({ + !(await hasPendingEnvMigration(opts.project)) && + (await projectEnvConfigExists({ projectDir: opts.project.projectDir, - }) + })) ) { return false; } @@ -6144,6 +6149,20 @@ async function handleNativeUp({ "Native up lifecycle readiness hook failed; values omitted.", }); } + + await recordEnvMigrationProjectRun({ + projectRoot: project.projectRoot, + projectDir: project.projectDir, + projectName, + serviceNames: await discoverComposeServiceNames({ + composeFile: project.composeFile, + }), + }).catch(() => + logger.warn({ + message: + "Could not record successful env migration startup; legacy cleanup remains blocked (values omitted).", + }) + ); }, }; }, @@ -6966,6 +6985,19 @@ async function runUpCommand({ exitCode: 1, }); } + if (!serviceScoped) { + await recordEnvMigrationProjectRun({ + projectRoot: project.projectRoot, + projectDir: project.projectDir, + projectName: baseProjectName, + serviceNames: allServiceNames, + }).catch(() => + logger.warn({ + message: + "Could not record successful env migration startup; legacy cleanup remains blocked (values omitted).", + }) + ); + } if (!json) { return 0; } diff --git a/src/commands/session.ts b/src/commands/session.ts index 429d08bc9..0824d23d7 100644 --- a/src/commands/session.ts +++ b/src/commands/session.ts @@ -13,6 +13,10 @@ import { } from "../cli/command.ts"; import { resolveRequestedEnvName } from "../cli/env-selection.ts"; import { optEnv, optJson, optPretty } from "../cli/options.ts"; +import { + hasPendingEnvMigration, + migrateLegacyEnvForCommand as migrateLegacyProjectEnv, +} from "../lib/env-migration.ts"; import { readHackEnvContract, resolveHackEnv, @@ -31,7 +35,6 @@ import { assertValidProjectEnvScopeName, discoverComposeServiceNames, finishImplicitLegacyEnvMigration, - migrateLegacyProjectEnv, projectEnvConfigExists, resolveProjectEnvConfig, selectProjectEnvValues, @@ -1047,9 +1050,10 @@ async function maybeMigrateLegacySessionEnv(opts: { readonly projectName: string; }): Promise { if ( - await projectEnvConfigExists({ + !(await hasPendingEnvMigration(opts.project)) && + (await projectEnvConfigExists({ projectDir: opts.project.projectDir, - }) + })) ) { return; } diff --git a/src/lib/env-migration-files.ts b/src/lib/env-migration-files.ts new file mode 100644 index 000000000..b6f9df5bc --- /dev/null +++ b/src/lib/env-migration-files.ts @@ -0,0 +1,83 @@ +import { createHash } from "node:crypto"; +import { constants } from "node:fs"; +import { type FileHandle, lstat, open } from "node:fs/promises"; +import { ClosedCliError } from "./cli-result.ts"; +import type { EnvMigrationFileState } from "./env-migration-snapshot.ts"; +import { isRecord } from "./guards.ts"; + +const FILE_LIMIT = 32 * 1024 * 1024; + +export function migrationRefusal(): ClosedCliError { + return new ClosedCliError( + "Env migration refused: an input, journal, payload or destination is unsafe, changed or unreadable. Keep the legacy files and snapshot; values omitted." + ); +} + +export function migrationDigest(bytes: Uint8Array): string { + return createHash("sha256").update(bytes).digest("hex"); +} + +/** Bounded descriptor read: no final symlink, FIFO, device, or changing file is accepted. */ +export async function readMigrationFile(path: string): Promise<{ + readonly state: EnvMigrationFileState; + readonly text: string | null; +}> { + let handle: FileHandle; + try { + handle = await open( + path, + constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK + ); + } catch (error) { + if (isRecord(error) && error.code === "ENOENT") { + return { state: { path, sha256: null, mode: null, size: 0 }, text: null }; + } + throw migrationRefusal(); + } + try { + const before = await handle.stat(); + if (!before.isFile() || before.size > FILE_LIMIT) { + throw migrationRefusal(); + } + const bytes = await handle.readFile(); + const after = await handle.stat(); + const named = await lstat(path); + if ( + before.dev !== after.dev || + before.ino !== named.ino || + before.dev !== named.dev || + named.isSymbolicLink() || + before.size !== bytes.length || + before.mtimeMs !== after.mtimeMs || + before.ctimeMs !== after.ctimeMs || + after.mtimeMs !== named.mtimeMs || + after.ctimeMs !== named.ctimeMs + ) { + throw migrationRefusal(); + } + return { + state: { + path, + sha256: migrationDigest(bytes), + mode: before.mode & 0o777, + size: bytes.length, + }, + text: new TextDecoder("utf-8", { fatal: true }).decode(bytes), + }; + } catch { + throw migrationRefusal(); + } finally { + await handle.close(); + } +} + +export function sameMigrationFile( + left: EnvMigrationFileState, + right: EnvMigrationFileState +): boolean { + return ( + left.sha256 === right.sha256 && + left.mode === right.mode && + left.size === right.size + ); +} diff --git a/src/lib/env-migration-snapshot.ts b/src/lib/env-migration-snapshot.ts new file mode 100644 index 000000000..1d2658424 --- /dev/null +++ b/src/lib/env-migration-snapshot.ts @@ -0,0 +1,229 @@ +import { lstat } from "node:fs/promises"; +import { join } from "node:path"; +import { ClosedCliError } from "./cli-result.ts"; +import { + migrationRefusal, + readMigrationFile, + sameMigrationFile, +} from "./env-migration-files.ts"; +import { isRecord } from "./guards.ts"; + +/** Metadata only. An absent path has a null checksum and mode. */ +export type EnvMigrationFileState = { + readonly path: string; + readonly sha256: string | null; + readonly mode: number | null; + readonly size: number; +}; + +/** Small integration boundary owned by HACK-2189's private snapshot library. */ +export type EnvMigrationSnapshots = { + readonly capture: (input: { + readonly projectRoot: string; + readonly files: readonly EnvMigrationFileState[]; + }) => Promise<{ readonly manifestPath: string }>; + /** Must verify payload checksums, not merely the existence of a manifest. */ + readonly assertCoverage: (input: { + readonly manifestPath: string; + readonly files: readonly EnvMigrationFileState[]; + }) => Promise; + readonly recordAfter: (input: { + readonly manifestPath: string; + readonly files: readonly EnvMigrationFileState[]; + }) => Promise; + readonly rollback: (input: { + readonly manifestPath: string; + readonly expectedCurrent: readonly EnvMigrationFileState[]; + }) => Promise<{ + readonly status: "restored" | "incomplete"; + readonly results: readonly unknown[]; + }>; +}; + +/** Late binding keeps this unit independent until HACK-2189 lands; replace with a typed static import on integration. */ +export async function envMigrationSnapshots(): Promise { + const moduleUrl = new URL("./upgrade-snapshot.ts", import.meta.url).href; + let library: unknown; + try { + library = await import(moduleUrl); + } catch { + throw new ClosedCliError( + "Env migration requires the upgrade snapshot library. Preview is available; no migration files were changed." + ); + } + return bindEnvMigrationSnapshots(library); +} + +/** Adapter checks both the snapshot owner's verdict and the journal's exact expectations. */ +export function bindEnvMigrationSnapshots( + library: unknown +): EnvMigrationSnapshots { + if (!isRecord(library)) { + throw migrationRefusal(); + } + const invoke = async (name: string, input: unknown): Promise => { + const operation = library[name]; + if (typeof operation !== "function") { + throw migrationRefusal(); + } + return await Reflect.apply(operation, undefined, [input]); + }; + const load = async (manifestPath: string) => { + const loaded = await invoke("readUpgradeSnapshot", { manifestPath }); + if ( + !( + isRecord(loaded) && + loaded.status === "ok" && + isRecord(loaded.snapshot) && + isRecord(loaded.snapshot.manifest) && + Array.isArray(loaded.snapshot.manifest.entries) && + typeof loaded.snapshot.dir === "string" + ) + ) { + throw migrationRefusal(); + } + return { + dir: loaded.snapshot.dir, + entries: loaded.snapshot.manifest.entries as unknown[], + }; + }; + const matches = (state: unknown, expected: EnvMigrationFileState) => + isRecord(state) && + (expected.sha256 === null + ? state.kind === "absent" + : state.kind === "file" && + state.sha256 === expected.sha256 && + state.size === expected.size && + state.mode === expected.mode); + const verifyCurrent = async (files: readonly EnvMigrationFileState[]) => { + for (const expected of files) { + if ( + !sameMigrationFile( + (await readMigrationFile(expected.path)).state, + expected + ) + ) { + throw migrationRefusal(); + } + } + }; + const collectPending = async ( + snapshot: Awaited>, + expectedCurrent: readonly EnvMigrationFileState[] + ) => { + const pending: EnvMigrationFileState[] = []; + for (const expected of expectedCurrent) { + const current = (await readMigrationFile(expected.path)).state; + const original = snapshot.entries.find( + (entry) => isRecord(entry) && entry.path === expected.path + ); + if (isRecord(original) && matches(original.before, current)) { + continue; + } + if (!sameMigrationFile(current, expected)) { + throw migrationRefusal(); + } + pending.push(expected); + } + return pending; + }; + return { + capture: async ({ projectRoot, files }) => { + const result = await invoke("createUpgradeSnapshot", { + purpose: "env-migration", + targets: files.map((file) => ({ + path: file.path, + label: `env migration: ${file.path}`, + })), + }); + if ( + !( + isRecord(result) && + result.status === "created" && + isRecord(result.snapshot) && + typeof result.snapshot.manifestPath === "string" + ) + ) { + throw new ClosedCliError( + `Snapshot refused for ${projectRoot}; no migration files were changed (values omitted).` + ); + } + return { manifestPath: result.snapshot.manifestPath }; + }, + assertCoverage: async ({ manifestPath, files }) => { + const snapshot = await load(manifestPath); + for (const expected of files) { + const entry = snapshot.entries.find( + (candidate) => isRecord(candidate) && candidate.path === expected.path + ); + if (!(isRecord(entry) && matches(entry.before, expected))) { + throw migrationRefusal(); + } + if (expected.sha256 !== null) { + const path = join(snapshot.dir, "blobs", expected.sha256); + const info = await lstat(path); + if (info.uid !== process.getuid?.() || (info.mode & 0o077) !== 0) { + throw migrationRefusal(); + } + if ( + (await readMigrationFile(path)).state.sha256 !== expected.sha256 + ) { + throw migrationRefusal(); + } + } + } + }, + recordAfter: async ({ manifestPath, files }) => { + await verifyCurrent(files); + const result = await invoke("recordUpgradeSnapshotAfter", { + manifestPath, + paths: files.map((file) => file.path), + }); + if (!(isRecord(result) && result.status === "recorded")) { + throw migrationRefusal(); + } + await verifyCurrent(files); + }, + rollback: async ({ manifestPath, expectedCurrent }) => { + const snapshot = await load(manifestPath); + const pending = await collectPending(snapshot, expectedCurrent); + // A killed publication may predate the provider's after receipt; only journal-proven bytes may fill that gap. + const recorded = await invoke("recordUpgradeSnapshotAfter", { + manifestPath, + paths: pending.map((file) => file.path), + }); + if (!(isRecord(recorded) && recorded.status === "recorded")) { + throw migrationRefusal(); + } + const results: unknown[] = []; + let incomplete = false; + for (const target of expectedCurrent) { + const result = await invoke("restoreUpgradeSnapshot", { + manifestPath, + paths: [target.path], + beforeReplace: async (path: string) => { + const expected = pending.find((file) => file.path === path); + if (expected) { + await verifyCurrent([expected]); + } + }, + }); + if ( + !( + isRecord(result) && + (result.status === "restored" || result.status === "incomplete") && + Array.isArray(result.results) + ) + ) { + throw migrationRefusal(); + } + results.push(...result.results); + if (result.status !== "restored") { + incomplete = true; + break; + } + } + return { status: incomplete ? "incomplete" : "restored", results }; + }, + }; +} diff --git a/src/lib/env-migration.ts b/src/lib/env-migration.ts new file mode 100644 index 000000000..ab0c02142 --- /dev/null +++ b/src/lib/env-migration.ts @@ -0,0 +1,1023 @@ +import { createHash, randomUUID } from "node:crypto"; +import { lstat, realpath } from "node:fs/promises"; +import { + basename, + dirname, + isAbsolute, + join, + relative, + resolve, + sep, +} from "node:path"; +import { ClosedCliError } from "./cli-result.ts"; +import { resolveGlobalHackDir } from "./config-paths.ts"; +import { withEnvLayerLock } from "./env-layer-lock.ts"; +import { + migrationDigest as digest, + readMigrationFile as readState, + migrationRefusal as refuse, + sameMigrationFile as same, +} from "./env-migration-files.ts"; +import type { + EnvMigrationFileState, + EnvMigrationSnapshots, +} from "./env-migration-snapshot.ts"; +import { envMigrationSnapshots } from "./env-migration-snapshot.ts"; +import { isRecord } from "./guards.ts"; +import { + ensurePrivateDirectory, + replacePrivateFileAtomic, +} from "./private-file.ts"; +import { + acquireLegacyEnvCleanupContext, + assertMachineLocalFileGitPrivate, + assertProjectEnvLayerPaths, + discoverComposeServiceNames, + isGitTrackedOrUnknownProjectPath, + LegacyEnvCleanupError, + type LegacyProjectEnvMigration, + type PreparedLegacyEnvMigration, + prepareLegacyEnvCleanupConfig, + prepareLegacyProjectEnvMigration, + removeLegacyProjectEnvArtifacts, + resolveProjectEnvConfig, + resolveProjectEnvSharedKeyLocation, +} from "./project-env-config.ts"; +import { withProjectsRegistryLock } from "./projects-registry-lock.ts"; + +export const ENV_MIGRATION_VERSION = 1 as const; +const FILE_LIMIT = 32 * 1024 * 1024; +const SHA256 = /^[a-f0-9]{64}$/; +const PAYLOAD_NAME = /^[0-9]+\.payload$/; +const ENV_KEY = /^[A-Z_][A-Z0-9_]*$/; + +export type EnvMigrationProject = { + readonly projectRoot: string; + readonly projectDir: string; + readonly projectName: string; + readonly serviceNames: readonly string[]; +}; + +export type EnvMigrationPreview = { + readonly version: 1; + readonly project: EnvMigrationProject; + readonly status: "ready" | "complete"; + readonly files: readonly { + readonly path: string; + readonly action: "write" | "change" | "removable"; + readonly keys: readonly string[]; + readonly tracked: boolean; + }[]; + readonly journalPath: string; +}; + +type Publication = EnvMigrationFileState & { + readonly kind: PreparedLegacyEnvMigration["files"][number]["kind"]; + readonly keys: readonly string[]; + readonly before: EnvMigrationFileState; + readonly payload: string; + readonly done: boolean; +}; + +type Journal = { + readonly version: 1; + readonly kind: "hack-env-migration"; + readonly project: EnvMigrationProject; + readonly manifestPath: string; + readonly state: + | "applying" + | "complete" + | "cleaning" + | "cleaned" + | "rolled_back"; + readonly sources: readonly EnvMigrationFileState[]; + readonly writes: readonly Publication[]; + readonly removable: readonly string[]; + readonly overlayNames: readonly string[]; + readonly store?: { readonly path: string; readonly keyPath: string }; + readonly removed: readonly string[]; + /** Layer revision observed after a complete project startup succeeded. */ + readonly validated?: readonly EnvMigrationFileState[]; + readonly cleanupConfig?: Publication; +}; + +/** Test-only scheduling seam; events contain no payloads and production does not consult env flags. */ +export type EnvMigrationHooks = { + readonly boundary?: (event: { + readonly phase: + | "prepared" + | "before_publish" + | "published" + | "recorded" + | "verified" + | "complete"; + readonly index: number; + }) => Promise; +}; + +async function canonicalProject( + project: EnvMigrationProject +): Promise { + const projectRoot = await realpath(project.projectRoot); + const projectDir = await realpath(project.projectDir); + if ( + ![".hack", ".dev"].some((name) => projectDir === join(projectRoot, name)) + ) { + throw refuse(); + } + return { ...project, projectRoot, projectDir }; +} + +/** Location calculation is read-only, including when the data home has not been created. */ +export function envMigrationJournalPath(projectRoot: string): string { + const id = createHash("sha256").update(projectRoot).digest("hex"); + return join(resolveGlobalHackDir(), "env-migrations", id, "journal.json"); +} + +/** Implicit routes must resume a journal even after its first YAML file exists. */ +export async function hasPendingEnvMigration(input: { + readonly projectRoot: string; + readonly projectDir: string; +}): Promise { + const project = await canonicalProject({ + ...input, + projectName: "project", + serviceNames: [], + }); + return ( + (await readJournal(envMigrationJournalPath(project.projectRoot), project)) + ?.state === "applying" + ); +} + +async function assertOutsideWorkingTrees( + path: string, + project: EnvMigrationProject +): Promise { + if (!isAbsolute(path)) { + throw refuse(); + } + const inside = relative(project.projectRoot, path); + if ( + !(inside === ".." || inside.startsWith(`..${sep}`) || isAbsolute(inside)) + ) { + throw refuse(); + } + for (let parent = dirname(path); ; parent = dirname(parent)) { + for (const name of [".git"]) { + try { + await lstat(join(parent, name)); + throw refuse(); + } catch (error) { + if (!(isRecord(error) && error.code === "ENOENT")) { + throw error; + } + } + } + if (dirname(parent) === parent) { + break; + } + } +} + +async function privateDirectory(path: string): Promise { + await ensurePrivateDirectory(path); + await checkPrivateDirectory(path); +} + +async function checkPrivateDirectory(path: string): Promise { + const info = await lstat(path); + if ( + !info.isDirectory() || + info.isSymbolicLink() || + info.uid !== process.getuid?.() || + (info.mode & 0o077) !== 0 || + (await realpath(path)) !== path + ) { + throw refuse(); + } +} + +async function writeMetadata(path: string, value: unknown): Promise { + await replacePrivateFileAtomic({ + path, + directories: [dirname(path)], + text: `${JSON.stringify(value, null, 2)}\n`, + mode: 0o600, + refusal: refuse, + }); +} + +function validState( + raw: unknown +): raw is EnvMigrationFileState & Record { + return ( + isRecord(raw) && + typeof raw.path === "string" && + isAbsolute(raw.path) && + (raw.sha256 === null || + (typeof raw.sha256 === "string" && SHA256.test(raw.sha256))) && + (raw.mode === null || + (Number.isInteger(raw.mode) && + Number(raw.mode) >= 0 && + Number(raw.mode) <= 0o777)) && + Number.isSafeInteger(raw.size) && + Number(raw.size) >= 0 && + Number(raw.size) <= FILE_LIMIT + ); +} + +function validPublication(entry: unknown, path: string): entry is Publication { + return ( + validState(entry) && + validState(entry.before) && + entry.before.path === entry.path && + entry.sha256 !== null && + typeof entry.payload === "string" && + isAbsolute(entry.payload) && + dirname(dirname(entry.payload)) === dirname(path) && + basename(dirname(entry.payload)).startsWith("payloads-") && + PAYLOAD_NAME.test(basename(entry.payload)) && + ["ignore", "key", "local_layer", "layer", "config", "compose"].includes( + String(entry.kind) + ) && + typeof entry.done === "boolean" && + Array.isArray(entry.keys) && + entry.keys.every((key) => typeof key === "string" && ENV_KEY.test(key)) + ); +} + +async function readJournal( + path: string, + project: EnvMigrationProject +): Promise { + const read = await readState(path); + if (read.text === null) { + return null; + } + await checkPrivateDirectory(dirname(path)); + await checkPrivateDirectory(dirname(dirname(path))); + const info = await lstat(path); + if ( + info.uid !== process.getuid?.() || + (info.mode & 0o077) !== 0 || + info.nlink !== 1 + ) { + throw refuse(); + } + let raw: unknown; + try { + raw = JSON.parse(read.text); + } catch { + throw refuse(); + } + if ( + !( + isRecord(raw) && + raw.version === 1 && + raw.kind === "hack-env-migration" && + isRecord(raw.project) && + raw.project.projectRoot === project.projectRoot && + raw.project.projectDir === project.projectDir && + typeof raw.manifestPath === "string" && + isAbsolute(raw.manifestPath) && + ["applying", "complete", "cleaning", "cleaned", "rolled_back"].includes( + String(raw.state) + ) && + Array.isArray(raw.sources) && + raw.sources.every(validState) && + Array.isArray(raw.writes) && + raw.writes.every((entry) => validPublication(entry, path)) && + Array.isArray(raw.removable) && + raw.removable.every( + (entry) => typeof entry === "string" && isAbsolute(entry) + ) && + Array.isArray(raw.removed) && + raw.removed.every( + (entry) => Array.isArray(raw.removable) && raw.removable.includes(entry) + ) && + Array.isArray(raw.overlayNames) && + raw.overlayNames.every((entry) => typeof entry === "string") && + (raw.validated === undefined || + (Array.isArray(raw.validated) && raw.validated.every(validState))) && + (raw.cleanupConfig === undefined || + (validPublication(raw.cleanupConfig, path) && + raw.cleanupConfig.kind === "config" && + raw.cleanupConfig.path === + join(project.projectDir, "hack.config.json"))) + ) + ) { + throw refuse(); + } + // The validated wire shape contains metadata only; no source text is accepted. + return raw as Journal; +} + +async function previewPrepared( + project: EnvMigrationProject, + prepared: PreparedLegacyEnvMigration, + path: string +): Promise { + const files: EnvMigrationPreview["files"][number][] = []; + for (const file of prepared.files) { + files.push({ + path: file.path, + action: + (await readState(file.path)).state.sha256 === null ? "write" : "change", + keys: file.keys, + tracked: await isGitTrackedOrUnknownProjectPath({ + ...project, + path: file.path, + }), + }); + } + for (const removable of prepared.removable) { + files.push({ + path: removable, + action: "removable", + keys: [], + tracked: await isGitTrackedOrUnknownProjectPath({ + ...project, + path: removable, + }), + }); + } + return { + version: ENV_MIGRATION_VERSION, + project, + status: prepared.files.length ? "ready" : "complete", + files, + journalPath: path, + }; +} + +type Preparation = { + readonly prepared: PreparedLegacyEnvMigration; + readonly sources: readonly EnvMigrationFileState[]; +}; + +async function prepareConsistent( + project: EnvMigrationProject & { readonly repairCompose?: boolean } +): Promise { + // Discover the complete inventory before the pass whose immutable payloads + // will be published. A changed inventory or input refuses before consent. + const discovered = await prepareLegacyProjectEnvMigration(project); + const sources = await Promise.all( + discovered.sourcePaths.map(async (path) => (await readState(path)).state) + ); + const prepared = await prepareLegacyProjectEnvMigration(project); + if ( + JSON.stringify(discovered.sourcePaths) !== + JSON.stringify(prepared.sourcePaths) + ) { + throw refuse(); + } + for (const source of sources) { + if (!same(source, (await readState(source.path)).state)) { + throw refuse(); + } + } + return { prepared, sources }; +} + +export async function previewEnvMigration( + input: EnvMigrationProject +): Promise { + const project = await canonicalProject(input); + const path = envMigrationJournalPath(project.projectRoot); + const journal = await readJournal(path, project); + if (journal && journal.state !== "rolled_back") { + const files: EnvMigrationPreview["files"][number][] = []; + for (const file of journal.writes) { + files.push({ + path: file.path, + action: file.before.sha256 === null ? "write" : "change", + keys: file.keys, + tracked: await isGitTrackedOrUnknownProjectPath({ + ...project, + path: file.path, + }), + }); + } + for (const removable of journal.removable) { + files.push({ + path: removable, + action: "removable", + keys: [], + tracked: await isGitTrackedOrUnknownProjectPath({ + ...project, + path: removable, + }), + }); + } + return { + version: 1, + project, + status: journal.state === "applying" ? "ready" : "complete", + files, + journalPath: path, + }; + } + return await previewPrepared( + project, + (await prepareConsistent(project)).prepared, + path + ); +} + +async function assertConsent( + project: EnvMigrationProject, + paths: readonly string[], + accept: boolean +): Promise { + if (accept) { + return; + } + for (const path of paths) { + if (await isGitTrackedOrUnknownProjectPath({ ...project, path })) { + throw new ClosedCliError( + "Env migration needs consent to change tracked files. Pass --accept-tracked-env-changes; no files were changed." + ); + } + } +} + +async function assertSources( + journal: Journal, + allowLayerChanges = false +): Promise { + for (const source of journal.sources) { + if (journal.removed.includes(source.path)) { + continue; + } + const current = (await readState(source.path)).state; + const write = journal.writes.find((entry) => entry.path === source.path); + if ( + allowLayerChanges && + (write?.kind === "layer" || write?.kind === "local_layer") + ) { + continue; + } + if ( + write?.done + ? !same(current, write) + : !(same(current, source) || (write && same(current, write))) + ) { + throw refuse(); + } + } +} + +async function prepareJournal( + project: EnvMigrationProject, + path: string, + snapshots: EnvMigrationSnapshots, + { prepared, sources }: Preparation +): Promise { + const snapshot = await snapshots.capture({ + projectRoot: project.projectRoot, + files: sources, + }); + await assertOutsideWorkingTrees(snapshot.manifestPath, project); + await snapshots.assertCoverage({ ...snapshot, files: sources }); + await snapshots.recordAfter({ ...snapshot, files: sources }); + const payloadDir = join(dirname(path), `payloads-${randomUUID()}`); + // A per-run directory is retained on interruption; the journal lists each payload. + await privateDirectory(payloadDir); + const writes: Publication[] = []; + for (const [index, file] of prepared.files.entries()) { + const before = sources.find((entry) => entry.path === file.path); + if (!before) { + throw refuse(); + } + const text = prepared.payload(file.path); + const payload = `${index}.payload`; + await replacePrivateFileAtomic({ + path: join(payloadDir, payload), + directories: [payloadDir], + text, + mode: 0o600, + refusal: refuse, + }); + writes.push({ + ...file, + size: Buffer.byteLength(text), + sha256: digest(Buffer.from(text)), + before, + payload: `${payloadDir}/${payload}`, + done: false, + }); + } + const journal: Journal = { + version: 1, + kind: "hack-env-migration", + project, + manifestPath: snapshot.manifestPath, + state: "applying", + sources, + writes, + removable: prepared.removable, + overlayNames: prepared.overlayNames, + ...(prepared.store ? { store: prepared.store } : {}), + removed: [], + }; + await assertSources(journal); + await writeMetadata(path, journal); + return journal; +} + +async function publishEntry( + project: EnvMigrationProject, + entry: Publication +): Promise { + const run = async () => { + await checkPrivateDirectory(dirname(entry.payload)); + const current = await readState(entry.path); + if (same(current.state, entry)) { + return; + } + if (!same(current.state, entry.before)) { + throw refuse(); + } + const payload = await readState(entry.payload); + const payloadInfo = await lstat(entry.payload); + if ( + payload.state.sha256 !== entry.sha256 || + payload.text === null || + payloadInfo.uid !== process.getuid?.() || + payloadInfo.nlink !== 1 || + (payloadInfo.mode & 0o077) !== 0 + ) { + throw refuse(); + } + if (entry.kind === "local_layer") { + await assertMachineLocalFileGitPrivate({ + ...project, + path: entry.path, + }); + } + if (entry.kind === "layer" || entry.kind === "local_layer") { + await assertProjectEnvLayerPaths(project); + } + await replacePrivateFileAtomic({ + path: entry.path, + directories: [], + text: payload.text, + mode: entry.mode ?? 0o600, + refusal: refuse, + beforeRename: async () => { + if (!same((await readState(entry.path)).state, entry.before)) { + throw refuse(); + } + }, + }); + if (!same((await readState(entry.path)).state, entry)) { + throw refuse(); + } + }; + if (entry.kind === "key") { + const shared = await resolveProjectEnvSharedKeyLocation(project); + await withProjectsRegistryLock({ + lockPath: `${shared?.path ?? entry.path}.lock`, + label: "project env key", + run, + }); + } else if ( + entry.kind === "ignore" && + dirname(entry.path) === project.projectRoot + ) { + await run(); + } else { + await withEnvLayerLock({ layerPath: entry.path, run }); + } +} + +/** Uses recorded before/after images; an unrecorded successful rename is adopted on resume. */ +export async function applyEnvMigration(input: { + readonly project: EnvMigrationProject; + readonly acceptTrackedChanges: boolean; + readonly snapshots: EnvMigrationSnapshots; + readonly hooks?: EnvMigrationHooks; + readonly repairCompose?: boolean; +}) { + const project = await canonicalProject(input.project); + const path = envMigrationJournalPath(project.projectRoot); + await assertOutsideWorkingTrees(path, project); + const existing = await readJournal(path, project); + if (existing && ["complete", "cleaned"].includes(existing.state)) { + return { + status: "already_complete" as const, + journalPath: path, + manifestPath: existing.manifestPath, + }; + } + const preparation = + existing && existing.state !== "rolled_back" + ? null + : await prepareConsistent({ + ...project, + repairCompose: input.repairCompose, + }); + const changed = + existing && existing.state !== "rolled_back" + ? existing.writes.map((entry) => entry.path) + : (preparation?.prepared.files.map((entry) => entry.path) ?? []); + await assertConsent(project, changed, input.acceptTrackedChanges); + await privateDirectory(dirname(dirname(path))); + await privateDirectory(dirname(path)); + return await withProjectsRegistryLock({ + lockPath: `${path}.lock`, + label: "env migration", + run: async () => { + const retained = await readJournal(path, project); + let journal = + retained && retained.state !== "rolled_back" + ? retained + : await prepareJournal( + project, + path, + input.snapshots, + preparation ?? + (await prepareConsistent({ + ...project, + repairCompose: input.repairCompose, + })) + ); + await assertConsent( + project, + journal.writes.map((entry) => entry.path), + input.acceptTrackedChanges + ); + if (journal.state !== "applying") { + return { + status: "already_complete" as const, + journalPath: path, + manifestPath: journal.manifestPath, + }; + } + await input.hooks?.boundary?.({ phase: "prepared", index: -1 }); + await assertSources(journal); + for (const [index, entry] of journal.writes.entries()) { + if (entry.done) { + continue; + } + await input.hooks?.boundary?.({ phase: "before_publish", index }); + await publishEntry(project, entry); + await input.hooks?.boundary?.({ phase: "published", index }); + await input.snapshots.recordAfter({ + manifestPath: journal.manifestPath, + files: [entry], + }); + journal = { + ...journal, + writes: journal.writes.map((write, current) => + current === index ? { ...write, done: true } : write + ), + }; + await writeMetadata(path, journal); + await input.hooks?.boundary?.({ phase: "recorded", index }); + } + await assertSources(journal); + for (const envName of [null, ...journal.overlayNames]) { + if (!(await resolveProjectEnvConfig({ ...project, envName }))) { + throw refuse(); + } + } + await input.hooks?.boundary?.({ phase: "verified", index: -1 }); + journal = { ...journal, state: "complete" }; + await writeMetadata(path, journal); + await input.hooks?.boundary?.({ phase: "complete", index: -1 }); + return { + status: "applied" as const, + journalPath: path, + manifestPath: journal.manifestPath, + }; + }, + }); +} + +/** Explicit later action: a snapshot and fresh legacy-to-current delivery proof are both required. */ +export async function cleanupEnvMigration(input: { + readonly project: EnvMigrationProject; + readonly acceptTrackedChanges: boolean; + readonly snapshots: EnvMigrationSnapshots; +}) { + const project = await canonicalProject(input.project); + const path = envMigrationJournalPath(project.projectRoot); + if (!(await readJournal(path, project))) { + throw refuse(); + } + return await withProjectsRegistryLock({ + lockPath: `${path}.lock`, + label: "env migration", + run: () => cleanupLocked(input, project, path), + }); +} + +async function cleanupLocked( + input: { + readonly acceptTrackedChanges: boolean; + readonly snapshots: EnvMigrationSnapshots; + }, + project: EnvMigrationProject, + path: string +) { + let journal = await readJournal(path, project); + if ( + !journal || + journal.state === "applying" || + journal.state === "rolled_back" + ) { + throw new ClosedCliError( + "Finish the env migration before requesting cleanup; no legacy files were removed." + ); + } + if (journal.state === "cleaned") { + return { + status: "already_cleaned" as const, + journalPath: path, + manifestPath: journal.manifestPath, + removed: journal.removed, + }; + } + if (journal.state === "cleaning") { + throw new ClosedCliError( + "Previous legacy cleanup was interrupted. Restore its snapshot with hack env migrate --rollback-manifest before retrying; remaining files were kept." + ); + } + if (!journal.validated) { + throw new ClosedCliError( + "Start the complete project successfully with hack up before requesting legacy cleanup; no files were removed." + ); + } + for (const validated of journal.validated) { + if (!same((await readState(validated.path)).state, validated)) { + throw new ClosedCliError( + "Env layers changed since the successful project run. Start the project again before cleanup; no files were removed." + ); + } + } + await assertConsent( + project, + [...journal.removable, join(project.projectDir, "hack.config.json")], + input.acceptTrackedChanges + ); + await input.snapshots.assertCoverage({ + manifestPath: journal.manifestPath, + files: journal.sources.filter((source) => + journal.removable.includes(source.path) + ), + }); + await assertSources(journal, true); + const context = await acquireLegacyEnvCleanupContext({ + ...project, + serviceNames: await discoverComposeServiceNames({ + composeFile: join(project.projectDir, "docker-compose.yml"), + }), + overlayNames: journal.overlayNames, + paths: journal.sources + .filter( + (source) => !journal.writes.some((write) => write.path === source.path) + ) + .map((source) => source.path), + ...(journal.store ? { store: journal.store } : {}), + }); + const repair = + journal.store && journal.removable.includes(journal.store.path) + ? await prepareLegacyEnvCleanupConfig(project) + : null; + if (repair) { + const payloadDir = join(dirname(path), `payloads-${randomUUID()}`); + await privateDirectory(payloadDir); + const payload = join(payloadDir, "0.payload"); + await replacePrivateFileAtomic({ + path: payload, + directories: [payloadDir], + text: repair.text, + mode: 0o600, + refusal: refuse, + }); + journal = { + ...journal, + cleanupConfig: { + path: repair.path, + kind: "config", + keys: [], + mode: repair.mode, + size: Buffer.byteLength(repair.text), + sha256: digest(Buffer.from(repair.text)), + before: (await readState(repair.path)).state, + payload, + done: false, + }, + }; + } + journal = { ...journal, state: "cleaning" }; + await writeMetadata(path, journal); + const recordCleanup = async (removed: readonly string[]) => { + journal = { + ...journal, + removed: [...new Set([...journal.removed, ...removed])], + }; + await writeMetadata(path, journal); + if ( + journal.cleanupConfig && + journal.store && + journal.removed.includes(journal.store.path) + ) { + await publishEntry(project, journal.cleanupConfig); + await input.snapshots.recordAfter({ + manifestPath: journal.manifestPath, + files: [journal.cleanupConfig], + }); + journal = { + ...journal, + cleanupConfig: { ...journal.cleanupConfig, done: true }, + }; + await writeMetadata(path, journal); + } + await input.snapshots.recordAfter({ + manifestPath: journal.manifestPath, + files: journal.removed.map((removed) => ({ + path: removed, + sha256: null, + mode: null, + size: 0, + })), + }); + }; + let removed: readonly string[] = []; + try { + removed = await removeLegacyProjectEnvArtifacts({ + ...project, + paths: journal.removable, + context, + }); + } catch (error) { + if (error instanceof LegacyEnvCleanupError) { + removed = error.removedPaths; + // Preserve the original failure and confirmed removals even if repair fails. + await recordCleanup(removed).catch(() => undefined); + } + throw error; + } + await recordCleanup(removed); + await writeMetadata(path, { + ...journal, + state: "cleaned", + removed: [...new Set([...journal.removed, ...removed])], + }); + return { + status: "cleaned" as const, + journalPath: path, + manifestPath: journal.manifestPath, + removed, + }; +} + +/** Called only by successful full-project startup paths, never by preview or env resolution. */ +export async function recordEnvMigrationProjectRun( + input: EnvMigrationProject +): Promise { + const project = await canonicalProject(input); + const path = envMigrationJournalPath(project.projectRoot); + const existing = await readJournal(path, project); + if (!existing || existing.state !== "complete") { + return; + } + await withProjectsRegistryLock({ + lockPath: `${path}.lock`, + label: "env migration", + run: async () => { + const journal = await readJournal(path, project); + if (!journal || journal.state !== "complete") { + return; + } + const validated = await Promise.all( + journal.writes + .filter( + (entry) => entry.kind === "layer" || entry.kind === "local_layer" + ) + .map(async (entry) => (await readState(entry.path)).state) + ); + await writeMetadata(path, { ...journal, validated }); + }, + }); +} + +/** Restore is delegated to the snapshot owner with current-state guards for all migration effects. */ +export async function rollbackEnvMigration(input: { + readonly project: EnvMigrationProject; + readonly manifestPath: string; + readonly snapshots: EnvMigrationSnapshots; +}) { + const project = await canonicalProject(input.project); + const path = envMigrationJournalPath(project.projectRoot); + if (!(await readJournal(path, project))) { + throw refuse(); + } + return await withProjectsRegistryLock({ + lockPath: `${path}.lock`, + label: "env migration", + run: () => rollbackLocked(input, project, path), + }); +} + +async function rollbackLocked( + input: { + readonly manifestPath: string; + readonly snapshots: EnvMigrationSnapshots; + }, + project: EnvMigrationProject, + path: string +) { + const journal = await readJournal(path, project); + if (!journal || journal.manifestPath !== resolve(input.manifestPath)) { + throw refuse(); + } + // Restore legacy sources first, then reverse publication: encrypted layers + // leave resolution before a newly-created project key can be moved aside. + const restoreOrder = [ + ...journal.sources.filter( + (source) => !journal.writes.some((write) => write.path === source.path) + ), + ...[...journal.writes].reverse(), + ]; + const expectedCurrent: EnvMigrationFileState[] = []; + for (const source of restoreOrder) { + const current = (await readState(source.path)).state; + const removed = + journal.removed.includes(source.path) || + (journal.state === "cleaning" && + journal.removable.includes(source.path) && + current.sha256 === null); + let expected = + journal.writes.find((write) => write.path === source.path) ?? source; + if (removed) { + expected = { ...source, sha256: null, mode: null, size: 0 }; + } else if ( + journal.cleanupConfig?.path === source.path && + same(current, journal.cleanupConfig) + ) { + expected = journal.cleanupConfig; + } + expectedCurrent.push(expected); + } + const result = await input.snapshots.rollback({ + manifestPath: journal.manifestPath, + expectedCurrent, + }); + if (result.status !== "restored") { + return result; + } + await writeMetadata(path, { ...journal, state: "rolled_back" }); + return result; +} + +/** Compatibility bridge for existing implicit command routes; publication is journalled and cleanup is deferred. */ +export async function migrateLegacyEnvForCommand( + project: EnvMigrationProject & { readonly materialize: boolean } +): Promise { + const canonical = await canonicalProject(project); + const path = envMigrationJournalPath(canonical.projectRoot); + const retained = await readJournal(path, canonical); + const preview = + retained && retained.state !== "rolled_back" + ? await previewEnvMigration(canonical) + : await previewPrepared( + canonical, + (await prepareConsistent({ ...canonical, repairCompose: false })) + .prepared, + path + ); + const result = await applyEnvMigration({ + project, + acceptTrackedChanges: false, + snapshots: await envMigrationSnapshots(), + repairCompose: false, + }); + return { + legacyDetected: result.status === "applied", + wroteFiles: preview.files + .filter((file) => file.action !== "removable") + .map((file) => file.path), + migratedOverlays: [], + movedPlainValues: preview.files + .filter( + (file) => + file.action !== "removable" && file.path.endsWith(".local.yaml") + ) + .flatMap((file) => file.keys.map((key) => ({ key, path: file.path }))), + updatedProjectConfig: preview.files.some( + (file) => + file.path === join(project.projectDir, "hack.config.json") && + file.action === "change" + ), + cleanupCandidates: preview.files + .filter((file) => file.action === "removable") + .map((file) => file.path), + blockedCleanupCandidates: [], + composeEnvFileReferences: [], + retainedEnvFiles: [], + }; +} diff --git a/src/lib/hack-env.ts b/src/lib/hack-env.ts index faaa6d854..273c90931 100644 --- a/src/lib/hack-env.ts +++ b/src/lib/hack-env.ts @@ -433,6 +433,8 @@ export async function resolveHackEnv(opts: { readonly projectDir: string; readonly projectName: string; readonly envName?: string | null; + /** Read-only migration planning must never provision or copy backend key material. */ + readonly allowKeyProvisioning?: boolean; }): Promise { const read = await readHackEnvContract({ projectDir: opts.projectDir }); const contract = read.contract; @@ -466,6 +468,7 @@ export async function resolveHackEnv(opts: { await resolveSecretStore({ projectName: opts.projectName, projectDir: opts.projectDir, + allowKeyProvisioning: opts.allowKeyProvisioning, }) ).descriptor : await resolveSecretStoreDescriptor({ @@ -476,6 +479,7 @@ export async function resolveHackEnv(opts: { ? await resolveSecretStore({ projectName: opts.projectName, projectDir: opts.projectDir, + allowKeyProvisioning: opts.allowKeyProvisioning, }) : createUnusedSecretStore({ descriptor: secretStoreDescriptor, diff --git a/src/lib/project-env-config.ts b/src/lib/project-env-config.ts index 03f74bc2f..4307b0bbe 100644 --- a/src/lib/project-env-config.ts +++ b/src/lib/project-env-config.ts @@ -44,6 +44,7 @@ import { } from "../templates.ts"; import { childEnvironment } from "./child-environment.ts"; import { ClosedCliError, HackCliError } from "./cli-result.ts"; +import { resolveGlobalConfigPath } from "./config-paths.ts"; import { parseDotEnv, serializeDotEnv } from "./env.ts"; import { withEnvLayerLock } from "./env-layer-lock.ts"; import { envParseDiagnostic } from "./env-parse-diagnostic.ts"; @@ -6233,6 +6234,587 @@ export function describeRetainedLegacyEnvFile(opts: { return `Kept ${opts.shownPath}: keys not preserved in env layers: ${keys} (values omitted).`; } +/** Public migration metadata never owns file contents; payload access is non-enumerable. */ +export type PreparedLegacyEnvMigration = { + readonly files: readonly { + readonly path: string; + readonly kind: + | "ignore" + | "key" + | "local_layer" + | "layer" + | "config" + | "compose"; + readonly keys: readonly string[]; + readonly mode: number; + }[]; + readonly sourcePaths: readonly string[]; + readonly removable: readonly string[]; + readonly overlayNames: readonly string[]; + readonly hasContract: boolean; + readonly store?: { readonly path: string; readonly keyPath: string }; + /** Private in-memory payload: never serialize this return value. */ + readonly payload: (path: string) => string; +}; + +/** + * Prepares the explicit, journalled migration without any filesystem mutation. + * Legacy sources stay in place throughout publication. The journal owner stages + * payloads privately, records only their checksums, and publishes under locks. + */ +export async function prepareLegacyProjectEnvMigration(opts: { + readonly projectRoot: string; + readonly projectDir: string; + readonly projectName: string; + readonly serviceNames: readonly string[]; + readonly repairCompose?: boolean; +}): Promise { + try { + return await prepareLegacyProjectEnvMigrationInternal(opts); + } catch (error) { + if (error instanceof ClosedCliError || error instanceof HackCliError) { + throw error; + } + throw new ClosedCliError( + "Cannot prepare env migration: an input is unreadable, malformed or unsafe; values omitted." + ); + } +} + +type PreparedMigrationProject = Parameters< + typeof prepareLegacyProjectEnvMigration +>[0]; +type AddPreparedMigrationFile = ( + file: PreparedLegacyEnvMigration["files"][number], + text: string +) => Promise; +type AddPreparedMigrationLayer = ( + write: PlannedEnvLayerWrite, + local: boolean +) => Promise; +type ResolvedMigrationValues = readonly (Awaited< + ReturnType +> & { readonly name: string | null })[]; + +function mergePreparedLocalValues( + previous: ProjectEnvConfig, + incoming: ProjectEnvConfig +): ProjectEnvConfig { + const values = structuredClone(previous.values); + for (const [scope, entries] of Object.entries(incoming.values)) { + for (const [key, value] of Object.entries(entries)) { + const existing = values[scope]?.[key]; + if ( + existing !== undefined && + JSON.stringify(existing) !== JSON.stringify(value) + ) { + throw new ClosedCliError( + `Migration conflicts with an existing local value for ${key}; values omitted.` + ); + } + } + values[scope] = { ...values[scope], ...entries }; + } + return { ...previous, values }; +} + +async function prepareContractMigrationKey( + opts: PreparedMigrationProject, + resolved: ResolvedMigrationValues, + extraSources: Set, + add: AddPreparedMigrationFile +): Promise { + if ( + resolved.some((entry) => + entry.values.some( + (value) => value.source === "keychain" && value.value !== null + ) + ) + ) { + await refuseMigrationKeyShadowingEnvironmentKey(opts); + const localPath = resolveProjectEnvKeyPath(opts); + const shared = await resolveProjectEnvSharedKeyPath(opts); + const inherited = await resolveInheritedProjectEnvKey(opts); + let existingPath = inherited?.keyPath; + if (shared && (await pathExists(shared))) { + existingPath = shared; + } + if (await pathExists(localPath)) { + existingPath = localPath; + } + const keyPath = existingPath ?? shared ?? localPath; + extraSources.add(keyPath); + if (existingPath) { + await assertProjectEnvKeyUntracked({ keyPath }); + } + const keyText = existingPath + ? await readLegacyProjectEnvKeyFile({ path: keyPath }) + : randomBytes(32).toString("base64url"); + if (!keyText) { + throw projectEnvKeyFileError(); + } + await add( + { path: keyPath, kind: "key", keys: [], mode: 0o600 }, + `${keyText}\n` + ); + if (keyPath === localPath) { + await prepareMigrationKeyIgnore(opts, add); + } + return keyText; + } + return null; +} + +async function prepareMigrationKeyIgnore( + opts: PreparedMigrationProject, + add: AddPreparedMigrationFile +): Promise { + const path = resolve(opts.projectRoot, ".gitignore"); + const existing = await readTextFile(path); + // Appending after possible negations ensures a freshly written private key + // is ignored. Existing effective final entries require no change. + if (existing?.split("\n").at(-2) === PROJECT_ENV_KEY_FILENAME) { + return; + } + const separator = !existing || existing.endsWith("\n") ? "" : "\n"; + await add( + { path, kind: "ignore", keys: [], mode: 0o644 }, + `${existing ?? ""}${separator}# project env key\n${PROJECT_ENV_KEY_FILENAME}\n` + ); +} + +async function prepareContractMigrationLayers( + opts: PreparedMigrationProject, + resolved: ResolvedMigrationValues, + keyText: string | null, + addLayer: AddPreparedMigrationLayer +): Promise { + const baseValues = new Map( + resolved[0]?.values.map((entry) => [entry.key, entry.value]) + ); + for (const entry of resolved) { + const values = buildMigratedValues({ + resolvedValues: entry.values.filter( + (value) => value.source === "plain_env" + ), + baseValues: entry.name === null ? null : baseValues, + keyText: null, + }); + if (Object.values(values).some((scope) => Object.keys(scope).length > 0)) { + await addLayer( + { + path: await resolveProjectEnvEffectiveLocalConfigPath({ + ...opts, + envName: entry.name, + }), + config: buildMigratedProjectEnvConfig({ + environment: entry.name ?? "default", + values, + }), + }, + true + ); + } + const secrets = buildMigratedValues({ + resolvedValues: entry.values.filter( + (value) => value.source === "keychain" + ), + baseValues: entry.name === null ? null : baseValues, + keyText, + }); + if ( + entry.name === null || + Object.values(secrets).some((scope) => Object.keys(scope).length > 0) || + Object.values(values).some((scope) => Object.keys(scope).length > 0) + ) { + await addLayer( + { + path: resolveProjectEnvConfigPath({ + ...opts, + envName: entry.name, + }), + config: buildMigratedProjectEnvConfig({ + environment: entry.name ?? "default", + values: secrets, + }), + }, + false + ); + } + } +} + +async function prepareContractMigration( + opts: PreparedMigrationProject, + contract: Awaited>["contract"], + layersExist: boolean, + extraSources: Set, + add: AddPreparedMigrationFile, + addLayer: AddPreparedMigrationLayer +) { + const selected = await selectContractMigrationOverlays({ + ...opts, + contract, + composeRepairFollows: true, + }); + const overlayNames = selected.overlayNames; + const store = { + path: selected.secretStoreListing.storePath, + keyPath: selected.secretStoreListing.keyPath, + }; + await assertNoShellValuesInContractMigration({ + values: await findProcessOnlyContractValues({ ...opts, overlayNames }), + }); + const resolved = await Promise.all( + [null, ...overlayNames].map(async (envName) => ({ + name: envName, + ...(await resolveHackEnv({ + ...opts, + envName, + allowKeyProvisioning: false, + })), + })) + ); + const base = resolved[0]; + if (!base) { + throw new ClosedCliError("Migration base is unavailable; values omitted."); + } + assertContractMigrationLayersAreWritable({ + baseValues: base.values, + overlays: resolved + .slice(1) + .map((entry) => ({ name: entry.name ?? "", values: entry.values })), + }); + await preflightLegacyEnvMigrationPaths({ ...opts, overlayNames }); + if (overlayNames.includes("default") || overlayNames.includes("local")) { + throw new ClosedCliError( + "Reserved legacy overlay name; rename it before migration; values omitted." + ); + } + const keyText = layersExist + ? null + : await prepareContractMigrationKey(opts, resolved, extraSources, add); + if (!layersExist) { + await prepareContractMigrationLayers(opts, resolved, keyText, addLayer); + } + const deliveredValues = new Map( + resolved.map((entry) => [ + resolveEnvFilePath({ ...opts, envName: entry.name }), + new Map(entry.values.map((value) => [value.key, value.value])), + ]) + ); + const planned = await planLegacyProjectConfigChanges(opts); + const retained = await retainLegacySecretStoreFiles({ + candidates: planned.cleanupCandidates, + listing: selected.secretStoreListing, + deliveredByEnv: new Map( + resolved.map((entry) => [ + entry.name, + new Map(entry.values.map((value) => [value.key, value.value])), + ]) + ), + }); + const configCandidates = retained.candidates; + const configText = await readTextFile(planned.path); + if (configText) { + const parsed: unknown = JSON.parse(configText); + if (isRecord(parsed)) { + const normalized = normalizeLegacyProjectTopLevelConfig({ + parsed, + projectRoot: opts.projectRoot, + keepStorePointer: true, + }); + if (normalized.changed) { + await add( + { path: planned.path, kind: "config", keys: [], mode: 0o644 }, + `${JSON.stringify(normalized.config, null, 2)}\n` + ); + } + } + } + return { overlayNames, store, deliveredValues, configCandidates }; +} + +async function prepareUndeclaredMigration( + opts: PreparedMigrationProject, + addLayer: AddPreparedMigrationLayer +) { + const prepared = await prepareUndeclaredLegacyDotenv({ + ...opts, + materialize: false, + }); + const overlayNames = prepared.overlayNames; + for (const write of prepared.localWrites) { + await addLayer(write, true); + } + for (const write of prepared.trackedWrites) { + await addLayer(write, false); + } + const legacyValues = await readValidatedLegacyDotenvFiles({ + ...opts, + overlayNames, + skipBase: prepared.generatedBase, + }); + const base = legacyValues.get(null) ?? {}; + const deliveredValues = new Map( + [null, ...overlayNames].map((envName) => [ + resolveEnvFilePath({ ...opts, envName }), + new Map(Object.entries({ ...base, ...legacyValues.get(envName) })), + ]) + ); + return { + overlayNames, + deliveredValues, + configCandidates: [] as readonly string[], + store: undefined, + }; +} + +async function prepareMigrationCompose( + opts: PreparedMigrationProject, + add: AddPreparedMigrationFile +): Promise { + const composeFile = resolve(opts.projectDir, PROJECT_COMPOSE_FILENAME); + const references = await inspectLegacyComposeEnvFileReferences({ + ...opts, + composeFile, + }); + if (references.length > 0) { + const parsed = await readComposeFile({ composeFile }); + if (!parsed) { + throw new ClosedCliError( + "Cannot prepare Compose repair; values omitted." + ); + } + for (const [service, raw] of Object.entries( + getRecord(parsed, "services") ?? {} + )) { + if (isRecord(raw) && "env_file" in raw) { + raw.env_file = removeLegacyComposeEnvFileEntries({ + service, + rawEnvFile: raw.env_file, + composeDir: opts.projectDir, + projectDir: opts.projectDir, + removed: [], + }); + } + } + await add( + { path: composeFile, kind: "compose", keys: [], mode: 0o644 }, + YAML.stringify(parsed, null, 2) + ); + } +} + +async function prepareLegacyProjectEnvMigrationInternal(opts: { + readonly projectRoot: string; + readonly projectDir: string; + readonly projectName: string; + readonly serviceNames: readonly string[]; + readonly repairCompose?: boolean; +}): Promise { + await assertLegacyProjectDirectory(opts); + await assertLegacyProjectInputFamily(opts); + const contract = await readHackEnvContract(opts); + if (contract.parseError !== undefined) { + throw new ClosedCliError( + "Cannot prepare migration: the env contract is unreadable or malformed; values omitted." + ); + } + const texts = new Map(); + const extraSources = new Set([ + resolveGlobalConfigPath(), + resolve(opts.projectDir, ".gitignore"), + ]); + const files: PreparedLegacyEnvMigration["files"][number][] = []; + const add = async ( + file: PreparedLegacyEnvMigration["files"][number], + text: string + ) => { + const previous = await readReplaceablePrivateFile({ + path: file.path, + directories: [], + refusal: () => + new ClosedCliError( + "Migration input is unsafe or changed; values omitted." + ), + }); + const mode = + previous && file.kind !== "key" && file.kind !== "local_layer" + ? previous.mode & 0o777 + : file.mode; + if (previous?.text === text && (previous.mode & 0o777) === mode) { + return; + } + if (previous && (previous.mode & 0o222) === 0) { + throw new ClosedCliError( + `Migration target is read-only: ${file.path}; values omitted.` + ); + } + texts.set(file.path, text); + files.push({ ...file, mode }); + }; + const keysOf = (config: ProjectEnvConfig) => + [...new Set(Object.values(config.values).flatMap(Object.keys))].sort(); + const addLayer = async (write: PlannedEnvLayerWrite, local: boolean) => { + await assertProjectEnvLayerPath({ ...opts, path: write.path }); + if (local) { + await assertUntrackedMigrationLocalLayer({ ...opts, path: write.path }); + } + const previous = await readExistingLocalLayer({ + path: write.path, + environment: write.config.environment, + }); + if (previous && !local) { + return; + } + const config = previous + ? mergePreparedLocalValues(previous.config, write.config) + : write.config; + await add( + { + path: write.path, + kind: local ? "local_layer" : "layer", + keys: keysOf(config), + mode: local ? 0o600 : 0o644, + }, + serializeProjectEnvConfig(config) + ); + }; + const layersExist = await projectEnvConfigExistsForMutation(opts); + const { overlayNames, store, deliveredValues, configCandidates } = + contract.exists + ? await prepareContractMigration( + opts, + contract.contract, + layersExist, + extraSources, + add, + addLayer + ) + : await prepareUndeclaredMigration(opts, addLayer); + if (files.some((file) => file.kind === "local_layer")) { + const path = resolve(opts.projectDir, ".gitignore"); + const previous = await readReplaceablePrivateFile( + hackDirGitignoreFileOptions(opts) + ); + await add( + { path, kind: "ignore", keys: [], mode: previous?.mode ?? 0o644 }, + mergeHackDirGitignore({ existing: previous?.text ?? null }) + ); + } + const composeFile = resolve(opts.projectDir, PROJECT_COMPOSE_FILENAME); + if (opts.repairCompose !== false) { + await prepareMigrationCompose(opts, add); + } + const candidates = await collectLegacyProjectEnvCleanupCandidates({ + ...opts, + overlayNames, + materialize: false, + deliveredValues, + configCleanupCandidates: configCandidates, + composeEnvFileReferences: + opts.repairCompose === false + ? await inspectLegacyComposeEnvFileReferences({ ...opts, composeFile }) + : [], + composeInterpolatedKeys: new Map([ + [ + resolve(opts.projectDir, PROJECT_ENV_FILENAME), + await composeInterpolatedServiceOnlyKeys({ + projectDir: opts.projectDir, + resolvedValues: contract.exists + ? ( + await resolveHackEnv({ + ...opts, + envName: null, + allowKeyProvisioning: false, + }) + ).values + : [], + }), + ], + ]), + }); + const removable = contract.exists + ? candidates.allowed + : await filterUntrackedLegacyCleanupCandidates({ + ...opts, + paths: candidates.allowed, + }); + for (const envName of [null, ...overlayNames]) { + const metadata = await resolveProjectEnvMetadata({ ...opts, envName }); + for (const path of Object.values(metadata?.files ?? {})) { + if (typeof path === "string") { + extraSources.add(path); + } + } + } + const sourcePaths = [ + ...new Set([ + contract.path, + composeFile, + resolve(opts.projectDir, PROJECT_CONFIG_FILENAME), + ...[null, ...overlayNames].map((envName) => + resolveEnvFilePath({ ...opts, envName }) + ), + ...(store ? [store.path, store.keyPath] : []), + ...files.map((file) => file.path), + ...extraSources, + ]), + ].sort(); + const prepared: PreparedLegacyEnvMigration = { + files: files.sort((a, b) => { + const order = { + ignore: 0, + key: 1, + local_layer: 2, + layer: 3, + config: 4, + compose: 5, + }; + return order[a.kind] - order[b.kind] || a.path.localeCompare(b.path); + }), + sourcePaths, + removable, + overlayNames, + hasContract: contract.exists, + ...(store ? { store } : {}), + payload: (path) => { + const text = texts.get(path); + if (text === undefined) { + throw new ClosedCliError("Unknown migration payload; values omitted."); + } + return text; + }, + }; + Object.defineProperty(prepared, "payload", { enumerable: false }); + return prepared; +} + +/** Rebuild ephemeral cleanup authority only after the journal owner verifies its source checksums. */ +export async function acquireLegacyEnvCleanupContext(opts: { + readonly projectRoot: string; + readonly projectDir: string; + readonly projectName: string; + readonly serviceNames: readonly string[]; + readonly overlayNames: readonly string[]; + readonly paths: readonly string[]; + readonly store?: { readonly path: string; readonly keyPath: string }; +}): Promise { + const contract = await readHackEnvContract(opts); + if (contract.parseError !== undefined) { + throw legacyCleanupChanged(); + } + const receipts: LegacyEnvCleanupReceipt[] = []; + for (const path of opts.paths) { + const receipt = await legacyEnvCleanupReceipt(path); + if (receipt) { + receipts.push(receipt); + } + } + return { ...opts, contract: contract.contract, receipts }; +} + /** * Migrates a legacy v2 `.env`-based project env layout to the v3 * `hack.env.*.yaml` config system. @@ -6883,23 +7465,17 @@ export type ImplicitLegacyEnvMigrationReport = { }; /** - * Finishes a contract migration that another command ran implicitly (such as - * `hack env exec`), the way doctor's cleanup does for untracked files. It - * removes a legacy `.hack/.env` or `.hack/.env.` file only when every - * entry in it now lives in an env layer, Compose no longer names it as - * `env_file`, and git does not track it (or, inside a checkout, cannot say), - * so no second plaintext copy stays behind. This step never removes or edits - * `hack.env.json`, the Compose file, an encrypted backend file or any other - * tracked file; `hack doctor --migrate-env-config` offers those. (The - * migration itself still rewrites legacy keys in `hack.config.json`.) + * Reports an implicit migration without removing any legacy input. Cleanup + * is a separate, snapshot-backed action after project validation, including + * for files Git does not track. Retention reasons remain value-free. * * The report names written paths and moved keys, never values. */ -export async function finishImplicitLegacyEnvMigration(opts: { +export function finishImplicitLegacyEnvMigration(opts: { readonly projectRoot: string; readonly projectDir: string; readonly migrated: LegacyProjectEnvMigration; -}): Promise { +}): ImplicitLegacyEnvMigrationReport { const { migrated } = opts; if (!migrated.legacyDetected || migrated.wroteFiles.length === 0) { return { removed: [], lines: [] }; @@ -6917,30 +7493,10 @@ export async function finishImplicitLegacyEnvMigration(opts: { resolve(opts.projectDir, `${PROJECT_ENV_FILENAME}.${overlay}`) ), ]); - let removed: readonly string[] = []; - try { - const removable = await filterUntrackedLegacyCleanupCandidates({ - projectRoot: opts.projectRoot, - paths: migrated.cleanupCandidates.filter((path) => dotenvFiles.has(path)), - }); - removed = await removeLegacyProjectEnvArtifacts({ - projectRoot: opts.projectRoot, - paths: removable, - context: migrated.cleanupContext, - }); - } catch (error) { - const failure = - error instanceof LegacyEnvCleanupError - ? error - : new LegacyEnvCleanupError({ reason: "unverified" }); - removed = failure.removedPaths; - lines.push(`Warning: kept remaining legacy env files. ${failure.message}`); - } - if (removed.length > 0) { - lines.push( - `Removed ${removed.map(shown).join(", ")}: env layers now hold every value ${removed.length === 1 ? "it" : "they"} set.` - ); - } + const removed: readonly string[] = []; + lines.push( + "Legacy env files were kept. After validating the project, preview the separate cleanup step with hack env migrate --cleanup." + ); if (migrated.keptSecretStorePointer) { lines.push( describeKeptSecretStorePointer({ @@ -9022,12 +9578,18 @@ async function deliveredValuesPerLegacyFile(opts: { * for cleanup. Values are written to the layer files only, never returned or * printed. */ -async function migrateUndeclaredLegacyDotenv(opts: { +async function prepareUndeclaredLegacyDotenv(opts: { readonly projectRoot: string; readonly projectDir: string; readonly serviceNames: readonly string[]; readonly materialize: boolean; -}): Promise { +}): Promise<{ + overlayNames: readonly string[]; + generatedBase: boolean; + localWrites: PlannedEnvLayerWrite[]; + trackedWrites: PlannedEnvLayerWrite[]; + merges: Awaited>; +}> { // Also refuses a dangling or non-file layer entry (E_PATH_CONFLICT), which // an atomic write would otherwise replace or write through. if ( @@ -9176,6 +9738,17 @@ async function migrateUndeclaredLegacyDotenv(opts: { path: write.path, }); } + return { overlayNames, generatedBase, localWrites, trackedWrites, merges }; +} + +async function migrateUndeclaredLegacyDotenv(opts: { + readonly projectRoot: string; + readonly projectDir: string; + readonly serviceNames: readonly string[]; + readonly materialize: boolean; +}): Promise { + const { overlayNames, generatedBase, localWrites, trackedWrites, merges } = + await prepareUndeclaredLegacyDotenv(opts); if (localWrites.length > 0) { await ensureHackDirGitignoreForLocalEnvWrite({ projectDir: opts.projectDir, @@ -9771,6 +10344,42 @@ export async function dropRemovedLegacySecretStorePointer(opts: { return migrated.changed; } +/** Prepare the exact pointer repair before an explicit cleanup starts. No writes. */ +export async function prepareLegacyEnvCleanupConfig(opts: { + readonly projectRoot: string; + readonly projectDir: string; +}): Promise<{ + readonly path: string; + readonly text: string; + readonly mode: number; +} | null> { + const path = resolve(opts.projectDir, PROJECT_CONFIG_FILENAME); + const existing = await readReplaceablePrivateFile({ + path, + directories: [], + refusal: legacyCleanupChanged, + }); + if (!existing) { + return null; + } + const parsed: unknown = JSON.parse(existing.text); + if (!isRecord(parsed)) { + throw legacyCleanupChanged(); + } + const normalized = normalizeLegacyProjectTopLevelConfig({ + parsed, + projectRoot: opts.projectRoot, + keepStorePointer: false, + }); + return normalized.changed + ? { + path, + text: `${JSON.stringify(normalized.config, null, 2)}\n`, + mode: existing.mode & 0o777, + } + : null; +} + async function discoverLegacyOverlayNames(opts: { readonly projectDir: string; readonly includeSymlinks: boolean; diff --git a/tests/env-migration-journal.test.ts b/tests/env-migration-journal.test.ts new file mode 100644 index 000000000..801c3e54b --- /dev/null +++ b/tests/env-migration-journal.test.ts @@ -0,0 +1,467 @@ +import { afterEach, beforeEach, expect, test } from "bun:test"; +import { + lstat, + mkdir, + mkdtemp, + readdir, + readFile, + realpath, + rm, + writeFile, +} from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { + applyEnvMigration, + cleanupEnvMigration, + type EnvMigrationProject, + previewEnvMigration, + recordEnvMigrationProjectRun, + rollbackEnvMigration, +} from "../src/lib/env-migration.ts"; +import { resolveProjectEnvConfig } from "../src/lib/project-env-config.ts"; +import { resolveSecretStore } from "../src/lib/secret-store.ts"; +import { restoreEnv } from "./helpers/env.ts"; +import { fixtureEnvSnapshots } from "./helpers/env-migration-fixture.ts"; +import { privateCliEnv } from "./helpers/private-cli-env.ts"; + +const SENTINEL = "synthetic-upgrade-journal-value"; +const QA = "synthetic-upgrade-qa-value"; +const isolated = [ + "HOME", + "HACK_HOME", + "HACK_ENV_SECRET_KEY", + "HACK_SECRETS_FILE_KEY", + "HACK_GLOBAL_CONFIG_PATH", +] as const; +let root = ""; +let saved: (string | undefined)[] = []; +let project: EnvMigrationProject; + +beforeEach(async () => { + saved = isolated.map((key) => process.env[key]); + root = await realpath(await mkdtemp(join(tmpdir(), "env-journal-"))); + for (const key of isolated) { + Reflect.deleteProperty(process.env, key); + } + process.env.HOME = join(root, "home"); + process.env.HACK_HOME = join(root, "hack-home"); + await mkdir(process.env.HOME); + const projectRoot = join(root, "project"); + const projectDir = join(projectRoot, ".hack"); + await mkdir(projectDir, { recursive: true }); + await writeFile(join(projectDir, "hack.config.json"), '{"name":"journal"}\n'); + await writeFile( + join(projectDir, "hack.env.json"), + JSON.stringify({ + version: 1, + vars: [{ key: "TOKEN", source: "plain_env" }], + }) + ); + await writeFile( + join(projectDir, "docker-compose.yml"), + "services:\n api:\n image: alpine:3.22\n" + ); + await writeFile(join(projectDir, ".env"), `TOKEN=${SENTINEL}\n`); + await writeFile(join(projectDir, ".env.qa"), `TOKEN=${QA}\n`); + project = { + projectRoot, + projectDir, + projectName: "journal", + serviceNames: ["api"], + }; +}); + +afterEach(async () => { + for (const [index, key] of isolated.entries()) { + restoreEnv(key, saved[index]); + } + if (root) { + await rm(root, { recursive: true, force: true }); + } +}); + +async function projectFiles(): Promise> { + const files = new Map(); + const visit = async (path: string): Promise => { + for (const entry of await readdir(path, { withFileTypes: true })) { + const selected = join(path, entry.name); + if (entry.isDirectory()) { + await visit(selected); + } else { + files.set(selected, await readFile(selected, "base64")); + } + } + }; + await visit(project.projectRoot); + return files; +} + +test("preview makes no writes and exactly names what apply publishes; completed retry is a no-op", async () => { + const before = await projectFiles(); + const preview = await previewEnvMigration(project); + expect(await projectFiles()).toEqual(before); + expect(await Bun.file(preview.journalPath).exists()).toBe(false); + expect(JSON.stringify(preview)).not.toContain(SENTINEL); + const snapshots = fixtureEnvSnapshots(root); + const result = await applyEnvMigration({ + project, + snapshots, + acceptTrackedChanges: true, + }); + await expect( + cleanupEnvMigration({ project, snapshots, acceptTrackedChanges: true }) + ).rejects.toThrow("hack up"); + expect(result.status).toBe("applied"); + const after = await projectFiles(); + expect( + [...after] + .filter(([path, text]) => before.get(path) !== text) + .map(([path]) => path) + .sort() + ).toEqual( + preview.files + .filter((file) => file.action !== "removable") + .map((file) => file.path) + .sort() + ); + expect(await readFile(join(project.projectDir, ".env"), "utf8")).toContain( + SENTINEL + ); + expect( + (await resolveProjectEnvConfig({ ...project, envName: null }))?.globalEnv + .TOKEN + ).toBe(SENTINEL); + expect( + (await resolveProjectEnvConfig({ ...project, envName: "qa" }))?.globalEnv + .TOKEN + ).toBe(QA); + const journal = await readFile(result.journalPath, "utf8"); + expect(journal).not.toContain(SENTINEL); + expect(journal).not.toContain(QA); + expect(await readFile(result.manifestPath, "utf8")).not.toContain(SENTINEL); + expect( + ( + await applyEnvMigration({ + project, + snapshots, + acceptTrackedChanges: true, + }) + ).status + ).toBe("already_complete"); + expect(await projectFiles()).toEqual(after); + expect(await readFile(result.journalPath, "utf8")).toBe(journal); +}); + +test("cleanup is explicit, rechecks delivery, and requires intact snapshot payloads", async () => { + const snapshots = fixtureEnvSnapshots(root); + const result = await applyEnvMigration({ + project, + snapshots, + acceptTrackedChanges: true, + }); + await writeFile( + join(project.projectDir, "hack.env.local.yaml"), + "version: 1\nenvironment: default\nsecretsprovider: project_key\nvalues:\n global:\n TOKEN: changed\n" + ); + await recordEnvMigrationProjectRun(project); + await expect( + cleanupEnvMigration({ project, snapshots, acceptTrackedChanges: true }) + ).rejects.toThrow(); + expect(await readFile(join(project.projectDir, ".env"), "utf8")).toContain( + SENTINEL + ); + expect(await Bun.file(result.manifestPath).exists()).toBe(true); +}); + +test("rollback restores original bytes and refuses a later user edit", async () => { + const before = await projectFiles(); + const snapshots = fixtureEnvSnapshots(root); + const result = await applyEnvMigration({ + project, + snapshots, + acceptTrackedChanges: true, + }); + const target = join(project.projectDir, "hack.env.local.yaml"); + const original = await readFile(target, "utf8"); + await writeFile(target, `${original}\n# user edit\n`); + await expect( + rollbackEnvMigration({ + project, + snapshots, + manifestPath: result.manifestPath, + }) + ).rejects.toThrow("changed target"); + await writeFile(target, original); + await rollbackEnvMigration({ + project, + snapshots, + manifestPath: result.manifestPath, + }); + expect(await projectFiles()).toEqual(before); +}); + +test("cleanup snapshots must still hold every original; cleanup followed by rollback restores legacy files", async () => { + const before = await projectFiles(); + const snapshots = fixtureEnvSnapshots(root); + const result = await applyEnvMigration({ + project, + snapshots, + acceptTrackedChanges: true, + }); + await recordEnvMigrationProjectRun(project); + const manifest = JSON.parse(await readFile(result.manifestPath, "utf8")); + const entry = manifest.files.find( + (file: { path: string }) => file.path === join(project.projectDir, ".env") + ); + const original = await readFile(entry.payload); + await writeFile(entry.payload, "damaged synthetic backup"); + await expect( + cleanupEnvMigration({ project, snapshots, acceptTrackedChanges: true }) + ).rejects.toThrow("coverage"); + expect(await projectFiles()).not.toEqual(before); + expect(await readFile(join(project.projectDir, ".env"), "utf8")).toContain( + SENTINEL + ); + await writeFile(entry.payload, original); + await cleanupEnvMigration({ project, snapshots, acceptTrackedChanges: true }); + expect(await Bun.file(join(project.projectDir, ".env")).exists()).toBe(false); + await rollbackEnvMigration({ + project, + snapshots, + manifestPath: result.manifestPath, + }); + expect(await projectFiles()).toEqual(before); +}); + +test("preview does not provision a new project key; apply encrypts with a private key and rollback retains every legacy value", async () => { + await writeFile( + join(project.projectDir, "hack.config.json"), + JSON.stringify({ + name: "journal", + controlPlane: { + secrets: { + backend: "encrypted_file", + encryptedFile: { path: "legacy-store.json", keyPath: "legacy.key" }, + }, + }, + }) + ); + await writeFile( + join(project.projectDir, "hack.env.json"), + JSON.stringify({ + version: 1, + vars: [ + { key: "TOKEN", source: "plain_env" }, + { key: "SEC", source: "keychain", services: ["api"] }, + ], + }) + ); + const store = await resolveSecretStore(project); + await store.set({ key: "SEC", value: SENTINEL }); + const before = await projectFiles(); + const keyPath = join(project.projectRoot, ".hack.secret.key"); + const preview = await previewEnvMigration(project); + expect(await projectFiles()).toEqual(before); + expect(preview.files).toContainEqual({ + path: keyPath, + action: "write", + keys: [], + tracked: false, + }); + const snapshots = fixtureEnvSnapshots(root); + const result = await applyEnvMigration({ + project, + snapshots, + acceptTrackedChanges: true, + }); + expect((await lstat(keyPath)).mode & 0o777).toBe(0o600); + expect( + (await resolveProjectEnvConfig({ ...project, envName: null }))?.serviceEnv + .api?.SEC + ).toBe(SENTINEL); + await recordEnvMigrationProjectRun(project); + await cleanupEnvMigration({ project, snapshots, acceptTrackedChanges: true }); + expect( + await readFile(join(project.projectDir, "hack.config.json"), "utf8") + ).not.toContain("encryptedFile"); + await rollbackEnvMigration({ + project, + snapshots, + manifestPath: result.manifestPath, + }); + expect(await projectFiles()).toEqual(before); +}); + +test("encrypted scoped values and store-only overlays survive; cleanup never removes their live project key", async () => { + const keyPath = join(project.projectRoot, ".hack.secret.key"); + const keyText = "synthetic-upgrade-key-material"; + await writeFile(keyPath, `${keyText}\n`, { mode: 0o600 }); + await writeFile( + join(project.projectDir, "hack.config.json"), + JSON.stringify({ + name: "journal", + controlPlane: { + secrets: { + backend: "encrypted_file", + encryptedFile: { + path: "legacy-store.json", + keyPath: ".hack.secret.key", + }, + }, + }, + }) + ); + await writeFile( + join(project.projectDir, "hack.env.json"), + JSON.stringify({ + version: 1, + vars: [ + { key: "TOKEN", source: "plain_env" }, + { key: "SEC", source: "keychain", services: ["api"] }, + ], + }) + ); + const store = await resolveSecretStore(project); + await store.set({ key: "SEC", value: SENTINEL }); + await store.set({ key: "env.store-only.SEC", value: QA }); + const before = await projectFiles(); + const preview = await previewEnvMigration(project); + expect(await projectFiles()).toEqual(before); + expect( + preview.files + .filter((file) => file.action === "removable") + .map((file) => file.path) + ).not.toContain(keyPath); + const snapshots = fixtureEnvSnapshots(root); + const result = await applyEnvMigration({ + project, + snapshots, + acceptTrackedChanges: true, + }); + expect( + (await resolveProjectEnvConfig({ ...project, envName: null }))?.serviceEnv + .api?.SEC + ).toBe(SENTINEL); + expect( + (await resolveProjectEnvConfig({ ...project, envName: "store-only" })) + ?.serviceEnv.api?.SEC + ).toBe(QA); + for (const path of [result.journalPath, result.manifestPath]) { + const text = await readFile(path, "utf8"); + for (const value of [SENTINEL, QA, keyText]) { + expect(text).not.toContain(value); + } + } + await recordEnvMigrationProjectRun(project); + await cleanupEnvMigration({ project, snapshots, acceptTrackedChanges: true }); + expect(await readFile(keyPath, "utf8")).toBe(`${keyText}\n`); + expect( + (await resolveProjectEnvConfig({ ...project, envName: "store-only" })) + ?.serviceEnv.api?.SEC + ).toBe(QA); +}); + +test("SIGKILL at every journal/publication boundary resumes without losing an overlay value", async () => { + const preview = await previewEnvMigration(project); + const count = preview.files.filter( + (file) => file.action !== "removable" + ).length; + const boundaries = [ + "prepared:-1", + ...Array.from({ length: count }, (_, index) => [ + `before_publish:${index}`, + `published:${index}`, + `recorded:${index}`, + ]).flat(), + "verified:-1", + "complete:-1", + ]; + // Each interruption owns a fresh HOME, data root, project and snapshot set. + for (const boundary of boundaries) { + const selected = join(root, boundary.replaceAll(":", "-")); + const selectedProject = { + ...project, + projectRoot: join(selected, "project"), + projectDir: join(selected, "project", ".hack"), + }; + await mkdir(selectedProject.projectDir, { recursive: true }); + for (const file of [ + "hack.config.json", + "hack.env.json", + "docker-compose.yml", + ".env", + ".env.qa", + ]) { + await writeFile( + join(selectedProject.projectDir, file), + await readFile(join(project.projectDir, file)) + ); + } + const dataHome = join(selected, "hack-home"); + const child = Bun.spawn( + [ + process.execPath, + "--no-env-file", + join(import.meta.dir, "helpers", "env-migration-child.ts"), + selected, + boundary, + ], + { + env: privateCliEnv({ root: selected, env: { HACK_HOME: dataHome } }), + stdin: "ignore", + stdout: "pipe", + stderr: "pipe", + } + ); + const stderr = new Response(child.stderr).text(); + let output = ""; + let reached = false; + try { + const reader = child.stdout.getReader(); + for (;;) { + const chunk = await reader.read(); + if (chunk.done) { + break; + } + output += new TextDecoder().decode(chunk.value); + if (output.includes(`"phase":"${boundary.split(":")[0]}"`)) { + reached = true; + child.kill("SIGKILL"); + break; + } + } + await child.exited; + expect(reached, `${boundary}: ${await stderr}`).toBe(true); + expect(output).not.toContain(SENTINEL); + const previousHome = process.env.HACK_HOME; + process.env.HACK_HOME = dataHome; + try { + await applyEnvMigration({ + project: selectedProject, + snapshots: fixtureEnvSnapshots(selected), + acceptTrackedChanges: true, + }); + expect( + (await resolveProjectEnvConfig({ ...selectedProject, envName: null })) + ?.globalEnv.TOKEN + ).toBe(SENTINEL); + expect( + (await resolveProjectEnvConfig({ ...selectedProject, envName: "qa" })) + ?.globalEnv.TOKEN + ).toBe(QA); + expect( + await readFile(join(selectedProject.projectDir, ".env"), "utf8") + ).toContain(SENTINEL); + } finally { + restoreEnv("HACK_HOME", previousHome); + } + } finally { + if (child.exitCode === null) { + child.kill("SIGKILL"); + } + await child.exited; + await stderr; + } + } +}, 60_000); diff --git a/tests/helpers/env-migration-child.ts b/tests/helpers/env-migration-child.ts new file mode 100644 index 000000000..0d36f5bbd --- /dev/null +++ b/tests/helpers/env-migration-child.ts @@ -0,0 +1,31 @@ +import { join } from "node:path"; +import { applyEnvMigration } from "../../src/lib/env-migration.ts"; +import { fixtureEnvSnapshots } from "./env-migration-fixture.ts"; + +const root = process.argv[2]; +const stop = process.argv[3]; +if (!(root && stop)) { + throw new Error("Fixture arguments required"); +} +const projectRoot = join(root, "project"); +const result = await applyEnvMigration({ + project: { + projectRoot, + projectDir: join(projectRoot, ".hack"), + projectName: "journal", + serviceNames: ["api"], + }, + snapshots: fixtureEnvSnapshots(root), + acceptTrackedChanges: true, + hooks: { + boundary: async (event) => { + if (`${event.phase}:${event.index}` === stop) { + process.stdout.write(`${JSON.stringify(event)}\n`); + // Owned test child: the parent kills and awaits it at this exact durable boundary. + await Bun.sleep(60_000); + throw new Error("Fixture parent did not kill the child"); + } + }, + }, +}); +process.stdout.write(`${JSON.stringify(result)}\n`); diff --git a/tests/helpers/env-migration-fixture.ts b/tests/helpers/env-migration-fixture.ts new file mode 100644 index 000000000..6d0736f52 --- /dev/null +++ b/tests/helpers/env-migration-fixture.ts @@ -0,0 +1,115 @@ +import { createHash, randomUUID } from "node:crypto"; +import { chmod, lstat, mkdir, readFile, rm, writeFile } from "node:fs/promises"; +import { dirname, join } from "node:path"; +import type { + EnvMigrationFileState, + EnvMigrationSnapshots, +} from "../../src/lib/env-migration-snapshot.ts"; + +const hash = (bytes: Uint8Array) => + createHash("sha256").update(bytes).digest("hex"); +type FixtureManifest = { + readonly files: readonly (EnvMigrationFileState & { + readonly payload: string | null; + })[]; +}; + +/** Synthetic snapshot owner for migration tests; replaced by HACK-2189 in production. */ +export function fixtureEnvSnapshots(root: string): EnvMigrationSnapshots { + const readManifest = async (path: string): Promise => + JSON.parse(await readFile(path, "utf8")); + return { + capture: async ({ files }) => { + const dir = join(root, `snapshot-${randomUUID()}`); + await mkdir(dir, { mode: 0o700 }); + const entries: FixtureManifest["files"][number][] = []; + for (const [index, file] of files.entries()) { + const payload = + file.sha256 === null ? null : join(dir, `${index}.original`); + if (payload) { + await writeFile(payload, await readFile(file.path), { mode: 0o600 }); + } + entries.push({ ...file, payload }); + } + const manifestPath = join(dir, "manifest.json"); + await writeFile(manifestPath, JSON.stringify({ files: entries }), { + mode: 0o600, + }); + return { manifestPath }; + }, + assertCoverage: async ({ manifestPath, files }) => { + const manifest = await readManifest(manifestPath); + for (const file of files) { + const stored = manifest.files.find((entry) => entry.path === file.path); + if ( + !stored || + stored.sha256 !== file.sha256 || + stored.mode !== file.mode || + stored.size !== file.size || + (stored.payload && + hash(await readFile(stored.payload)) !== file.sha256) + ) { + throw new Error("Fixture snapshot coverage failed"); + } + } + }, + recordAfter: () => Promise.resolve(), + rollback: async ({ manifestPath, expectedCurrent }) => { + const manifest = await readManifest(manifestPath); + const currentState = async (path: string) => { + if (!(await Bun.file(path).exists())) { + return { path, sha256: null, mode: null, size: 0 }; + } + const bytes = await readFile(path); + return { + path, + sha256: hash(bytes), + mode: (await lstat(path)).mode & 0o777, + size: bytes.length, + }; + }; + const matches = ( + left: EnvMigrationFileState, + right: EnvMigrationFileState + ) => + left.sha256 === right.sha256 && + left.mode === right.mode && + left.size === right.size; + for (const file of expectedCurrent) { + const original = manifest.files.find( + (entry) => entry.path === file.path + ); + const current = await currentState(file.path); + if ( + !(original && (matches(current, file) || matches(current, original))) + ) { + throw new Error("Fixture rollback refuses changed target"); + } + } + for (const expected of expectedCurrent) { + const file = manifest.files.find( + (entry) => entry.path === expected.path + ); + if (!file) { + throw new Error("Fixture snapshot target missing"); + } + if (matches(await currentState(file.path), file)) { + continue; + } + if (file.payload === null) { + await rm(file.path, { force: true }); + continue; + } + await mkdir(dirname(file.path), { recursive: true }); + await writeFile(file.path, await readFile(file.payload), { + mode: file.mode ?? 0o600, + }); + await chmod(file.path, file.mode ?? 0o600); + } + return { + status: "restored", + results: manifest.files.map((file) => file.path), + }; + }, + }; +} From 00cf467366e51e0c0ab76d6280003ea0082c0165 Mon Sep 17 00:00:00 2001 From: Kit <344842+roodboi@users.noreply.github.com> Date: Fri, 13 Nov 2026 20:37:52 +0000 Subject: [PATCH 02/32] fix(env): guard recovery, retain shared keys and document migration protocol --- docs/env.md | 70 ++- docs/guides/upgrade-to-v5.md | 22 + docs/reference/cli.md | 26 + docs/reference/env-migration.md | 122 +++++ docs/reference/env-migration.mmd | 30 + docs/reference/env-migration.svg | 66 +++ docs/reference/v5-model-coverage.md | 1 + scripts/lib/tla-runtime-models.ts | 17 + src/commands/doctor.ts | 10 +- src/commands/env-migrate.ts | 34 +- src/commands/env.ts | 13 +- src/commands/project.ts | 8 +- src/commands/session.ts | 8 +- src/lib/env-migration-payloads.ts | 164 ++++++ src/lib/env-migration-snapshot.ts | 22 +- src/lib/env-migration.ts | 304 ++++++++--- src/lib/project-env-config.ts | 29 +- tests/env-migration-journal.test.ts | 543 +++++++++++++++---- tests/env-migration-preview-cli.test.ts | 155 ++++++ tests/env-migration-tracked-consent.test.ts | 62 +-- tests/helpers/env-migration-fixture.ts | 45 +- tests/legacy-env-cleanup.test.ts | 10 +- tests/missing-project-error-code.test.ts | 4 + tests/models/tla/env-migration/COVERAGE.md | 29 + tests/models/tla/env-migration/Migration.tla | 61 +++ tests/models/tla/env-migration/negative.cfg | 5 + tests/models/tla/env-migration/positive.cfg | 5 + 27 files changed, 1564 insertions(+), 301 deletions(-) create mode 100644 docs/reference/env-migration.md create mode 100644 docs/reference/env-migration.mmd create mode 100644 docs/reference/env-migration.svg create mode 100644 src/lib/env-migration-payloads.ts create mode 100644 tests/env-migration-preview-cli.test.ts create mode 100644 tests/models/tla/env-migration/COVERAGE.md create mode 100644 tests/models/tla/env-migration/Migration.tla create mode 100644 tests/models/tla/env-migration/negative.cfg create mode 100644 tests/models/tla/env-migration/positive.cfg diff --git a/docs/env.md b/docs/env.md index 4ae857d85..b62597c75 100644 --- a/docs/env.md +++ b/docs/env.md @@ -1298,53 +1298,49 @@ YAML inputs, including filesystem aliases. It does not recover keys already remo files until their original key is recovered; creating a replacement key is not recovery. -Hack still detects and reads that legacy format. After those backup checks, -the explicit migration path is: +Hack still detects and reads that legacy format. Preview the migration first: ```bash -hack doctor --migrate-env-config +hack env migrate --json +hack env migrate --apply --accept-tracked-env-changes ``` -That migrates the repo into: +Preview is read-only and lists file actions and key names, never values. Apply +creates a private snapshot outside the working tree and records idempotent steps +in a journal. Re-run it after interruption; a finished migration is a no-op. +`doctor --migrate-env-config` remains an applying alias. Both backends use the +same file migration; Compose users need no native runtime, and neither preview +nor apply needs Docker or a daemon. -- `.hack/hack.env.default.yaml` -- `.hack/hack.env..yaml` -- `.hack/hack.env.local.yaml` and `.hack/hack.env..local.yaml` for plain values -- `.hack.secret.key` when needed - -Tracked layers receive secret-backed ciphertext, not legacy plaintext. - -The migration runs the same way on the Compose and native backends: it changes project -files only, so native `hack doctor --migrate-env-config` needs no Docker and no native -runtime. +The migration writes tracked `hack.env.default.yaml` and overlay YAML layers, +private `hack.env.local.yaml` and overlay local layers for plain values, and a +private project key when required. Tracked layers receive secret-backed +ciphertext, never legacy plaintext. It can also repair Compose `env_file` +references and old config keys. A tracked edit always requires consent; +`--accept-tracked-env-changes` keeps its meaning and never bypasses safety checks. +Without a prompt, doctor refuses tracked edits before its fix steps write. -`hack.env.json` and the Compose file are shared, git-tracked configuration, so the -migration never deletes or edits a git-tracked file without your consent. Interactively, -doctor lists the git-tracked files that may change before it migrates, and asks before it -removes `env_file` entries from a tracked Compose file and, separately, before it removes -git-tracked legacy files such as `hack.env.json`. A file whose git status can't be read -inside a git checkout counts as tracked. A run that can't prompt (no TTY, -`HACK_NO_INTERACTIVE`, or `--no-interactive`) refuses with `E_INTERACTIVE_REQUIRED` before -any doctor step writes, naming each tracked file it would delete or edit, unless you pass -`--accept-tracked-env-changes`: +**Migration keeps all legacy files.** Cleanup is a separate later action: ```bash -hack doctor --migrate-env-config --no-interactive --accept-tracked-env-changes +hack up +hack env migrate --cleanup --json +hack env migrate --cleanup --apply --accept-tracked-env-changes ``` -The list is conservative; with a contract it can include `hack.config.json`, which the -migration rewrites when it still holds legacy keys such as `defaultEnvConfig`, and a legacy -dotenv file the migration ends up keeping. Commit the changes so teammates get them. - -Commands that need env values, such as `hack env exec`, also migrate a -project that has `hack.env.json` and no env layers, without asking. That migration reports -each moved key, never its value, and removes a legacy `.hack/.env` or `.hack/.env.` -file only when every entry now lives in an env layer, Compose no longer names it as -`env_file`, Compose doesn't interpolate a key from it that the env layers now set only for a -service, and git doesn't track it, so no second plaintext copy stays behind. It keeps -`hack.env.json`, never edits the Compose file and never deletes a git-tracked file; it names -a kept file and says why. Run `hack doctor --migrate-env-config` for those -changes. Like doctor, it rewrites legacy keys in `hack.config.json`. +Cleanup requires a successful complete-project run on the current layers, an +intact snapshot, fresh delivery of every migrated value and the live-input guard. +Current env keys, config and YAML files are never cleanup targets, including +filesystem aliases. Use `--rollback-manifest ` to restore the snapshot; +later edits are refused. See the [versioned preview and recovery +contract](reference/env-migration.md) for the JSON shape, journal boundaries and +partial-cleanup recovery. + +Commands that need env values, such as `hack env exec`, can also migrate a +project that has `hack.env.json` and no env layers. These implicit migrations +report moved key names, retain every legacy copy and keep Compose `env_file` +references. They never delete a tracked file or silently edit tracked config. +Use the explicit preview/apply command when a shared config repair needs consent. A repo that has no `.hack/hack.env.json` is migrated as well. Its values live only in `.hack/.env` and `.hack/.env.`, which are gitignored and may hold passwords or diff --git a/docs/guides/upgrade-to-v5.md b/docs/guides/upgrade-to-v5.md index 6913afa03..0b024f7d8 100644 --- a/docs/guides/upgrade-to-v5.md +++ b/docs/guides/upgrade-to-v5.md @@ -114,6 +114,28 @@ See [Env and secrets](../env.md#legacy-repos-and-migration) and the [threat model](../security/env-secrets-threat-model.md). These are manual checks, not evidence that the real-machine upgrade or rollback has passed. +## Preview, resume and undo env migration + +In builds containing HACK-2191, use `hack env migrate --json` to review names and +file paths before applying. `hack env migrate --apply` snapshots originals outside +the working tree and records resumable steps; repeat the same command after an +interruption. Add `--accept-tracked-env-changes` only when consenting to the +previewed shared edits. `doctor --migrate-env-config` still applies the migration, +but now retains all legacy copies, including untracked dotenv files and tracked +`hack.env.json`. Implicit migrations retain them too. + +After checking a successful complete-project `hack up`, preview the separate +cleanup with `hack env migrate --cleanup --json`, then apply it explicitly with +`--cleanup --apply`. Cleanup rechecks delivery, live inputs and the original +snapshot. Restore with `hack env migrate --rollback-manifest `; it refuses +later edits and preserves replaced data. Keep snapshot blobs and the migration +journal until the upgrade is accepted. See the [env migration +protocol](../reference/env-migration.md). +If later edits block rollback, `--rollback-manifest --preserve-changes` +explicitly restores originals and keeps those edits privately for recovery. +Rollback retains project/shared keys and their required ignore entry because +another checkout may already depend on them. + ## The daemon after an upgrade Installing 5.0.0 over a running 4.x does not replace the daemon. On Linux the old diff --git a/docs/reference/cli.md b/docs/reference/cli.md index 46880255c..0bfb2a396 100644 --- a/docs/reference/cli.md +++ b/docs/reference/cli.md @@ -2476,6 +2476,7 @@ hack env [options] | Command | Summary | | --- | --- | +| `hack env migrate` | Preview, resume or undo project env migration; cleanup is a separate later step | | `hack env list` | List env names, scopes, layers and set state | | `hack env protect [keys...]` | List plaintext keys, or encrypt chosen keys in their existing layers | | `hack env get ` | Print one resolved env value without an added newline | @@ -2498,6 +2499,31 @@ hack env [options] | `--help, -h` | Show help | | `--version, -v` | Show version | +## `hack env migrate` + +Preview, resume or undo project env migration; cleanup is a separate later step + +### Usage + +```bash +hack env migrate [options] +``` + +### Options + +| Option | Description | +| --- | --- | +| `--path, -p ` | Run a project command against a repo path (overrides cwd search) | +| `--json` | Output JSON (machine-readable) | +| `--apply` | Apply the previewed migration with a snapshot and resumable journal | +| `--cleanup` | Preview later legacy cleanup; combine with --apply only after migration has completed | +| `--rollback-manifest ` | Restore the migration snapshot, refusing changes made afterward | +| `--accept-tracked-env-changes` | Consent to the displayed edits or removals of Git-tracked legacy env files | +| `--preserve-changes` | With rollback, restore originals and retain later edits privately in the snapshot | +| `--no-interactive` | Never prompt: apply documented defaults or fail with E_INTERACTIVE_REQUIRED (also via HACK_NO_INTERACTIVE=1) | +| `--help, -h` | Show help | +| `--version, -v` | Show version | + ## `hack env list` List env names, scopes, layers and set state diff --git a/docs/reference/env-migration.md b/docs/reference/env-migration.md new file mode 100644 index 000000000..e0ad3212e --- /dev/null +++ b/docs/reference/env-migration.md @@ -0,0 +1,122 @@ +# Env migration protocol + +![Journal publication and separate cleanup](env-migration.svg) + +The [diagram source](env-migration.mmd) uses the same control states and guards as +the [finite model](../../tests/models/tla/env-migration/COVERAGE.md). An idle state +can mean interrupted or complete; only completed migration can request cleanup. + +`hack env migrate` prepares a project migration without writing files. It works +with Compose on Linux and macOS and does not contact Docker, a daemon or a native +runtime. `--path` selects one project; every legacy overlay is included. + +```sh +hack env migrate --path ./app --json +hack env migrate --path ./app --apply --accept-tracked-env-changes +# Run and check the complete project before retiring its old copies. +hack up --path ./app +hack env migrate --path ./app --cleanup --json +hack env migrate --path ./app --cleanup --apply --accept-tracked-env-changes +hack env migrate --path ./app --rollback-manifest /private/snapshot/manifest.json +# Explicit recovery when a later edit prevents ordinary rollback: +hack env migrate --path ./app --rollback-manifest /private/snapshot/manifest.json --preserve-changes +``` + +Omit `--accept-tracked-env-changes` when the operation changes no tracked file. +It grants consent to tracked edits for apply, or tracked removals and config +pointer repair for cleanup. It never disables the delivery, snapshot or live-input +checks. Non-interactive invocations do not choose values or overwrite later edits. + +## Preview JSON, version 1 + +`--json` uses the CLI result envelope: `{ "ok": true, "data": ... }`. +The preview's `data` has these stable fields: + +| Field | Meaning | +| --- | --- | +| `version` | Integer `1`; consumers must reject unsupported versions. | +| `project` | `projectRoot`, `projectDir`, `projectName` and `serviceNames`; paths are absolute and canonical. | +| `status` | `ready` for unfinished migration work; `complete` when migration has completed; `recovery_required` for interrupted cleanup or rollback. | +| `action` | `preview`, or `cleanup` with `--cleanup`. | +| `dryRun` | Always `true` for a preview. | +| `files` | Ordered array of `{ path, action, keys, tracked }`. | +| `journalPath` | Absolute path where apply keeps its private journal; preview does not create it. | + +Each file `action` is `write` (absent destination), `change` (existing destination) +or `removable` (a legacy copy eligible for the separate cleanup checks). `keys` +contains names only; config, key and ignore files have an empty array. `tracked` +is conservative: unknown Git status in a checkout counts as tracked. A cleanup +preview includes only `removable` entries. These are candidates, not deletion +authority: apply checks the successful project run, original snapshot and current +delivery again. Finished publications are omitted on resume, and a completed +migration previews no further write or change. Confirmed removals are omitted. + +For example, an entry can be: + +```json +{"path":"/project/.hack/hack.env.local.yaml","action":"write","keys":["API_TOKEN"],"tracked":false} +``` + +There are no values, ciphertext or key bytes in this JSON. Preview does not create +keys, ignore entries, locks, journals, snapshots or env files. Inputs that cannot +be read safely refuse with a value-free error. Preview is an observation; apply +prepares and checks its own immutable plan against the inputs before publication. +The exact plan requires reading legacy values and checking delivery in memory. +A configured native credential backend can request its ordinary authorization; +Hack does not bypass that prompt or include the values in the preview. + +## Apply and recovery + +Apply takes a private snapshot of originals and absent destinations before any +project write. The journal and staged payloads live outside working trees under +Hack's private data directory. The snapshot manifest and journal hold only names, +paths, statuses, modes, sizes and SHA-256 checksums. Original bytes and staged +payloads are separate private files; they can contain values and keys and must +remain private. + +Each publication has an original checksum, an intended checksum and a completion +receipt. A retry skips completed steps and adopts a matching publication whose +rename succeeded before its receipt. An unexpected input or destination change +refuses instead of guessing. Re-run the same apply command after interruption. +After completion it returns `already_complete` without rewriting the project or +journal. `doctor --migrate-env-config` uses the same apply/journal path and keeps +all legacy files. Implicit migrations from env, session and startup commands also +keep legacy files and resume an unfinished journal; they preserve existing +Compose `env_file` references. +Implicit commands refuse an unfinished apply or rollback journal before using +partially published layers. Resume the explicit apply or rollback command first. + +Completed publication payloads are removed; retries also sweep receipted orphan +staging directories. Unknown or edited entries are retained, reported as +`payloadCleanup: "kept"` in the apply result. Original snapshot blobs remain for +rollback. A private data home inside a Git working tree refuses apply; choose a +private `HACK_HOME` outside repositories, including a home managed as dotfiles. + +Migration never includes cleanup. A later explicit cleanup requires: + +1. A completed migration and a successful full-project `hack up`; service-only + startup does not qualify. Layer changes since that run require another run. + The run uses its selected overlay. Other overlays receive a fresh static + delivery check; this is not a claim that each overlay was run. +2. An intact snapshot containing each original cleanup target. +3. Fresh agreement between legacy values and current YAML delivery, under the + existing source/destination writer locks. +4. The live-input guard at selection and immediately before each deletion. + Current project/shared keys, configuration and YAML layers, including aliases + and hard links, cannot be removed. + +Legacy stores leave before their keys. Confirmed store removal also repairs its +config pointer. A partial cleanup preserves confirmed removals in the journal and +reports the error. If cleanup is interrupted, use its rollback manifest before +retrying; it does not infer fresh deletion authority from an old receipt. + +Rollback restores originals before reversing publications, so legacy values are +available before new layers leave resolution. Project keys remain in place, +including a newly created shared key: another checkout may already use it. The +root key ignore entry is retained with a checkout-local key. The rollback result +names these paths in `retained`; key removal is not part of rollback. +Ordinary rollback refuses later edits. Explicit `--preserve-changes` restores the +originals and keeps replaced bytes privately in the snapshot instead; it does not +select or merge values. Rollback can resume an incomplete restoration. Keep the manifest, blobs and journal together +until the upgrade is accepted. Neither file-level atomic writes nor cooperative +locks protect against an unrelated process replacing files outside Hack's locks. diff --git a/docs/reference/env-migration.mmd b/docs/reference/env-migration.mmd new file mode 100644 index 000000000..20373b65b --- /dev/null +++ b/docs/reference/env-migration.mmd @@ -0,0 +1,30 @@ +--- +title: Migration · journal and separate cleanup +--- +stateDiagram-v2 + direction TB + [*] --> preview + preview --> apply : Prepare [snapshot] + apply --> apply : Publish / Record + apply --> idle : Crash / Finish + idle --> apply : Resume [unfinished] + idle --> idle : SuccessfulRun / DamageBackup + idle --> cleanup : RequestCleanup [proof] + cleanup --> idle : RemoveLegacy [safe] + idle --> rolled_back : Rollback [snapshot] + rolled_back --> apply : Prepare + note right of apply + INV legacy retained + Receipts follow publication + end note + note right of cleanup + INV intact snapshot + INV no live targets + Fresh delivery required + end note + classDef live stroke-width:1.6px + classDef wait stroke-width:1.6px + classDef idle stroke-dasharray:4 3 + class apply live + class cleanup wait + class preview,rolled_back idle diff --git a/docs/reference/env-migration.svg b/docs/reference/env-migration.svg new file mode 100644 index 000000000..fff80b11d --- /dev/null +++ b/docs/reference/env-migration.svg @@ -0,0 +1,66 @@ +Migration · journal and separate cleanupPrepare [snapshot]Publish / RecordCrash / FinishResume [unfinished]SuccessfulRun /DamageBackupRequestCleanup [proof]RemoveLegacy [safe]Rollback [snapshot]Preparepreviewidlerolled_backapplycleanupINV legacy retainedReceipts follow publicationINV intact snapshotINV no live targetsFresh delivery required \ No newline at end of file diff --git a/docs/reference/v5-model-coverage.md b/docs/reference/v5-model-coverage.md index aeaa28ce1..1675fd00d 100644 --- a/docs/reference/v5-model-coverage.md +++ b/docs/reference/v5-model-coverage.md @@ -36,6 +36,7 @@ must be attached to the eventual exact integration revision. | Property and critical states | Model coverage | Implementation/regression boundary | Remaining qualification | | --- | --- | --- | --- | +| Env migration publication, crash/resume and later cleanup | [env-migration](../../tests/models/tla/env-migration/COVERAGE.md): 35 states, two publication steps, separate publication and receipt; unsafe same-run deletion fails `NoSameRunCleanup` | `env-migration.ts` binds immutable payloads and source checksums to a private snapshot, adopts interrupted renames, records completion, requires a successful project run and fresh snapshot/delivery/live-input proofs for explicit cleanup. Real-file tests check preview parity, sentinels, backup damage and rollback; owned child processes receive SIGKILL at every journal/publication boundary | The model assumes atomic per-file publication/journal writes and cooperative locks; it does not qualify fsync durability, real snapshot provider behavior, partial cleanup/rollback, unsynchronized writers, actual runtime environment or macOS. Implementation and dependency integration gates remain separate. | | Guest control ready before exposed bridge sockets bind; identity changes or the startup deadline expires | No new TLA+ model; deterministic clock schedules both readiness orders against real Unix socket nodes | `provider/bridge.rs` waits only in fresh `finish_boot`, before adoption, and keeps running audits immediate. `bridge_startup_tests.rs` covers two slots, late creation and mode tightening, permanent absence/unsafe modes, type/UID/link refusals, provider loss before/after observation and late success rejection. No wait mutates sockets or grants cleanup authority; the existing verified-provider stop handles failure. `lifecycle/boot_sequence_tests.rs` drives the real waiter through the production fresh-boot sequence and rejects audit/adoption on timeout, unsafe sockets or provider loss. `native-runtime-client.test.ts` checks the fixed runtime recover command and identical-root hint reach the frontend | Two sockets, finite scheduled events, injected provider verification and monotonic clock. Kernel process identity, provider scheduling, disk durability and successful recovery remain host boundaries. Require ten consecutive real-VM Mac Ctrl-C scenarios with pool stop/start; portable tests are not that evidence. | | CI cancellation must not lose merged-change coverage; only exact-head full evidence may qualify a prerelease | No maintained GitHub scheduler model. Finite fixture interleavings cover predecessor success/failure/cancellation, two competing full/scoped runs and two rerun attempts | `ci.yml` serializes next pushes in one non-cancelling group (GitHub cancels only the replaced pending run), cancels stale PR runs, and requires a successful before run for selection. `selectPrereleaseCiRun` reads current attempts; `requireBranchChecks` binds suite and job check IDs. `ci-trusted-policy`, `prerelease-plan` and `release-prepare-gate` regressions reject cancelled predecessors, green subsets, stale checks, skipped hosts and changed attempts | API snapshots and GitHub group/event semantics are assumptions; races after the last read remain possible. Publication rechecks the frozen protected head and full evidence but does not transactionally lock GitHub. Hosted dispatch/cancellation checks qualify wiring; no scheduler liveness or formal refinement claim. | | Native log follower versus lifecycle mutation, frontend loss, stopped container and transport loss | No new state model: the follower observes without a mutation lease and grants no cleanup authority | `provider/graph/log_follow.rs` and its native counterpart pin Ready admission, container and start generation; `provider/engine/log_follow.rs` bounds framing and distinguishes real EOF from stopped-service idle cutoff. `lifecycle_log_follow_tests.rs` keeps the production transport on a fake Engine socket active while acquiring startup and operation leases, with a same-lock contention control and output while each lease is held. `native-log-follow-source.test.ts` guards the observer-only route; process tests cover cancellation, frontend loss, real output-pipe closure while idle, replacement-container restart (including retained adopted run/owner IDs) with one marker, NDJSON admission failures, truncated/partial output and slow idle-end delivery. `native-log-restart.test.ts` covers bounded re-admission, unchanged-container refusal and prompt down; `native-log-selection.test.ts` covers strict finite admission responses and owned-child cancellation/timeouts; advisory markers grant no authority | 4 KiB payload envelopes, 8 KiB wire lines, 64 Ki-character retained payload prefixes excluding Engine timestamps, 30-second restart wait, 1000-line new-container backlog. The portable lease test fakes identity/state checks; it does not exercise host process/disk auditing, graph admission or the VM relay. Real relay EOF on stop/down/restart, VM loss and concurrent lifecycle latency require Apple Silicon acceptance. No performance or whole-product parity claim. | diff --git a/scripts/lib/tla-runtime-models.ts b/scripts/lib/tla-runtime-models.ts index 1bee5962b..28ca0e6dc 100644 --- a/scripts/lib/tla-runtime-models.ts +++ b/scripts/lib/tla-runtime-models.ts @@ -141,6 +141,23 @@ const restartPolicyStaleUses = [ { engine: "exited", bound: 3 }, ] as const; const contracts: readonly ModelContract[] = [ + { + name: "env-migration", + module: "Migration", + states: 35, + invariant: "NoSameRunCleanup", + action: "UnsafeRemoveDuringApply", + fields: [ + 'phase = "apply"', + "legacy = FALSE", + "snapshot = TRUE", + "separate = FALSE", + "published = 0", + "recorded = 0", + "complete = FALSE", + "runProof = FALSE", + ], + }, { name: "shutdown-rebase", module: "ShutdownRebase", diff --git a/src/commands/doctor.ts b/src/commands/doctor.ts index 36d807b8e..c20be5afb 100644 --- a/src/commands/doctor.ts +++ b/src/commands/doctor.ts @@ -4248,10 +4248,12 @@ async function maybeMigrateProjectEnvConfig(opts: { : "Env migration completed. Legacy files were kept for a separate cleanup step.", "env migration" ); - note( - `Journal: ${result.journalPath}\nRollback snapshot: ${result.manifestPath}\nAfter validating the project, preview cleanup with: hack env migrate --cleanup --path ${JSON.stringify(project.projectRoot)}`, - "env migration" - ); + if (result.manifestPath) { + note( + `Journal: ${result.journalPath}\nRollback snapshot: ${result.manifestPath}\nAfter validating the project, preview cleanup with: hack env migrate --cleanup --path ${JSON.stringify(project.projectRoot)}`, + "env migration" + ); + } await printMigrationProtectionHint(project); } diff --git a/src/commands/env-migrate.ts b/src/commands/env-migrate.ts index 343ad3dca..57827d14d 100644 --- a/src/commands/env-migrate.ts +++ b/src/commands/env-migrate.ts @@ -49,6 +49,13 @@ const optAccept = defineOption({ description: "Consent to the displayed edits or removals of Git-tracked legacy env files", } as const); +const optPreserve = defineOption({ + name: "preserveChanges", + type: "boolean", + long: "--preserve-changes", + description: + "With rollback, restore originals and retain later edits privately in the snapshot", +} as const); const options = [ optPath, @@ -57,6 +64,7 @@ const options = [ optCleanup, optRollback, optAccept, + optPreserve, ] as const; const spec = defineCommand({ name: "migrate", @@ -68,14 +76,9 @@ const spec = defineCommand({ subcommands: [], } as const); -export async function handleEnvMigrate({ - ctx, - args, -}: { - readonly ctx: CliContext; - readonly args: CommandArgs; -}): Promise { - const opts = args.options; +function validateMigrationOptions( + opts: CommandArgs["options"] +): void { if ( opts.rollbackManifest !== undefined && (opts.apply || opts.cleanup || opts.acceptTrackedChanges) @@ -87,6 +90,20 @@ export async function handleEnvMigrate({ if (opts.acceptTrackedChanges && !opts.apply) { throw new CliUsageError("--accept-tracked-env-changes requires --apply."); } + if (opts.preserveChanges && !opts.rollbackManifest) { + throw new CliUsageError("--preserve-changes requires --rollback-manifest."); + } +} + +export async function handleEnvMigrate({ + ctx, + args, +}: { + readonly ctx: CliContext; + readonly args: CommandArgs; +}): Promise { + const opts = args.options; + validateMigrationOptions(opts); const context = await findProjectContext(resolve(ctx.cwd, opts.path ?? ".")); if (!context) { throw new MissingProjectContextError(); @@ -116,6 +133,7 @@ export async function handleEnvMigrate({ project, manifestPath: resolve(ctx.cwd, opts.rollbackManifest), snapshots: await envMigrationSnapshots(), + preserveChanges: opts.preserveChanges === true, }); } else if (opts.apply) { data = await (opts.cleanup ? cleanupEnvMigration : applyEnvMigration)({ diff --git a/src/commands/env.ts b/src/commands/env.ts index 46dde3a4c..4c3fe2709 100644 --- a/src/commands/env.ts +++ b/src/commands/env.ts @@ -29,7 +29,7 @@ import { readEnvHydrateInput, } from "../lib/env-hydrate-input.ts"; import { - hasPendingEnvMigration, + assertEnvMigrationFinished, migrateLegacyEnvForCommand as migrateLegacyProjectEnv, } from "../lib/env-migration.ts"; import { @@ -741,9 +741,11 @@ async function resolveProjectForEnv(opts: { } if (opts.projectOpt !== undefined) { - return await resolveRegisteredProjectOption({ + const registered = await resolveRegisteredProjectOption({ projectOpt: opts.projectOpt, }); + await assertEnvMigrationFinished(registered); + return registered; } const startDir = opts.pathOpt @@ -753,6 +755,7 @@ async function resolveProjectForEnv(opts: { if (!project) { throw new MissingProjectContextError(); } + await assertEnvMigrationFinished(project); return project; } @@ -995,11 +998,11 @@ async function maybeMigrateLegacyProjectEnv(input: { readonly projectName: string; readonly reason: "mutation" | "runtime"; }): Promise { + await assertEnvMigrationFinished(input.project); if ( - !(await hasPendingEnvMigration(input.project)) && - (await projectEnvConfigExists({ + await projectEnvConfigExists({ projectDir: input.project.projectDir, - })) + }) ) { return false; } diff --git a/src/commands/project.ts b/src/commands/project.ts index 297e07fbb..711ceade5 100644 --- a/src/commands/project.ts +++ b/src/commands/project.ts @@ -1,5 +1,5 @@ import { - hasPendingEnvMigration, + assertEnvMigrationFinished, migrateLegacyEnvForCommand as migrateLegacyProjectEnv, recordEnvMigrationProjectRun, } from "../lib/env-migration.ts"; @@ -1383,11 +1383,11 @@ async function maybePromptLegacyProjectEnvMigration(opts: { readonly projectName: string; readonly serviceNames: readonly string[]; }): Promise { + await assertEnvMigrationFinished(opts.project); if ( - !(await hasPendingEnvMigration(opts.project)) && - (await projectEnvConfigExists({ + await projectEnvConfigExists({ projectDir: opts.project.projectDir, - })) + }) ) { return false; } diff --git a/src/commands/session.ts b/src/commands/session.ts index 0824d23d7..574314026 100644 --- a/src/commands/session.ts +++ b/src/commands/session.ts @@ -14,7 +14,7 @@ import { import { resolveRequestedEnvName } from "../cli/env-selection.ts"; import { optEnv, optJson, optPretty } from "../cli/options.ts"; import { - hasPendingEnvMigration, + assertEnvMigrationFinished, migrateLegacyEnvForCommand as migrateLegacyProjectEnv, } from "../lib/env-migration.ts"; import { @@ -1049,11 +1049,11 @@ async function maybeMigrateLegacySessionEnv(opts: { readonly project: ProjectContext; readonly projectName: string; }): Promise { + await assertEnvMigrationFinished(opts.project); if ( - !(await hasPendingEnvMigration(opts.project)) && - (await projectEnvConfigExists({ + await projectEnvConfigExists({ projectDir: opts.project.projectDir, - })) + }) ) { return; } diff --git a/src/lib/env-migration-payloads.ts b/src/lib/env-migration-payloads.ts new file mode 100644 index 000000000..a113cbbba --- /dev/null +++ b/src/lib/env-migration-payloads.ts @@ -0,0 +1,164 @@ +import { randomUUID } from "node:crypto"; +import { lstat, mkdir, readdir, rmdir, unlink } from "node:fs/promises"; +import { dirname, join } from "node:path"; +import { migrationRefusal, readMigrationFile } from "./env-migration-files.ts"; +import { isRecord } from "./guards.ts"; +import { replacePrivateFileAtomic } from "./private-file.ts"; + +const DIRECTORY = /^payloads-[a-f0-9-]{36}$/; +const NAME = /^[0-9]+\.payload$/; +const SHA = /^[a-f0-9]{64}$/; + +/** An exclusive directory gets a value-free ownership receipt before any payload bytes. */ +export async function stageMigrationPayloads(input: { + readonly journalPath: string; + readonly projectRoot: string; + readonly files: readonly { readonly text: string; readonly sha256: string }[]; +}): Promise { + const directory = join( + dirname(input.journalPath), + `payloads-${randomUUID()}` + ); + await mkdir(directory, { mode: 0o700 }); + const entries = input.files.map((file, index) => ({ + name: `${index}.payload`, + sha256: file.sha256, + })); + await replacePrivateFileAtomic({ + path: join(directory, "receipt.json"), + directories: [directory], + text: JSON.stringify({ + version: 1, + kind: "hack-env-migration-payloads", + projectRoot: input.projectRoot, + entries, + }), + mode: 0o600, + refusal: migrationRefusal, + }); + const paths: string[] = []; + for (const [index, file] of input.files.entries()) { + const path = join(directory, `${index}.payload`); + await replacePrivateFileAtomic({ + path, + directories: [directory], + text: file.text, + mode: 0o600, + refusal: migrationRefusal, + }); + paths.push(path); + } + return paths; +} + +async function privateOwned( + path: string, + directory: boolean +): Promise { + const info = await lstat(path); + return ( + !info.isSymbolicLink() && + (directory ? info.isDirectory() : info.isFile() && info.nlink === 1) && + info.uid === process.getuid?.() && + (info.mode & 0o077) === 0 + ); +} + +async function removeOwnedDirectory( + directory: string, + projectRoot: string +): Promise { + if (!(await privateOwned(directory, true))) { + return false; + } + const receipt = join(directory, "receipt.json"); + if (!(await privateOwned(receipt, false))) { + return false; + } + const original = await readMigrationFile(receipt); + const raw: unknown = JSON.parse(original.text ?? "null"); + if ( + !( + isRecord(raw) && + raw.version === 1 && + raw.kind === "hack-env-migration-payloads" && + raw.projectRoot === projectRoot && + Array.isArray(raw.entries) + ) + ) { + return false; + } + const entries: { name: string; sha256: string }[] = []; + for (const entry of raw.entries) { + if ( + !( + isRecord(entry) && + typeof entry.name === "string" && + NAME.test(entry.name) && + typeof entry.sha256 === "string" && + SHA.test(entry.sha256) + ) + ) { + return false; + } + entries.push({ name: entry.name, sha256: entry.sha256 }); + } + const names = await readdir(directory); + if ( + names.some( + (name) => + name !== "receipt.json" && !entries.some((entry) => entry.name === name) + ) + ) { + return false; + } + for (const entry of entries) { + if (!names.includes(entry.name)) { + continue; + } + const path = join(directory, entry.name); + if ( + !(await privateOwned(path, false)) || + (await readMigrationFile(path)).state.sha256 !== entry.sha256 + ) { + return false; + } + } + // Only recognized, private staging artifacts are removed. Unknown or edited + // entries keep the whole directory; snapshots and project keys are excluded. + for (const name of names.filter((name) => name !== "receipt.json")) { + await unlink(join(directory, name)); + } + if ( + (await readMigrationFile(receipt)).state.sha256 !== original.state.sha256 + ) { + return false; + } + await unlink(receipt); + await rmdir(directory); + return true; +} + +/** Sweep completed stages and pre-journal orphans while the project journal lock is held. */ +export async function purgeMigrationPayloads(input: { + readonly journalPath: string; + readonly projectRoot: string; + readonly retain: readonly string[]; +}): Promise { + let complete = true; + for (const name of await readdir(dirname(input.journalPath))) { + if (!DIRECTORY.test(name)) { + continue; + } + const directory = join(dirname(input.journalPath), name); + if (input.retain.some((path) => dirname(path) === directory)) { + continue; + } + const removed = await removeOwnedDirectory( + directory, + input.projectRoot + ).catch(() => false); + complete = removed && complete; + } + return complete; +} diff --git a/src/lib/env-migration-snapshot.ts b/src/lib/env-migration-snapshot.ts index 1d2658424..ebe63af23 100644 --- a/src/lib/env-migration-snapshot.ts +++ b/src/lib/env-migration-snapshot.ts @@ -34,6 +34,7 @@ export type EnvMigrationSnapshots = { readonly rollback: (input: { readonly manifestPath: string; readonly expectedCurrent: readonly EnvMigrationFileState[]; + readonly preserveChanges?: boolean; }) => Promise<{ readonly status: "restored" | "incomplete"; readonly results: readonly unknown[]; @@ -109,7 +110,9 @@ export function bindEnvMigrationSnapshots( }; const collectPending = async ( snapshot: Awaited>, - expectedCurrent: readonly EnvMigrationFileState[] + expectedCurrent: readonly EnvMigrationFileState[], + preserveChanges: boolean, + manifestPath: string ) => { const pending: EnvMigrationFileState[] = []; for (const expected of expectedCurrent) { @@ -120,10 +123,12 @@ export function bindEnvMigrationSnapshots( if (isRecord(original) && matches(original.before, current)) { continue; } - if (!sameMigrationFile(current, expected)) { - throw migrationRefusal(); + if (!(sameMigrationFile(current, expected) || preserveChanges)) { + throw new ClosedCliError( + `Env rollback refused a changed file: ${JSON.stringify(expected.path)}. Review it, or restore originals while keeping later bytes privately with hack env migrate --rollback-manifest ${JSON.stringify(manifestPath)} --preserve-changes (values omitted).` + ); } - pending.push(expected); + pending.push(preserveChanges ? current : expected); } return pending; }; @@ -184,9 +189,14 @@ export function bindEnvMigrationSnapshots( } await verifyCurrent(files); }, - rollback: async ({ manifestPath, expectedCurrent }) => { + rollback: async ({ manifestPath, expectedCurrent, preserveChanges }) => { const snapshot = await load(manifestPath); - const pending = await collectPending(snapshot, expectedCurrent); + const pending = await collectPending( + snapshot, + expectedCurrent, + preserveChanges === true, + manifestPath + ); // A killed publication may predate the provider's after receipt; only journal-proven bytes may fill that gap. const recorded = await invoke("recordUpgradeSnapshotAfter", { manifestPath, diff --git a/src/lib/env-migration.ts b/src/lib/env-migration.ts index ab0c02142..7cd2869ed 100644 --- a/src/lib/env-migration.ts +++ b/src/lib/env-migration.ts @@ -1,4 +1,4 @@ -import { createHash, randomUUID } from "node:crypto"; +import { createHash } from "node:crypto"; import { lstat, realpath } from "node:fs/promises"; import { basename, @@ -18,6 +18,10 @@ import { migrationRefusal as refuse, sameMigrationFile as same, } from "./env-migration-files.ts"; +import { + purgeMigrationPayloads, + stageMigrationPayloads, +} from "./env-migration-payloads.ts"; import type { EnvMigrationFileState, EnvMigrationSnapshots, @@ -32,7 +36,6 @@ import { acquireLegacyEnvCleanupContext, assertMachineLocalFileGitPrivate, assertProjectEnvLayerPaths, - discoverComposeServiceNames, isGitTrackedOrUnknownProjectPath, LegacyEnvCleanupError, type LegacyProjectEnvMigration, @@ -61,7 +64,7 @@ export type EnvMigrationProject = { export type EnvMigrationPreview = { readonly version: 1; readonly project: EnvMigrationProject; - readonly status: "ready" | "complete"; + readonly status: "ready" | "complete" | "recovery_required"; readonly files: readonly { readonly path: string; readonly action: "write" | "change" | "removable"; @@ -89,7 +92,8 @@ type Journal = { | "complete" | "cleaning" | "cleaned" - | "rolled_back"; + | "rolled_back" + | "rolling_back"; readonly sources: readonly EnvMigrationFileState[]; readonly writes: readonly Publication[]; readonly removable: readonly string[]; @@ -125,7 +129,12 @@ async function canonicalProject( ) { throw refuse(); } - return { ...project, projectRoot, projectDir }; + return { + projectRoot, + projectDir, + projectName: project.projectName, + serviceNames: [...project.serviceNames], + }; } /** Location calculation is read-only, including when the data home has not been created. */ @@ -144,10 +153,22 @@ export async function hasPendingEnvMigration(input: { projectName: "project", serviceNames: [], }); - return ( - (await readJournal(envMigrationJournalPath(project.projectRoot), project)) - ?.state === "applying" - ); + const state = ( + await readJournal(envMigrationJournalPath(project.projectRoot), project) + )?.state; + return state === "applying" || state === "rolling_back"; +} + +/** A partial YAML publication must never become ordinary command input. */ +export async function assertEnvMigrationFinished(input: { + readonly projectRoot: string; + readonly projectDir: string; +}): Promise { + if (await hasPendingEnvMigration(input)) { + throw new ClosedCliError( + "An env migration is unfinished. Resume with hack env migrate --apply (add --accept-tracked-env-changes for its tracked edits), or use its rollback manifest. No project command was run." + ); + } } async function assertOutsideWorkingTrees( @@ -155,19 +176,25 @@ async function assertOutsideWorkingTrees( project: EnvMigrationProject ): Promise { if (!isAbsolute(path)) { - throw refuse(); + throw new ClosedCliError( + "Env migration requires private journal and snapshot storage outside Git working trees. Choose a HACK_HOME outside your repositories; preview remains read-only." + ); } const inside = relative(project.projectRoot, path); if ( !(inside === ".." || inside.startsWith(`..${sep}`) || isAbsolute(inside)) ) { - throw refuse(); + throw new ClosedCliError( + "Env migration requires private journal and snapshot storage outside Git working trees. Choose a HACK_HOME outside your repositories; preview remains read-only." + ); } for (let parent = dirname(path); ; parent = dirname(parent)) { for (const name of [".git"]) { try { await lstat(join(parent, name)); - throw refuse(); + throw new ClosedCliError( + "Env migration requires private journal and snapshot storage outside Git working trees. Choose a HACK_HOME outside your repositories; preview remains read-only." + ); } catch (error) { if (!(isRecord(error) && error.code === "ENOENT")) { throw error; @@ -281,9 +308,14 @@ async function readJournal( raw.project.projectDir === project.projectDir && typeof raw.manifestPath === "string" && isAbsolute(raw.manifestPath) && - ["applying", "complete", "cleaning", "cleaned", "rolled_back"].includes( - String(raw.state) - ) && + [ + "applying", + "complete", + "cleaning", + "cleaned", + "rolled_back", + "rolling_back", + ].includes(String(raw.state)) && Array.isArray(raw.sources) && raw.sources.every(validState) && Array.isArray(raw.writes) && @@ -389,6 +421,13 @@ export async function previewEnvMigration( if (journal && journal.state !== "rolled_back") { const files: EnvMigrationPreview["files"][number][] = []; for (const file of journal.writes) { + if ( + journal.state !== "applying" || + file.done || + same((await readState(file.path)).state, file) + ) { + continue; + } files.push({ path: file.path, action: file.before.sha256 === null ? "write" : "change", @@ -400,6 +439,9 @@ export async function previewEnvMigration( }); } for (const removable of journal.removable) { + if (journal.removed.includes(removable)) { + continue; + } files.push({ path: removable, action: "removable", @@ -413,7 +455,7 @@ export async function previewEnvMigration( return { version: 1, project, - status: journal.state === "applying" ? "ready" : "complete", + status: previewStatus(journal), files, journalPath: path, }; @@ -425,6 +467,13 @@ export async function previewEnvMigration( ); } +function previewStatus(journal: Journal): EnvMigrationPreview["status"] { + if (["cleaning", "rolling_back"].includes(journal.state)) { + return "recovery_required"; + } + return journal.state === "applying" ? "ready" : "complete"; +} + async function assertConsent( project: EnvMigrationProject, paths: readonly string[], @@ -436,7 +485,7 @@ async function assertConsent( for (const path of paths) { if (await isGitTrackedOrUnknownProjectPath({ ...project, path })) { throw new ClosedCliError( - "Env migration needs consent to change tracked files. Pass --accept-tracked-env-changes; no files were changed." + "Env migration needs consent to change tracked files. Run hack env migrate --apply --accept-tracked-env-changes; no files were changed." ); } } @@ -463,7 +512,9 @@ async function assertSources( ? !same(current, write) : !(same(current, source) || (write && same(current, write))) ) { - throw refuse(); + throw new ClosedCliError( + `Env migration input changed: ${JSON.stringify(source.path)}. Keep the legacy files and snapshot. To restore originals while retaining later edits privately, use hack env migrate --rollback-manifest ${JSON.stringify(journal.manifestPath)} --preserve-changes (values omitted).` + ); } } } @@ -481,33 +532,31 @@ async function prepareJournal( await assertOutsideWorkingTrees(snapshot.manifestPath, project); await snapshots.assertCoverage({ ...snapshot, files: sources }); await snapshots.recordAfter({ ...snapshot, files: sources }); - const payloadDir = join(dirname(path), `payloads-${randomUUID()}`); - // A per-run directory is retained on interruption; the journal lists each payload. - await privateDirectory(payloadDir); - const writes: Publication[] = []; - for (const [index, file] of prepared.files.entries()) { + const payloads = prepared.files.map((file) => { + const text = prepared.payload(file.path); + return { text, sha256: digest(Buffer.from(text)) }; + }); + const staged = await stageMigrationPayloads({ + journalPath: path, + projectRoot: project.projectRoot, + files: payloads, + }); + const writes: Publication[] = prepared.files.map((file, index) => { const before = sources.find((entry) => entry.path === file.path); - if (!before) { + const payload = staged[index]; + const bytes = payloads[index]; + if (!(before && payload && bytes)) { throw refuse(); } - const text = prepared.payload(file.path); - const payload = `${index}.payload`; - await replacePrivateFileAtomic({ - path: join(payloadDir, payload), - directories: [payloadDir], - text, - mode: 0o600, - refusal: refuse, - }); - writes.push({ + return { ...file, - size: Buffer.byteLength(text), - sha256: digest(Buffer.from(text)), + size: Buffer.byteLength(bytes.text), + sha256: bytes.sha256, before, - payload: `${payloadDir}/${payload}`, + payload, done: false, - }); - } + }; + }); const journal: Journal = { version: 1, kind: "hack-env-migration", @@ -561,7 +610,7 @@ async function publishEntry( } await replacePrivateFileAtomic({ path: entry.path, - directories: [], + directories: [dirname(entry.path)], text: payload.text, mode: entry.mode ?? 0o600, refusal: refuse, @@ -592,6 +641,17 @@ async function publishEntry( } } +function retainedPayloads(journal: Journal | null): readonly string[] { + if (!(journal && ["applying", "cleaning"].includes(journal.state))) { + return []; + } + const files = journal.writes.map((write) => write.payload); + if (journal.cleanupConfig) { + files.push(journal.cleanupConfig.payload); + } + return files; +} + /** Uses recorded before/after images; an unrecorded successful rename is adopted on resume. */ export async function applyEnvMigration(input: { readonly project: EnvMigrationProject; @@ -604,12 +664,28 @@ export async function applyEnvMigration(input: { const path = envMigrationJournalPath(project.projectRoot); await assertOutsideWorkingTrees(path, project); const existing = await readJournal(path, project); + if (existing?.state === "rolling_back") { + throw new ClosedCliError( + `Env rollback is unfinished. Resume hack env migrate --rollback-manifest ${JSON.stringify(existing.manifestPath)}; no migration files were changed.` + ); + } if (existing && ["complete", "cleaned"].includes(existing.state)) { - return { - status: "already_complete" as const, - journalPath: path, - manifestPath: existing.manifestPath, - }; + return await withProjectsRegistryLock({ + lockPath: `${path}.lock`, + label: "env migration", + run: async () => ({ + status: "already_complete" as const, + journalPath: path, + manifestPath: existing.manifestPath, + payloadCleanup: (await purgeMigrationPayloads({ + journalPath: path, + projectRoot: project.projectRoot, + retain: [], + })) + ? "complete" + : "kept", + }), + }); } const preparation = existing && existing.state !== "rolled_back" @@ -623,6 +699,17 @@ export async function applyEnvMigration(input: { ? existing.writes.map((entry) => entry.path) : (preparation?.prepared.files.map((entry) => entry.path) ?? []); await assertConsent(project, changed, input.acceptTrackedChanges); + if ( + preparation && + preparation.prepared.files.length === 0 && + preparation.prepared.removable.length === 0 + ) { + return { + status: "already_complete" as const, + journalPath: path, + manifestPath: null, + }; + } await privateDirectory(dirname(dirname(path))); await privateDirectory(dirname(path)); return await withProjectsRegistryLock({ @@ -630,6 +717,12 @@ export async function applyEnvMigration(input: { label: "env migration", run: async () => { const retained = await readJournal(path, project); + const retain = retainedPayloads(retained); + await purgeMigrationPayloads({ + journalPath: path, + projectRoot: project.projectRoot, + retain, + }); let journal = retained && retained.state !== "rolled_back" ? retained @@ -687,7 +780,13 @@ export async function applyEnvMigration(input: { journal = { ...journal, state: "complete" }; await writeMetadata(path, journal); await input.hooks?.boundary?.({ phase: "complete", index: -1 }); + const payloadCleanup = await purgeMigrationPayloads({ + journalPath: path, + projectRoot: project.projectRoot, + retain: [], + }); return { + payloadCleanup: payloadCleanup ? "complete" : "kept", status: "applied" as const, journalPath: path, manifestPath: journal.manifestPath, @@ -722,16 +821,18 @@ async function cleanupLocked( project: EnvMigrationProject, path: string ) { - let journal = await readJournal(path, project); + const selected = await readJournal(path, project); if ( - !journal || - journal.state === "applying" || - journal.state === "rolled_back" + !selected || + selected.state === "applying" || + selected.state === "rolled_back" || + selected.state === "rolling_back" ) { throw new ClosedCliError( "Finish the env migration before requesting cleanup; no legacy files were removed." ); } + let journal: Journal = selected; if (journal.state === "cleaned") { return { status: "already_cleaned" as const, @@ -757,23 +858,22 @@ async function cleanupLocked( ); } } + const repair = + journal.store && journal.removable.includes(journal.store.path) + ? await prepareLegacyEnvCleanupConfig(project) + : null; await assertConsent( project, - [...journal.removable, join(project.projectDir, "hack.config.json")], + [...journal.removable, ...(repair ? [repair.path] : [])], input.acceptTrackedChanges ); await input.snapshots.assertCoverage({ manifestPath: journal.manifestPath, - files: journal.sources.filter((source) => - journal.removable.includes(source.path) - ), + files: journal.sources, }); await assertSources(journal, true); const context = await acquireLegacyEnvCleanupContext({ ...project, - serviceNames: await discoverComposeServiceNames({ - composeFile: join(project.projectDir, "docker-compose.yml"), - }), overlayNames: journal.overlayNames, paths: journal.sources .filter( @@ -782,21 +882,19 @@ async function cleanupLocked( .map((source) => source.path), ...(journal.store ? { store: journal.store } : {}), }); - const repair = - journal.store && journal.removable.includes(journal.store.path) - ? await prepareLegacyEnvCleanupConfig(project) - : null; if (repair) { - const payloadDir = join(dirname(path), `payloads-${randomUUID()}`); - await privateDirectory(payloadDir); - const payload = join(payloadDir, "0.payload"); - await replacePrivateFileAtomic({ - path: payload, - directories: [payloadDir], - text: repair.text, - mode: 0o600, - refusal: refuse, - }); + const payload = ( + await stageMigrationPayloads({ + journalPath: path, + projectRoot: project.projectRoot, + files: [ + { text: repair.text, sha256: digest(Buffer.from(repair.text)) }, + ], + }) + )[0]; + if (!payload) { + throw refuse(); + } journal = { ...journal, cleanupConfig: { @@ -812,6 +910,7 @@ async function cleanupLocked( }, }; } + await assertSources(journal, true); journal = { ...journal, state: "cleaning" }; await writeMetadata(path, journal); const recordCleanup = async (removed: readonly string[]) => { @@ -867,6 +966,11 @@ async function cleanupLocked( state: "cleaned", removed: [...new Set([...journal.removed, ...removed])], }); + await purgeMigrationPayloads({ + journalPath: path, + projectRoot: project.projectRoot, + retain: [], + }); return { status: "cleaned" as const, journalPath: path, @@ -910,6 +1014,7 @@ export async function rollbackEnvMigration(input: { readonly project: EnvMigrationProject; readonly manifestPath: string; readonly snapshots: EnvMigrationSnapshots; + readonly preserveChanges?: boolean; }) { const project = await canonicalProject(input.project); const path = envMigrationJournalPath(project.projectRoot); @@ -927,22 +1032,37 @@ async function rollbackLocked( input: { readonly manifestPath: string; readonly snapshots: EnvMigrationSnapshots; + readonly preserveChanges?: boolean; }, project: EnvMigrationProject, path: string ) { - const journal = await readJournal(path, project); - if (!journal || journal.manifestPath !== resolve(input.manifestPath)) { + const selected = await readJournal(path, project); + if (!selected || selected.manifestPath !== resolve(input.manifestPath)) { throw refuse(); } + let journal: Journal = selected; + await input.snapshots.assertCoverage({ + manifestPath: journal.manifestPath, + files: journal.sources, + }); // Restore legacy sources first, then reverse publication: encrypted layers // leave resolution before a newly-created project key can be moved aside. + const retainedKeys = journal.writes.filter((write) => write.kind === "key"); + const retainedPaths = new Set(retainedKeys.map((write) => write.path)); + if ( + retainedKeys.some( + (write) => write.path === join(project.projectRoot, ".hack.secret.key") + ) + ) { + retainedPaths.add(join(project.projectRoot, ".gitignore")); + } const restoreOrder = [ ...journal.sources.filter( (source) => !journal.writes.some((write) => write.path === source.path) ), ...[...journal.writes].reverse(), - ]; + ].filter((source) => !retainedPaths.has(source.path)); const expectedCurrent: EnvMigrationFileState[] = []; for (const source of restoreOrder) { const current = (await readState(source.path)).state; @@ -963,15 +1083,37 @@ async function rollbackLocked( } expectedCurrent.push(expected); } + journal = { + ...journal, + state: "rolling_back", + removed: [ + ...new Set([ + ...journal.removed, + ...expectedCurrent + .filter( + (file) => + file.sha256 === null && journal.removable.includes(file.path) + ) + .map((file) => file.path), + ]), + ], + }; + await writeMetadata(path, journal); const result = await input.snapshots.rollback({ manifestPath: journal.manifestPath, expectedCurrent, + preserveChanges: input.preserveChanges, }); if (result.status !== "restored") { return result; } await writeMetadata(path, { ...journal, state: "rolled_back" }); - return result; + await purgeMigrationPayloads({ + journalPath: path, + projectRoot: project.projectRoot, + retain: [], + }); + return { ...result, retained: [...retainedPaths] }; } /** Compatibility bridge for existing implicit command routes; publication is journalled and cleanup is deferred. */ @@ -981,13 +1123,18 @@ export async function migrateLegacyEnvForCommand( const canonical = await canonicalProject(project); const path = envMigrationJournalPath(canonical.projectRoot); const retained = await readJournal(path, canonical); + const preparation = + retained && retained.state !== "rolled_back" + ? null + : await prepareConsistent({ ...canonical, repairCompose: false }); const preview = retained && retained.state !== "rolled_back" ? await previewEnvMigration(canonical) : await previewPrepared( canonical, - (await prepareConsistent({ ...canonical, repairCompose: false })) - .prepared, + preparation?.prepared ?? + (await prepareConsistent({ ...canonical, repairCompose: false })) + .prepared, path ); const result = await applyEnvMigration({ @@ -1016,8 +1163,9 @@ export async function migrateLegacyEnvForCommand( cleanupCandidates: preview.files .filter((file) => file.action === "removable") .map((file) => file.path), - blockedCleanupCandidates: [], + blockedCleanupCandidates: + preparation?.prepared.blockedCleanupCandidates ?? [], composeEnvFileReferences: [], - retainedEnvFiles: [], + retainedEnvFiles: preparation?.prepared.retainedEnvFiles ?? [], }; } diff --git a/src/lib/project-env-config.ts b/src/lib/project-env-config.ts index 4307b0bbe..8d8add011 100644 --- a/src/lib/project-env-config.ts +++ b/src/lib/project-env-config.ts @@ -6252,6 +6252,8 @@ export type PreparedLegacyEnvMigration = { readonly removable: readonly string[]; readonly overlayNames: readonly string[]; readonly hasContract: boolean; + readonly retainedEnvFiles: readonly RetainedLegacyEnvFile[]; + readonly blockedCleanupCandidates: readonly string[]; readonly store?: { readonly path: string; readonly keyPath: string }; /** Private in-memory payload: never serialize this return value. */ readonly payload: (path: string) => string; @@ -6682,6 +6684,24 @@ async function prepareLegacyProjectEnvMigrationInternal(opts: { ); }; const layersExist = await projectEnvConfigExistsForMutation(opts); + if ( + !contract.exists && + layersExist && + !(await canMigrateCheckoutLegacyDotenv(opts)) + ) { + return { + files: [], + sourcePaths: [], + removable: [], + overlayNames: [], + hasContract: false, + retainedEnvFiles: [], + blockedCleanupCandidates: [], + payload: () => { + throw new ClosedCliError("No migration publication is required."); + }, + }; + } const { overlayNames, store, deliveredValues, configCandidates } = contract.exists ? await prepareContractMigration( @@ -6778,6 +6798,8 @@ async function prepareLegacyProjectEnvMigrationInternal(opts: { removable, overlayNames, hasContract: contract.exists, + retainedEnvFiles: candidates.retainedEnvFiles, + blockedCleanupCandidates: candidates.blocked, ...(store ? { store } : {}), payload: (path) => { const text = texts.get(path); @@ -10362,7 +10384,12 @@ export async function prepareLegacyEnvCleanupConfig(opts: { if (!existing) { return null; } - const parsed: unknown = JSON.parse(existing.text); + let parsed: unknown; + try { + parsed = JSON.parse(existing.text); + } catch { + throw legacyCleanupChanged(); + } if (!isRecord(parsed)) { throw legacyCleanupChanged(); } diff --git a/tests/env-migration-journal.test.ts b/tests/env-migration-journal.test.ts index 801c3e54b..42337fefd 100644 --- a/tests/env-migration-journal.test.ts +++ b/tests/env-migration-journal.test.ts @@ -10,19 +10,28 @@ import { writeFile, } from "node:fs/promises"; import { tmpdir } from "node:os"; -import { join } from "node:path"; +import { dirname, join, relative } from "node:path"; import { applyEnvMigration, cleanupEnvMigration, type EnvMigrationProject, + envMigrationJournalPath, previewEnvMigration, recordEnvMigrationProjectRun, rollbackEnvMigration, } from "../src/lib/env-migration.ts"; -import { resolveProjectEnvConfig } from "../src/lib/project-env-config.ts"; +import { migrationDigest } from "../src/lib/env-migration-files.ts"; +import { stageMigrationPayloads } from "../src/lib/env-migration-payloads.ts"; +import { + resolveProjectEnvConfig, + resolveProjectEnvSharedKeyPath, + setProjectEnvValue, +} from "../src/lib/project-env-config.ts"; import { resolveSecretStore } from "../src/lib/secret-store.ts"; import { restoreEnv } from "./helpers/env.ts"; import { fixtureEnvSnapshots } from "./helpers/env-migration-fixture.ts"; +import { fixtureChildren } from "./helpers/fixture-children.ts"; +import { runFixtureGit } from "./helpers/fixture-git.ts"; import { privateCliEnv } from "./helpers/private-cli-env.ts"; const SENTINEL = "synthetic-upgrade-journal-value"; @@ -37,6 +46,7 @@ const isolated = [ let root = ""; let saved: (string | undefined)[] = []; let project: EnvMigrationProject; +const commands = fixtureChildren("env-journal-command"); beforeEach(async () => { saved = isolated.map((key) => process.env[key]); @@ -73,6 +83,7 @@ beforeEach(async () => { }); afterEach(async () => { + await commands.stopAll(); for (const [index, key] of isolated.entries()) { restoreEnv(key, saved[index]); } @@ -81,6 +92,14 @@ afterEach(async () => { } }); +async function apply(input: Parameters[0]) { + const result = await applyEnvMigration(input); + if (!result.manifestPath) { + throw new Error("Fixture expected a snapshot-backed migration"); + } + return { ...result, manifestPath: result.manifestPath }; +} + async function projectFiles(): Promise> { const files = new Map(); const visit = async (path: string): Promise => { @@ -97,6 +116,37 @@ async function projectFiles(): Promise> { return files; } +test("an already modern project without legacy inputs previews complete and applies without creating state", async () => { + for (const name of ["hack.env.json", ".env", ".env.qa"]) { + await rm(join(project.projectDir, name)); + } + await writeFile( + join(project.projectDir, "hack.env.default.yaml"), + "version: 1\nenvironment: default\nsecretsprovider: project_key\nvalues:\n global: {}\n" + ); + const before = await projectFiles(); + const preview = await previewEnvMigration(project); + expect(preview.status).toBe("complete"); + expect(preview.files).toEqual([]); + const snapshots = { + ...fixtureEnvSnapshots(root), + capture: async (): Promise<{ manifestPath: string }> => { + throw new Error("No-op must not create snapshots"); + }, + }; + expect( + ( + await applyEnvMigration({ + project, + snapshots, + acceptTrackedChanges: false, + }) + ).status + ).toBe("already_complete"); + expect(await projectFiles()).toEqual(before); + expect(await Bun.file(preview.journalPath).exists()).toBe(false); +}); + test("preview makes no writes and exactly names what apply publishes; completed retry is a no-op", async () => { const before = await projectFiles(); const preview = await previewEnvMigration(project); @@ -104,7 +154,7 @@ test("preview makes no writes and exactly names what apply publishes; completed expect(await Bun.file(preview.journalPath).exists()).toBe(false); expect(JSON.stringify(preview)).not.toContain(SENTINEL); const snapshots = fixtureEnvSnapshots(root); - const result = await applyEnvMigration({ + const result = await apply({ project, snapshots, acceptTrackedChanges: true, @@ -137,12 +187,17 @@ test("preview makes no writes and exactly names what apply publishes; completed .TOKEN ).toBe(QA); const journal = await readFile(result.journalPath, "utf8"); + expect( + (await readdir(dirname(result.journalPath))).filter((name) => + name.startsWith("payloads-") + ) + ).toEqual([]); expect(journal).not.toContain(SENTINEL); expect(journal).not.toContain(QA); expect(await readFile(result.manifestPath, "utf8")).not.toContain(SENTINEL); expect( ( - await applyEnvMigration({ + await apply({ project, snapshots, acceptTrackedChanges: true, @@ -153,9 +208,36 @@ test("preview makes no writes and exactly names what apply publishes; completed expect(await readFile(result.journalPath, "utf8")).toBe(journal); }); +test("retry sweeps only receipted staging artifacts and retains unknown contents", async () => { + const preview = await previewEnvMigration(project); + await mkdir(dirname(preview.journalPath), { recursive: true, mode: 0o700 }); + const input = { + journalPath: preview.journalPath, + projectRoot: project.projectRoot, + files: [{ text: SENTINEL, sha256: migrationDigest(Buffer.from(SENTINEL)) }], + }; + const orphan = await stageMigrationPayloads(input); + const kept = await stageMigrationPayloads(input); + await writeFile( + join(dirname(kept[0] ?? ""), "user-note"), + "synthetic user note" + ); + const result = await apply({ + project, + snapshots: fixtureEnvSnapshots(root), + acceptTrackedChanges: true, + }); + expect(await Bun.file(orphan[0] ?? "").exists()).toBe(false); + expect(await readFile(kept[0] ?? "", "utf8")).toBe(SENTINEL); + expect( + await readFile(join(dirname(kept[0] ?? ""), "user-note"), "utf8") + ).toBe("synthetic user note"); + expect("payloadCleanup" in result && result.payloadCleanup).toBe("kept"); +}); + test("cleanup is explicit, rechecks delivery, and requires intact snapshot payloads", async () => { const snapshots = fixtureEnvSnapshots(root); - const result = await applyEnvMigration({ + const result = await apply({ project, snapshots, acceptTrackedChanges: true, @@ -177,7 +259,7 @@ test("cleanup is explicit, rechecks delivery, and requires intact snapshot paylo test("rollback restores original bytes and refuses a later user edit", async () => { const before = await projectFiles(); const snapshots = fixtureEnvSnapshots(root); - const result = await applyEnvMigration({ + const result = await apply({ project, snapshots, acceptTrackedChanges: true, @@ -201,10 +283,163 @@ test("rollback restores original bytes and refuses a later user edit", async () expect(await projectFiles()).toEqual(before); }); +test("a partial publication refuses implicit commands before they can use incomplete layers", async () => { + await expect( + apply({ + project, + snapshots: fixtureEnvSnapshots(root), + acceptTrackedChanges: true, + hooks: { + boundary: async ({ phase, index }) => { + if (phase === "published" && index === 1) { + throw new Error("synthetic interruption"); + } + }, + }, + }) + ).rejects.toThrow("synthetic interruption"); + for (const args of [ + ["env", "list"], + ["env", "get", "TOKEN"], + ["env", "exec", "--", "true"], + ]) { + const result = await commands.run({ + cmd: [ + process.execPath, + "--no-env-file", + join(import.meta.dir, "..", "index.ts"), + ...args, + "--path", + project.projectRoot, + ], + cwd: project.projectRoot, + env: privateCliEnv({ + root, + env: { + HACK_HOME: process.env.HACK_HOME, + HACK_NO_INTERACTIVE: "1", + PATH: "/nonexistent", + }, + }), + }); + expect(result.exitCode).not.toBe(0); + expect(result.stdout + result.stderr).toContain( + args[1] === "get" + ? "Unable to read env value." + : "An env migration is unfinished." + ); + expect(result.stdout).not.toContain(SENTINEL); + expect(result.stdout + result.stderr).not.toContain(SENTINEL); + } +}); + +test("explicit preserve-changes rollback recovers from an edited pending migration and keeps the edited bytes privately", async () => { + const snapshots = fixtureEnvSnapshots(root); + await expect( + apply({ + project, + snapshots, + acceptTrackedChanges: true, + hooks: { + boundary: async ({ phase }) => { + if (phase === "verified") { + throw new Error("synthetic interruption"); + } + }, + }, + }) + ).rejects.toThrow("synthetic interruption"); + const result = JSON.parse( + await readFile(envMigrationJournalPath(project.projectRoot), "utf8") + ); + expect(result.state).toBe("applying"); + const edited = "version: 1\n# synthetic later user edit\n"; + const layer = join(project.projectDir, "hack.env.local.yaml"); + await writeFile(layer, edited); + await expect( + rollbackEnvMigration({ + project, + snapshots, + manifestPath: result.manifestPath, + }) + ).rejects.toThrow("changed target"); + const restored = await rollbackEnvMigration({ + project, + snapshots, + manifestPath: result.manifestPath, + preserveChanges: true, + }); + expect(restored.status).toBe("restored"); + expect(await readFile(join(project.projectDir, ".env"), "utf8")).toContain( + SENTINEL + ); + const payloads = (await readdir(dirname(result.manifestPath))).filter( + (name) => name.startsWith("preserved-") + ); + expect( + await Promise.all( + payloads.map((name) => + readFile(join(dirname(result.manifestPath), name), "utf8") + ) + ) + ).toContain(edited); +}); + +test("incomplete rollback leaves its journal resumable and skips already restored files", async () => { + const before = await projectFiles(); + const base = fixtureEnvSnapshots(root); + const result = await apply({ + project, + snapshots: base, + acceptTrackedChanges: true, + }); + let first = true; + const snapshots = { + ...base, + rollback: async (input: Parameters[0]) => { + if (!first) { + return await base.rollback(input); + } + first = false; + const index = input.expectedCurrent.findIndex((file) => + file.path.endsWith("hack.env.qa.yaml") + ); + expect(index).toBeGreaterThanOrEqual(0); + const partial = await base.rollback({ + ...input, + expectedCurrent: input.expectedCurrent.slice(0, index + 1), + }); + return { ...partial, status: "incomplete" as const }; + }, + }; + expect( + ( + await rollbackEnvMigration({ + project, + snapshots, + manifestPath: result.manifestPath, + }) + ).status + ).toBe("incomplete"); + expect(JSON.parse(await readFile(result.journalPath, "utf8")).state).toBe( + "rolling_back" + ); + expect( + ( + await rollbackEnvMigration({ + project, + snapshots, + manifestPath: result.manifestPath, + }) + ).status + ).toBe("restored"); + expect(await projectFiles()).toEqual(before); +}); + test("cleanup snapshots must still hold every original; cleanup followed by rollback restores legacy files", async () => { const before = await projectFiles(); const snapshots = fixtureEnvSnapshots(root); - const result = await applyEnvMigration({ + const result = await apply({ project, snapshots, acceptTrackedChanges: true, @@ -234,7 +469,7 @@ test("cleanup snapshots must still hold every original; cleanup followed by roll expect(await projectFiles()).toEqual(before); }); -test("preview does not provision a new project key; apply encrypts with a private key and rollback retains every legacy value", async () => { +async function addLegacySecret() { await writeFile( join(project.projectDir, "hack.config.json"), JSON.stringify({ @@ -259,6 +494,10 @@ test("preview does not provision a new project key; apply encrypts with a privat ); const store = await resolveSecretStore(project); await store.set({ key: "SEC", value: SENTINEL }); +} + +test("preview does not provision a new project key; apply encrypts with a private key and rollback retains every legacy value", async () => { + await addLegacySecret(); const before = await projectFiles(); const keyPath = join(project.projectRoot, ".hack.secret.key"); const preview = await previewEnvMigration(project); @@ -270,7 +509,7 @@ test("preview does not provision a new project key; apply encrypts with a privat tracked: false, }); const snapshots = fixtureEnvSnapshots(root); - const result = await applyEnvMigration({ + const result = await apply({ project, snapshots, acceptTrackedChanges: true, @@ -285,12 +524,17 @@ test("preview does not provision a new project key; apply encrypts with a privat expect( await readFile(join(project.projectDir, "hack.config.json"), "utf8") ).not.toContain("encryptedFile"); - await rollbackEnvMigration({ + const restored = await rollbackEnvMigration({ project, snapshots, manifestPath: result.manifestPath, }); - expect(await projectFiles()).toEqual(before); + expect("retained" in restored && restored.retained).toContain(keyPath); + expect((await lstat(keyPath)).mode & 0o777).toBe(0o600); + const after = await projectFiles(); + after.delete(keyPath); + after.delete(join(project.projectRoot, ".gitignore")); + expect(after).toEqual(before); }); test("encrypted scoped values and store-only overlays survive; cleanup never removes their live project key", async () => { @@ -334,7 +578,7 @@ test("encrypted scoped values and store-only overlays survive; cleanup never rem .map((file) => file.path) ).not.toContain(keyPath); const snapshots = fixtureEnvSnapshots(root); - const result = await applyEnvMigration({ + const result = await apply({ project, snapshots, acceptTrackedChanges: true, @@ -362,106 +606,193 @@ test("encrypted scoped values and store-only overlays survive; cleanup never rem ).toBe(QA); }); +test("rollback retains a new shared key used by another checkout", async () => { + const primary = project; + const git = async (args: readonly string[]) => { + const result = await runFixtureGit({ cwd: primary.projectRoot, args }); + expect(result.exitCode, result.stderr).toBe(0); + }; + await git(["init", "--quiet"]); + await git([ + "add", + ".hack/hack.config.json", + ".hack/docker-compose.yml", + ".hack/hack.env.json", + ]); + await git(["commit", "--quiet", "-m", "fixture"]); + const linked = join(root, "linked"); + await git(["worktree", "add", "--quiet", "-b", "migration-fixture", linked]); + project = { + ...primary, + projectRoot: linked, + projectDir: join(linked, ".hack"), + }; + await writeFile(join(project.projectDir, ".env"), `TOKEN=${SENTINEL}\n`); + await writeFile(join(project.projectDir, ".env.qa"), `TOKEN=${QA}\n`); + await addLegacySecret(); + const shared = await resolveProjectEnvSharedKeyPath(project); + expect(shared).toBeString(); + const snapshots = fixtureEnvSnapshots(root); + const result = await apply({ + project, + snapshots, + acceptTrackedChanges: true, + }); + const keyBefore = await readFile(shared ?? "", "utf8"); + await writeFile( + join(primary.projectDir, "hack.env.default.yaml"), + await readFile(join(project.projectDir, "hack.env.default.yaml")) + ); + await setProjectEnvValue({ + ...primary, + envName: null, + scope: "global", + key: "SIBLING", + value: QA, + secret: true, + local: false, + }); + expect( + (await resolveProjectEnvConfig({ ...primary, envName: null }))?.globalEnv + .SIBLING + ).toBe(QA); + await rollbackEnvMigration({ + project, + snapshots, + manifestPath: result.manifestPath, + }); + expect(await readFile(shared ?? "", "utf8")).toBe(keyBefore); + expect( + (await resolveProjectEnvConfig({ ...primary, envName: null }))?.globalEnv + .SIBLING + ).toBe(QA); +}); + test("SIGKILL at every journal/publication boundary resumes without losing an overlay value", async () => { - const preview = await previewEnvMigration(project); - const count = preview.files.filter( - (file) => file.action !== "removable" - ).length; - const boundaries = [ - "prepared:-1", - ...Array.from({ length: count }, (_, index) => [ - `before_publish:${index}`, - `published:${index}`, - `recorded:${index}`, - ]).flat(), - "verified:-1", - "complete:-1", - ]; - // Each interruption owns a fresh HOME, data root, project and snapshot set. - for (const boundary of boundaries) { - const selected = join(root, boundary.replaceAll(":", "-")); - const selectedProject = { - ...project, - projectRoot: join(selected, "project"), - projectDir: join(selected, "project", ".hack"), - }; - await mkdir(selectedProject.projectDir, { recursive: true }); - for (const file of [ - "hack.config.json", - "hack.env.json", - "docker-compose.yml", - ".env", - ".env.qa", - ]) { - await writeFile( - join(selectedProject.projectDir, file), - await readFile(join(project.projectDir, file)) - ); + for (const scenario of ["plain", "encrypted"] as const) { + if (scenario === "encrypted") { + await addLegacySecret(); } - const dataHome = join(selected, "hack-home"); - const child = Bun.spawn( - [ - process.execPath, - "--no-env-file", - join(import.meta.dir, "helpers", "env-migration-child.ts"), - selected, - boundary, - ], - { - env: privateCliEnv({ root: selected, env: { HACK_HOME: dataHome } }), - stdin: "ignore", - stdout: "pipe", - stderr: "pipe", + const sources = await projectFiles(); + const preview = await previewEnvMigration(project); + const count = preview.files.filter( + (file) => file.action !== "removable" + ).length; + const boundaries = [ + "prepared:-1", + ...Array.from({ length: count }, (_, index) => [ + `before_publish:${index}`, + `published:${index}`, + `recorded:${index}`, + ]).flat(), + "verified:-1", + "complete:-1", + ]; + // Each interruption owns a fresh HOME, data root, project and snapshot set. + for (const boundary of boundaries) { + const selected = join( + root, + `${scenario}-${boundary.replaceAll(":", "-")}` + ); + const selectedProject = { + ...project, + projectRoot: join(selected, "project"), + projectDir: join(selected, "project", ".hack"), + }; + await mkdir(selectedProject.projectDir, { recursive: true }); + for (const [path, encoded] of sources) { + const target = join( + selectedProject.projectRoot, + relative(project.projectRoot, path) + ); + await mkdir(dirname(target), { recursive: true }); + await writeFile(target, Buffer.from(encoded, "base64"), { + mode: (await lstat(path)).mode & 0o777, + }); } - ); - const stderr = new Response(child.stderr).text(); - let output = ""; - let reached = false; - try { - const reader = child.stdout.getReader(); - for (;;) { - const chunk = await reader.read(); - if (chunk.done) { - break; - } - output += new TextDecoder().decode(chunk.value); - if (output.includes(`"phase":"${boundary.split(":")[0]}"`)) { - reached = true; - child.kill("SIGKILL"); - break; + const dataHome = join(selected, "hack-home"); + const child = Bun.spawn( + [ + process.execPath, + "--no-env-file", + join(import.meta.dir, "helpers", "env-migration-child.ts"), + selected, + boundary, + ], + { + env: privateCliEnv({ root: selected, env: { HACK_HOME: dataHome } }), + stdin: "ignore", + stdout: "pipe", + stderr: "pipe", } - } - await child.exited; - expect(reached, `${boundary}: ${await stderr}`).toBe(true); - expect(output).not.toContain(SENTINEL); - const previousHome = process.env.HACK_HOME; - process.env.HACK_HOME = dataHome; + ); + const stderr = new Response(child.stderr).text(); + let output = ""; + let reached = false; try { - await applyEnvMigration({ - project: selectedProject, - snapshots: fixtureEnvSnapshots(selected), - acceptTrackedChanges: true, - }); - expect( - (await resolveProjectEnvConfig({ ...selectedProject, envName: null })) - ?.globalEnv.TOKEN - ).toBe(SENTINEL); - expect( - (await resolveProjectEnvConfig({ ...selectedProject, envName: "qa" })) - ?.globalEnv.TOKEN - ).toBe(QA); - expect( - await readFile(join(selectedProject.projectDir, ".env"), "utf8") - ).toContain(SENTINEL); + const reader = child.stdout.getReader(); + for (;;) { + const chunk = await reader.read(); + if (chunk.done) { + break; + } + output += new TextDecoder().decode(chunk.value); + if (output.includes(`"phase":"${boundary.split(":")[0]}"`)) { + reached = true; + child.kill("SIGKILL"); + break; + } + } + await child.exited; + expect(reached, `${boundary}: ${await stderr}`).toBe(true); + expect(output).not.toContain(SENTINEL); + const previousHome = process.env.HACK_HOME; + process.env.HACK_HOME = dataHome; + try { + await apply({ + project: selectedProject, + snapshots: fixtureEnvSnapshots(selected), + acceptTrackedChanges: true, + }); + expect( + ( + await resolveProjectEnvConfig({ + ...selectedProject, + envName: null, + }) + )?.globalEnv.TOKEN + ).toBe(SENTINEL); + expect( + ( + await resolveProjectEnvConfig({ + ...selectedProject, + envName: "qa", + }) + )?.globalEnv.TOKEN + ).toBe(QA); + if (scenario === "encrypted") { + expect( + ( + await resolveProjectEnvConfig({ + ...selectedProject, + envName: null, + }) + )?.serviceEnv.api?.SEC + ).toBe(SENTINEL); + } + expect( + await readFile(join(selectedProject.projectDir, ".env"), "utf8") + ).toContain(SENTINEL); + } finally { + restoreEnv("HACK_HOME", previousHome); + } } finally { - restoreEnv("HACK_HOME", previousHome); - } - } finally { - if (child.exitCode === null) { - child.kill("SIGKILL"); + if (child.exitCode === null) { + child.kill("SIGKILL"); + } + await child.exited; + await stderr; } - await child.exited; - await stderr; } } -}, 60_000); +}, 120_000); diff --git a/tests/env-migration-preview-cli.test.ts b/tests/env-migration-preview-cli.test.ts new file mode 100644 index 000000000..8d964ebae --- /dev/null +++ b/tests/env-migration-preview-cli.test.ts @@ -0,0 +1,155 @@ +import { afterEach, expect, test } from "bun:test"; +import { + mkdir, + mkdtemp, + readdir, + readFile, + realpath, + rm, + writeFile, +} from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join, resolve } from "node:path"; +import { fixtureChildren } from "./helpers/fixture-children.ts"; +import { privateCliEnv, privateCliHome } from "./helpers/private-cli-env.ts"; + +const sentinel = "synthetic-preview-cli-value"; +const children = fixtureChildren("env-migration-preview"); +let root: string | null = null; +afterEach(async () => { + await children.stopAll(); + if (root) { + await rm(root, { recursive: true, force: true }); + } + root = null; +}); + +async function fixture() { + root = await realpath(await mkdtemp(join(tmpdir(), "env-preview-"))); + const project = join(root, "project"); + const projectDir = join(project, ".hack"); + await mkdir(projectDir, { recursive: true }); + await writeFile(join(projectDir, "hack.config.json"), '{"name":"preview"}\n'); + await writeFile( + join(projectDir, "docker-compose.yml"), + "services:\n api:\n image: alpine:3.22\n" + ); + await writeFile( + join(projectDir, "hack.env.json"), + JSON.stringify({ + version: 1, + vars: [{ key: "TOKEN", source: "plain_env" }], + }) + ); + await writeFile(join(projectDir, ".env"), `TOKEN=${sentinel}\n`); + const env = privateCliEnv({ + root, + env: { + HACK_HOME: join(root, "data"), + HACK_NO_INTERACTIVE: "1", + PATH: "/nonexistent", + }, + }); + return { root, project, projectDir, env, home: privateCliHome(root) }; +} + +test("source CLI preview JSON is versioned and value-free with no runtime, HOME state or project writes", async () => { + const selected = await fixture(); + const names = (await readdir(selected.projectDir)).sort(); + const before = await Promise.all( + names.map((name) => readFile(join(selected.projectDir, name), "utf8")) + ); + for (const flags of [[], ["--cleanup"]]) { + const result = await children.run({ + cmd: [ + process.execPath, + "--no-env-file", + resolve(import.meta.dir, "..", "index.ts"), + "env", + "migrate", + "--path", + selected.project, + "--json", + ...flags, + ], + cwd: selected.project, + env: selected.env, + }); + expect(result.exitCode, result.stderr).toBe(0); + expect(result.stdout + result.stderr).not.toContain(sentinel); + const output = JSON.parse(result.stdout); + expect(output).toMatchObject({ + ok: true, + data: { + version: 1, + status: "ready", + dryRun: true, + action: flags.length ? "cleanup" : "preview", + project: { + projectRoot: selected.project, + projectDir: selected.projectDir, + projectName: "preview", + serviceNames: ["api"], + }, + }, + }); + expect(Object.keys(output.data).sort()).toEqual( + [ + "version", + "status", + "dryRun", + "action", + "project", + "files", + "journalPath", + ].sort() + ); + for (const file of output.data.files) { + expect(Object.keys(file).sort()).toEqual( + ["path", "action", "keys", "tracked"].sort() + ); + if (flags.length) { + expect(file.action).toBe("removable"); + } + } + expect((await readdir(selected.projectDir)).sort()).toEqual(names); + expect( + await Promise.all( + names.map((name) => readFile(join(selected.projectDir, name), "utf8")) + ) + ).toEqual(before); + expect(await Bun.file(output.data.journalPath).exists()).toBe(false); + expect(await readdir(selected.home)).toEqual([]); + } +}); + +test("malformed legacy input cannot print its sentinel in JSON errors", async () => { + const selected = await fixture(); + await writeFile( + join(selected.projectDir, "hack.env.json"), + `{"vars":[${sentinel}` + ); + const result = await children.run({ + cmd: [ + process.execPath, + "--no-env-file", + resolve(import.meta.dir, "..", "index.ts"), + "env", + "migrate", + "--path", + selected.project, + "--json", + ], + cwd: selected.project, + env: selected.env, + }); + expect(result.exitCode).not.toBe(0); + expect(result.stdout + result.stderr).not.toContain(sentinel); + expect(JSON.parse(result.stdout).ok).toBe(false); + expect((await readdir(selected.projectDir)).sort()).toEqual([ + ".env", + "docker-compose.yml", + "hack.config.json", + "hack.env.json", + ]); +}); diff --git a/tests/env-migration-tracked-consent.test.ts b/tests/env-migration-tracked-consent.test.ts index 8b49125fb..57dc7ead7 100644 --- a/tests/env-migration-tracked-consent.test.ts +++ b/tests/env-migration-tracked-consent.test.ts @@ -15,7 +15,7 @@ import { privateCliEnv } from "./helpers/private-cli-env.ts"; /** * HACK-1904: the env migration never deletes or edits a git-tracked file * without consent, native doctor runs the same migration as Compose doctor, - * and the implicit route reports moved keys and drops its plaintext copy. + * and every applying route retains legacy copies for a separate cleanup step. * Every case uses a real temporary git repository and synthetic values. */ @@ -181,7 +181,7 @@ for (const backend of ["compose", "native"] as const) { expect(run.output).not.toContain(VALUE); expect(run.output).toContain("E_INTERACTIVE_REQUIRED"); expect(run.output).toContain( - "git-tracked files (edit .hack/docker-compose.yml; delete .hack/hack.env.json)" + "git-tracked files (edit .hack/docker-compose.yml)" ); expect(run.output).toContain("--accept-tracked-env-changes"); expect(run.output).not.toContain( @@ -212,16 +212,10 @@ for (const backend of ["compose", "native"] as const) { expect(run.output).toContain(KEY); expect(run.output).not.toContain(VALUE); expect(run.output).not.toContain("E_INTERACTIVE_REQUIRED"); - expect(run.output).toContain( - `Moved ${KEY} to .hack/hack.env.local.yaml (values omitted).` - ); expect( run.output.replace(/[│\r\n]/g, " ").replace(/\s+/g, " ") - ).toContain( - "These legacy env files are tracked by git, or are keys kept with a tracked store, and can now be removed. Teammates share the tracked files, so commit their removal together with the new env layers:" - ); + ).toContain("Legacy files were kept for a separate cleanup step."); expect(await status(project)).toEqual([ - " D .hack/hack.env.json", " M .hack/docker-compose.yml", "?? .hack/.gitignore", "?? .hack/hack.env.default.yaml", @@ -229,10 +223,13 @@ for (const backend of ["compose", "native"] as const) { expect( await readFile(join(project.projectDir, "hack.env.local.yaml"), "utf8") ).toContain(VALUE); - // The plaintext copy is gone once Compose no longer names it. - await expect( - readFile(join(project.projectDir, ".env"), "utf8") - ).rejects.toMatchObject({ code: "ENOENT" }); + // Migration does not authorize removal, even when Compose no longer names it. + expect(await readFile(join(project.projectDir, ".env"), "utf8")).toBe( + `${KEY}=${VALUE}\n` + ); + expect( + await readFile(join(project.projectDir, "hack.env.json"), "utf8") + ).toBe(CONTRACT); }, CLI_TEST_TIMEOUT_MS ); @@ -250,10 +247,11 @@ test( expect(run.exitCode, run.output).toBe(0); expect(run.output).not.toContain(VALUE); - expect(run.output).toContain(`Moved ${KEY} to .hack/hack.env.local.yaml`); + expect(run.output).toContain(KEY); expect(await status(project)).toEqual([ "?? .hack/.gitignore", "?? .hack/hack.env.default.yaml", + "?? .hack/hack.env.json", ]); }, CLI_TEST_TIMEOUT_MS @@ -268,7 +266,7 @@ function runImplicit(project: Project, env: Record = {}) { } test( - "implicit migration names moved keys, drops the plaintext copy and leaves tracked files alone", + "implicit migration names moved keys, retains the plaintext copy and leaves tracked files alone", async () => { const project = await createProject({ compose: COMPOSE_PLAIN }); @@ -279,7 +277,7 @@ test( expect(run.output).toContain( `Moved ${KEY} to .hack/hack.env.local.yaml (values omitted).` ); - expect(run.output).toContain("Removed .hack/.env"); + expect(run.output).toContain("Legacy env files were kept."); expect(await status(project)).toEqual([ "?? .hack/.gitignore", "?? .hack/hack.env.default.yaml", @@ -290,9 +288,9 @@ test( expect( await readFile(join(project.projectDir, "hack.env.local.yaml"), "utf8") ).toContain(VALUE); - await expect( - readFile(join(project.projectDir, ".env"), "utf8") - ).rejects.toMatchObject({ code: "ENOENT" }); + expect(await readFile(join(project.projectDir, ".env"), "utf8")).toBe( + `${KEY}=${VALUE}\n` + ); }, CLI_TEST_TIMEOUT_MS ); @@ -341,17 +339,15 @@ for (const route of ["implicit", "doctor"] as const) { expect(run.exitCode, run.output).toBe(0); expect(run.output).not.toContain(VALUE); - expect(run.output).toContain( - `Moved ${KEY} to .hack/hack.env.local.yaml (values omitted).` - ); - // Doctor wraps notes in a box and names absolute paths, so compare words. const words = run.output.replace(/[\s\u2502]+/g, " "); - expect(words).toContain( - `Compose interpolates ${KEY}, which the env layers now set only for a service, and service values don't reach interpolation (values omitted).` - ); if (route === "implicit") { - expect(run.output).toContain( - `Kept .hack/.env: Compose interpolates ${KEY}` + expect(words).toContain( + `Moved ${KEY} to .hack/hack.env.local.yaml (values omitted).` + ); + expect(words).toContain(`Kept .hack/.env: Compose interpolates ${KEY}`); + } else { + expect(words).toContain( + "Legacy files were kept for a separate cleanup step." ); } expect(words).not.toContain("Removed .hack/.env"); @@ -399,7 +395,7 @@ test( ); test( - "a service-scoped key Compose doesn't interpolate still drops the plaintext copy", + "a service-scoped key Compose does not interpolate still retains the plaintext copy", async () => { const project = await createProject({ compose: COMPOSE_PLAIN, @@ -410,10 +406,10 @@ test( expect(run.exitCode, run.output).toBe(0); expect(run.output).not.toContain(VALUE); - expect(run.output).toContain("Removed .hack/.env"); - await expect( - readFile(join(project.projectDir, ".env"), "utf8") - ).rejects.toMatchObject({ code: "ENOENT" }); + expect(run.output).toContain("Legacy env files were kept."); + expect(await readFile(join(project.projectDir, ".env"), "utf8")).toBe( + `${KEY}=${VALUE}\n` + ); }, CLI_TEST_TIMEOUT_MS ); diff --git a/tests/helpers/env-migration-fixture.ts b/tests/helpers/env-migration-fixture.ts index 6d0736f52..cd6434fc3 100644 --- a/tests/helpers/env-migration-fixture.ts +++ b/tests/helpers/env-migration-fixture.ts @@ -14,6 +14,32 @@ type FixtureManifest = { })[]; }; +async function restoreFixtureFile( + file: FixtureManifest["files"][number], + manifestPath: string, + preserveChanges: boolean +): Promise { + if (preserveChanges && (await Bun.file(file.path).exists())) { + await writeFile( + join( + dirname(manifestPath), + `preserved-${hash(await readFile(file.path))}` + ), + await readFile(file.path), + { mode: 0o600 } + ); + } + if (file.payload === null) { + await rm(file.path, { force: true }); + return; + } + await mkdir(dirname(file.path), { recursive: true }); + await writeFile(file.path, await readFile(file.payload), { + mode: file.mode ?? 0o600, + }); + await chmod(file.path, file.mode ?? 0o600); +} + /** Synthetic snapshot owner for migration tests; replaced by HACK-2189 in production. */ export function fixtureEnvSnapshots(root: string): EnvMigrationSnapshots { const readManifest = async (path: string): Promise => @@ -54,7 +80,7 @@ export function fixtureEnvSnapshots(root: string): EnvMigrationSnapshots { } }, recordAfter: () => Promise.resolve(), - rollback: async ({ manifestPath, expectedCurrent }) => { + rollback: async ({ manifestPath, expectedCurrent, preserveChanges }) => { const manifest = await readManifest(manifestPath); const currentState = async (path: string) => { if (!(await Bun.file(path).exists())) { @@ -81,7 +107,12 @@ export function fixtureEnvSnapshots(root: string): EnvMigrationSnapshots { ); const current = await currentState(file.path); if ( - !(original && (matches(current, file) || matches(current, original))) + !( + original && + (matches(current, file) || + matches(current, original) || + preserveChanges) + ) ) { throw new Error("Fixture rollback refuses changed target"); } @@ -96,15 +127,7 @@ export function fixtureEnvSnapshots(root: string): EnvMigrationSnapshots { if (matches(await currentState(file.path), file)) { continue; } - if (file.payload === null) { - await rm(file.path, { force: true }); - continue; - } - await mkdir(dirname(file.path), { recursive: true }); - await writeFile(file.path, await readFile(file.payload), { - mode: file.mode ?? 0o600, - }); - await chmod(file.path, file.mode ?? 0o600); + await restoreFixtureFile(file, manifestPath, preserveChanges === true); } return { status: "restored", diff --git a/tests/legacy-env-cleanup.test.ts b/tests/legacy-env-cleanup.test.ts index c6241c4ed..cb88c7ca4 100644 --- a/tests/legacy-env-cleanup.test.ts +++ b/tests/legacy-env-cleanup.test.ts @@ -99,7 +99,7 @@ test("unchanged sources are removed only after current delivery and Compose refe expect(await Bun.file(options.path).exists()).toBe(false); }); -test("implicit cleanup reports a real writer lock timeout without failing the command", async () => { +test("implicit migration defers cleanup without waiting on a legacy writer lock", async () => { const options = await fixture(); await writeFile( join(options.projectDir, "docker-compose.yml"), @@ -133,7 +133,9 @@ test("implicit cleanup reports a real writer lock timeout without failing the co migrated, }); expect(report.removed).toEqual([]); - expect(report.lines.join("\n")).toContain("writer may still hold a lock"); + expect(report.lines.join("\n")).toContain( + "preview the separate cleanup step" + ); expect(report.lines.join("\n")).not.toContain(SENTINEL); expect(await Bun.file(options.path).exists()).toBe(true); } finally { @@ -402,7 +404,7 @@ test("an equal-content replacement has a new identity and is kept", async () => expect(await readFile(options.path, "utf8")).toBe(current); }); -test("implicit migration also carries the source receipt into cleanup", async () => { +test("implicit migration retains a source changed after publication", async () => { const options = await fixture(); await writeFile( join(options.projectDir, "docker-compose.yml"), @@ -426,7 +428,7 @@ test("implicit migration also carries the source receipt into cleanup", async () migrated, }); expect(report.removed).toEqual([]); - expect(report.lines.join("\n")).toContain("kept remaining legacy env files"); + expect(report.lines.join("\n")).toContain("Legacy env files were kept."); expect(report.lines.join("\n")).not.toContain(SENTINEL); expect(await readFile(options.path, "utf8")).toBe(`TOKEN=${SENTINEL}-new\n`); }); diff --git a/tests/missing-project-error-code.test.ts b/tests/missing-project-error-code.test.ts index 7580eb3af..259c060db 100644 --- a/tests/missing-project-error-code.test.ts +++ b/tests/missing-project-error-code.test.ts @@ -134,6 +134,10 @@ const COMMANDS: Readonly> = { kind: "envelope", argv: ["env", "protect", "-p", "DIR", "--json"], }, + "env migrate": { + kind: "envelope", + argv: ["env", "migrate", "-p", "DIR", "--json"], + }, "env apply": { kind: "line", argv: ["env", "apply", "--service", "web", "-p", "DIR"], diff --git a/tests/models/tla/env-migration/COVERAGE.md b/tests/models/tla/env-migration/COVERAGE.md new file mode 100644 index 000000000..cc85ceb6d --- /dev/null +++ b/tests/models/tla/env-migration/COVERAGE.md @@ -0,0 +1,29 @@ +# Env migration journal + +`Migration.tla` bounds publication to two files. Publish and receipt are separate +actions: an invocation can crash between them, then resume from retained bytes. +Completion, successful project startup and a later explicit cleanup request are +distinct. Snapshot damage while legacy files remain blocks cleanup. Live targets +cannot be removed. The unsafe control restores same-run deletion and must fail +`NoSameRunCleanup` at `UnsafeRemoveDuringApply`. +The positive TLC exploration exhausts 35 distinct states (two initial states +cover ordinary and live targets). The negative witness has zero publications, +an intact snapshot, a retained applying journal, no later cleanup request and a +removed legacy target. Both live-target initial states admit that witness. + +| Action / invariant | Source and regression | +| --- | --- | +| Prepare / NoLostOriginal | `prepareConsistent` and `prepareJournal` bind immutable payloads and source checksums to a checked private snapshot before project writes. Preview/apply parity and damaged-blob refusal use real files. | +| Publish, Record, Crash, Resume | `applyEnvMigration` publishes before recording provider/journal receipts; matching intended bytes are adopted on retry. `env-migration-journal.test.ts` SIGKILLs an owned child at every journal/publication boundary. | +| Finish / CompleteHasAllSteps | Resolution succeeds before the complete journal is published; completed retry is a no-op. | +| SuccessfulRun, RequestCleanup, RemoveLegacy / NoSameRunCleanup | `recordEnvMigrationProjectRun` and `cleanupEnvMigration` require a later request, unchanged successful-run layer revision, intact snapshot and fresh delivery. Migration, including implicit routes, retains originals. | +| LiveInputsRetained | Existing #769 identity guard remains at candidate selection and immediately before deletion. The live-key regression uses real files. | +| Rollback | Snapshot adapter guards expected current bytes, restores legacy sources before reversing publications and preserves overwritten bytes. Round-trip and later-edit refusal use real files. | + +The model abstracts each per-file rename and each journal write as atomic; it +does not prove filesystem fsync durability, descriptor/path identity, lock +reclamation, cryptography, real snapshot-provider behavior, partial cleanup or +rollback, unsynchronized external writers, process signal delivery, a successful +runtime's actual environment or any macOS behavior. No fairness or eventual +recovery is asserted. Arbitrarily many crashes stutter over a finite persistent +state; the real process test covers only its enumerated boundaries. diff --git a/tests/models/tla/env-migration/Migration.tla b/tests/models/tla/env-migration/Migration.tla new file mode 100644 index 000000000..b14177e0b --- /dev/null +++ b/tests/models/tla/env-migration/Migration.tla @@ -0,0 +1,61 @@ +----------------------------- MODULE Migration ----------------------------- +EXTENDS Naturals, TLC +CONSTANTS Steps, UnsafeCleanup +VARIABLES phase, published, recorded, snapshot, legacy, complete, runProof, + separate, live +vars == <> + +Init == /\ phase = "preview" /\ published = 0 /\ recorded = 0 + /\ snapshot = FALSE /\ legacy = TRUE /\ complete = FALSE + /\ runProof = FALSE /\ separate = FALSE /\ live \in BOOLEAN + +Prepare == /\ phase \in {"preview", "rolled_back"} + /\ phase' = "apply" /\ snapshot' = TRUE + /\ UNCHANGED <> +Publish == /\ phase = "apply" /\ published = recorded /\ published < Steps + /\ published' = published + 1 + /\ UNCHANGED <> +Record == /\ phase = "apply" /\ recorded < published + /\ recorded' = published + /\ UNCHANGED <> +Crash == /\ phase = "apply" /\ phase' = "idle" + /\ UNCHANGED <> +Resume == /\ phase = "idle" /\ ~complete /\ snapshot /\ phase' = "apply" + /\ UNCHANGED <> +Finish == /\ phase = "apply" /\ recorded = Steps + /\ complete' = TRUE /\ phase' = "idle" + /\ UNCHANGED <> +SuccessfulRun == /\ phase = "idle" /\ complete /\ ~runProof + /\ runProof' = TRUE + /\ UNCHANGED <> +DamageBackup == /\ phase = "idle" /\ complete /\ legacy /\ snapshot + /\ snapshot' = FALSE + /\ UNCHANGED <> +RequestCleanup == /\ phase = "idle" /\ complete /\ runProof /\ snapshot /\ ~live + /\ phase' = "cleanup" /\ separate' = TRUE + /\ UNCHANGED <> +RemoveLegacy == /\ phase = "cleanup" /\ snapshot /\ runProof /\ ~live + /\ legacy' = FALSE /\ phase' = "idle" + /\ UNCHANGED <> +UnsafeRemoveDuringApply == /\ UnsafeCleanup /\ phase = "apply" /\ legacy /\ published = 0 + /\ legacy' = FALSE + /\ UNCHANGED <> +Rollback == /\ phase = "idle" /\ snapshot + /\ phase' = "rolled_back" /\ published' = 0 /\ recorded' = 0 + /\ legacy' = TRUE /\ complete' = FALSE /\ runProof' = FALSE + /\ separate' = FALSE /\ UNCHANGED <> + +Next == Prepare \/ Publish \/ Record \/ Crash \/ Resume \/ Finish \/ SuccessfulRun + \/ DamageBackup \/ RequestCleanup \/ RemoveLegacy \/ UnsafeRemoveDuringApply \/ Rollback +Spec == Init /\ [][Next]_vars +TypeOK == /\ phase \in {"preview", "apply", "idle", "cleanup", "rolled_back"} + /\ published \in 0..Steps /\ recorded \in 0..Steps + /\ snapshot \in BOOLEAN /\ legacy \in BOOLEAN /\ complete \in BOOLEAN + /\ runProof \in BOOLEAN /\ separate \in BOOLEAN /\ live \in BOOLEAN +NoSameRunCleanup == ~legacy => separate +NoLostOriginal == legacy \/ snapshot +LiveInputsRetained == live => legacy +ReceiptsFollowPublication == recorded <= published +CompleteHasAllSteps == complete => recorded = Steps +============================================================================= diff --git a/tests/models/tla/env-migration/negative.cfg b/tests/models/tla/env-migration/negative.cfg new file mode 100644 index 000000000..2869ccc5c --- /dev/null +++ b/tests/models/tla/env-migration/negative.cfg @@ -0,0 +1,5 @@ +CONSTANTS Steps = 2 + UnsafeCleanup = TRUE +SPECIFICATION Spec +INVARIANTS TypeOK NoSameRunCleanup NoLostOriginal LiveInputsRetained ReceiptsFollowPublication CompleteHasAllSteps +CHECK_DEADLOCK FALSE diff --git a/tests/models/tla/env-migration/positive.cfg b/tests/models/tla/env-migration/positive.cfg new file mode 100644 index 000000000..ee5550271 --- /dev/null +++ b/tests/models/tla/env-migration/positive.cfg @@ -0,0 +1,5 @@ +CONSTANTS Steps = 2 + UnsafeCleanup = FALSE +SPECIFICATION Spec +INVARIANTS TypeOK NoSameRunCleanup NoLostOriginal LiveInputsRetained ReceiptsFollowPublication CompleteHasAllSteps +CHECK_DEADLOCK FALSE From aedf65acff98ced73bbfff8614622fd1fe6e45d6 Mon Sep 17 00:00:00 2001 From: Kit <344842+roodboi@users.noreply.github.com> Date: Fri, 13 Nov 2026 22:55:30 +0000 Subject: [PATCH 03/32] fix(env): close recovery admission and cleanup receipt gaps --- docs/env.md | 19 +- docs/reference/env-migration.md | 36 +++- src/commands/env-migrate.ts | 9 +- src/lib/env-migration-payloads.ts | 2 + src/lib/env-migration-snapshot.ts | 17 +- src/lib/env-migration-state.ts | 103 ++++++++++ src/lib/env-migration.ts | 244 +++++++++++++++++------ src/lib/project-env-config.ts | 147 +++++++++++--- tests/doctor-fix-noninteractive.test.ts | 50 ++--- tests/env-migration-journal.test.ts | 163 +++++++++++++++- tests/env-migration-preview-cli.test.ts | 27 ++- tests/env-migration-provider.test.ts | 245 ++++++++++++++++++++++++ 12 files changed, 922 insertions(+), 140 deletions(-) create mode 100644 src/lib/env-migration-state.ts create mode 100644 tests/env-migration-provider.test.ts diff --git a/docs/env.md b/docs/env.md index 3a0c08e60..38f853f30 100644 --- a/docs/env.md +++ b/docs/env.md @@ -1637,7 +1637,7 @@ lists the store's entries read-only before it writes anything, without creating An overlay that exists only in the store, such as a value from `hack env set KEY --secret --env staging` with no `.hack/.env.staging`, is migrated like an overlay file. A stored overlay whose name can't be an env layer name refuses -(`E_CONFIG_INVALID`). Doctor offers a configured store inside the project for cleanup only +(`E_CONFIG_INVALID`). The separate cleanup preview lists a configured store inside the project only when the env layers now deliver every entry it holds for this project and it holds none for another; otherwise it keeps the store and names the entries that would be lost, never their values. A store it can't list (its key isn't available, it can't be read, or the backend is @@ -1645,15 +1645,18 @@ the keychain) is kept too. A store key file inside the project is offered only w that needs it stays behind. Project env keys (including the shared Git key), current YAML layers and project config files are never legacy cleanup targets, even when an old backend configuration names the same file. Cleanup refuses a request that -includes one, because migrated values may still depend on it. While a store exists, the migration keeps its pointer in +includes one, because migrated values may still depend on it. Migration keeps every +legacy file and its store pointer in `hack.config.json` (`controlPlane.secrets.encryptedFile` and `backend: encrypted_file`) on every route, including implicit ones such as `hack env add`, so entries it still holds stay -readable through the secret store without hand edits; Hack reports the kept pointer with the -key names still in the store, never values. The pointer is dropped only after doctor's -cleanup has removed the store. If a later removal fails, doctor drops the pointer -for that confirmed store removal before reporting the cleanup error. If the config -cannot be updated, doctor warns about the pointer and still reports the original -cleanup failure and confirmed removals. If the store remains, its pointer stays +readable through the secret store without hand edits. After a successful full-project +`hack up`, preview cleanup with `hack env migrate --cleanup --json` and request it +with `--cleanup --apply`, adding `--accept-tracked-env-changes` for tracked edits. +The pointer is dropped only after that explicit cleanup removes the store. If a +later removal or receipt fails, cleanup still attempts pointer repair and reports +every confirmed removal. If the config cannot be updated, its error also warns +about the pointer. Restore the snapshot before retrying an incomplete cleanup. +If the store remains, its pointer stays in place. A `cloud` backend reads the same pointer, so it always keeps it, and its store and key file are never offered for cleanup. The keychain backend can't list its entries, so with it an overlay diff --git a/docs/reference/env-migration.md b/docs/reference/env-migration.md index e0ad3212e..6c3e161a2 100644 --- a/docs/reference/env-migration.md +++ b/docs/reference/env-migration.md @@ -41,15 +41,21 @@ The preview's `data` has these stable fields: | `dryRun` | Always `true` for a preview. | | `files` | Ordered array of `{ path, action, keys, tracked }`. | | `journalPath` | Absolute path where apply keeps its private journal; preview does not create it. | +| `manifestPath` | Absolute rollback manifest path once a journal exists, otherwise `null`. Available after interrupted apply even if that apply printed no result. | Each file `action` is `write` (absent destination), `change` (existing destination) or `removable` (a legacy copy eligible for the separate cleanup checks). `keys` contains names only; config, key and ignore files have an empty array. `tracked` is conservative: unknown Git status in a checkout counts as tracked. A cleanup -preview includes only `removable` entries. These are candidates, not deletion +preview includes `removable` entries and any `change` to repair a store pointer. +These are candidates, not deletion authority: apply checks the successful project run, original snapshot and current delivery again. Finished publications are omitted on resume, and a completed migration previews no further write or change. Confirmed removals are omitted. +When needed, the plan includes `.hack-staging/.gitignore` beside a destination. +This self-ignoring private staging guard is published before any value file. +Rollback retains these guards so interrupted or concurrent staging remains +invisible to ordinary Git additions; they hold only the fixed `*` ignore rule. For example, an entry can be: @@ -107,7 +113,10 @@ Migration never includes cleanup. A later explicit cleanup requires: Legacy stores leave before their keys. Confirmed store removal also repairs its config pointer. A partial cleanup preserves confirmed removals in the journal and -reports the error. If cleanup is interrupted, use its rollback manifest before +reports every confirmed removal even if pointer repair or snapshot receipt +recording fails. Pointer repair is attempted before recording removal receipts; +if repair cannot be confirmed, the error warns about a possible stale pointer. +If cleanup is interrupted, use its rollback manifest before retrying; it does not infer fresh deletion authority from an old receipt. Rollback restores originals before reversing publications, so legacy values are @@ -120,3 +129,26 @@ originals and keeps replaced bytes privately in the snapshot instead; it does no select or merge values. Rollback can resume an incomplete restoration. Keep the manifest, blobs and journal together until the upgrade is accepted. Neither file-level atomic writes nor cooperative locks protect against an unrelated process replacing files outside Hack's locks. + +### Recovering retained edits + +Rollback JSON includes `preserved: [{ path, backupPath }]` for files the snapshot +provider kept while restoring. `path` identifies the original file and +`backupPath` identifies its retained bytes. The lower-level per-file `results` +also retain the provider's `keptAs` and status, including incomplete outcomes. +These fields contain paths only. The files themselves can hold values and keys. + +Use the named backup path to recover a file into a **new private path outside Git +working trees**. For example, after choosing an existing private recovery +directory, this copies bytes without printing them or overwriting an existing +destination: + +```sh +umask 077 +set -C +cat /private/snapshot/blobs/RECORDED_SHA256 > /private/recovery/recovered.yaml +``` + +Replace the source with the returned `backupPath`. Inspect or reconcile the +recovered file privately; do not paste its contents into reports. Keep the +snapshot until every retained edit has been recovered or deliberately retired. diff --git a/src/commands/env-migrate.ts b/src/commands/env-migrate.ts index 57827d14d..bf0bb0d8f 100644 --- a/src/commands/env-migrate.ts +++ b/src/commands/env-migrate.ts @@ -12,6 +12,7 @@ import { emitCliResult, okResult } from "../lib/cli-result.ts"; import { applyEnvMigration, cleanupEnvMigration, + previewEnvCleanup, previewEnvMigration, rollbackEnvMigration, } from "../lib/env-migration.ts"; @@ -142,14 +143,14 @@ export async function handleEnvMigrate({ snapshots: await envMigrationSnapshots(), }); } else { - const preview = await previewEnvMigration(project); + const preview = await (opts.cleanup + ? previewEnvCleanup + : previewEnvMigration)(project); data = { ...preview, action, dryRun: true, - files: opts.cleanup - ? preview.files.filter((entry) => entry.action === "removable") - : preview.files, + files: preview.files, }; } if (opts.json) { diff --git a/src/lib/env-migration-payloads.ts b/src/lib/env-migration-payloads.ts index a113cbbba..668b3a40e 100644 --- a/src/lib/env-migration-payloads.ts +++ b/src/lib/env-migration-payloads.ts @@ -26,6 +26,7 @@ export async function stageMigrationPayloads(input: { })); await replacePrivateFileAtomic({ path: join(directory, "receipt.json"), + staging: "sibling", directories: [directory], text: JSON.stringify({ version: 1, @@ -41,6 +42,7 @@ export async function stageMigrationPayloads(input: { const path = join(directory, `${index}.payload`); await replacePrivateFileAtomic({ path, + staging: "sibling", directories: [directory], text: file.text, mode: 0o600, diff --git a/src/lib/env-migration-snapshot.ts b/src/lib/env-migration-snapshot.ts index ebe63af23..cdb790655 100644 --- a/src/lib/env-migration-snapshot.ts +++ b/src/lib/env-migration-snapshot.ts @@ -38,6 +38,10 @@ export type EnvMigrationSnapshots = { }) => Promise<{ readonly status: "restored" | "incomplete"; readonly results: readonly unknown[]; + readonly preserved?: readonly { + readonly path: string; + readonly backupPath: string; + }[]; }>; }; @@ -233,7 +237,18 @@ export function bindEnvMigrationSnapshots( break; } } - return { status: incomplete ? "incomplete" : "restored", results }; + const preserved = results.flatMap((result) => + isRecord(result) && + typeof result.path === "string" && + typeof result.keptAs === "string" + ? [{ path: result.path, backupPath: result.keptAs }] + : [] + ); + return { + status: incomplete ? "incomplete" : "restored", + results, + preserved, + }; }, }; } diff --git a/src/lib/env-migration-state.ts b/src/lib/env-migration-state.ts new file mode 100644 index 000000000..a62b65ab0 --- /dev/null +++ b/src/lib/env-migration-state.ts @@ -0,0 +1,103 @@ +import { AsyncLocalStorage } from "node:async_hooks"; +import { createHash } from "node:crypto"; +import { lstat, realpath } from "node:fs/promises"; +import { join, resolve } from "node:path"; +import { ClosedCliError } from "./cli-result.ts"; +import { resolveGlobalHackDir } from "./config-paths.ts"; +import { migrationRefusal, readMigrationFile } from "./env-migration-files.ts"; +import { isRecord } from "./guards.ts"; + +const verification = new AsyncLocalStorage(); + +/** Read-only path calculation shared by the journal owner and every env reader. */ +export function migrationJournalPath(projectRoot: string): string { + return join( + resolveGlobalHackDir(), + "env-migrations", + createHash("sha256").update(projectRoot).digest("hex"), + "journal.json" + ); +} + +export class EnvMigrationPendingError extends ClosedCliError { + constructor(state: "applying" | "rolling_back") { + super( + state === "applying" + ? "An env migration is unfinished. Resume with hack env migrate --apply (add --accept-tracked-env-changes for its tracked edits), or use its rollback manifest. No project command was run." + : "An env rollback is unfinished. Resume hack env migrate --rollback-manifest with the same manifest before running project commands." + ); + } +} + +async function canonicalRoot(path: string): Promise { + try { + return await realpath(path); + } catch (error) { + if (isRecord(error) && error.code === "ENOENT") { + return resolve(path); + } + throw migrationRefusal(); + } +} + +/** Central admission: native, Compose, daemon, metadata and value readers all use it. */ +export async function assertEnvMigrationReadable(input: { + readonly projectRoot: string; +}): Promise { + const projectRoot = await canonicalRoot(input.projectRoot); + if (verification.getStore() === projectRoot) { + return; + } + const path = migrationJournalPath(projectRoot); + const file = await readMigrationFile(path); + if (file.text === null) { + return; + } + const info = await lstat(path); + if ( + info.uid !== process.getuid?.() || + info.nlink !== 1 || + (info.mode & 0o077) !== 0 + ) { + throw migrationRefusal(); + } + let raw: unknown; + try { + raw = JSON.parse(file.text); + } catch { + throw migrationRefusal(); + } + if ( + !( + isRecord(raw) && + raw.version === 1 && + raw.kind === "hack-env-migration" && + isRecord(raw.project) && + raw.project.projectRoot === projectRoot && + [ + "applying", + "complete", + "cleaning", + "cleaned", + "rolling_back", + "rolled_back", + ].includes(String(raw.state)) + ) + ) { + throw migrationRefusal(); + } + if (raw.state === "applying" || raw.state === "rolling_back") { + throw new EnvMigrationPendingError(raw.state); + } +} + +/** Only the journal owner may inspect its completed publications before committing completion. */ +export async function withEnvMigrationVerification(input: { + readonly projectRoot: string; + readonly run: () => Promise; +}): Promise { + return await verification.run( + await canonicalRoot(input.projectRoot), + input.run + ); +} diff --git a/src/lib/env-migration.ts b/src/lib/env-migration.ts index 7cd2869ed..d666ffceb 100644 --- a/src/lib/env-migration.ts +++ b/src/lib/env-migration.ts @@ -1,4 +1,3 @@ -import { createHash } from "node:crypto"; import { lstat, realpath } from "node:fs/promises"; import { basename, @@ -10,7 +9,6 @@ import { sep, } from "node:path"; import { ClosedCliError } from "./cli-result.ts"; -import { resolveGlobalHackDir } from "./config-paths.ts"; import { withEnvLayerLock } from "./env-layer-lock.ts"; import { migrationDigest as digest, @@ -27,11 +25,20 @@ import type { EnvMigrationSnapshots, } from "./env-migration-snapshot.ts"; import { envMigrationSnapshots } from "./env-migration-snapshot.ts"; +import { + assertEnvMigrationReadable, + migrationJournalPath, + withEnvMigrationVerification, +} from "./env-migration-state.ts"; import { isRecord } from "./guards.ts"; import { ensurePrivateDirectory, replacePrivateFileAtomic, } from "./private-file.ts"; +import { + PRIVATE_STAGING_DIRNAME, + preparePrivateStaging, +} from "./private-staging.ts"; import { acquireLegacyEnvCleanupContext, assertMachineLocalFileGitPrivate, @@ -72,6 +79,7 @@ export type EnvMigrationPreview = { readonly tracked: boolean; }[]; readonly journalPath: string; + readonly manifestPath: string | null; }; type Publication = EnvMigrationFileState & { @@ -107,6 +115,7 @@ type Journal = { /** Test-only scheduling seam; events contain no payloads and production does not consult env flags. */ export type EnvMigrationHooks = { + readonly beforeLock?: () => Promise; readonly boundary?: (event: { readonly phase: | "prepared" @@ -138,10 +147,7 @@ async function canonicalProject( } /** Location calculation is read-only, including when the data home has not been created. */ -export function envMigrationJournalPath(projectRoot: string): string { - const id = createHash("sha256").update(projectRoot).digest("hex"); - return join(resolveGlobalHackDir(), "env-migrations", id, "journal.json"); -} +export const envMigrationJournalPath = migrationJournalPath; /** Implicit routes must resume a journal even after its first YAML file exists. */ export async function hasPendingEnvMigration(input: { @@ -164,11 +170,7 @@ export async function assertEnvMigrationFinished(input: { readonly projectRoot: string; readonly projectDir: string; }): Promise { - if (await hasPendingEnvMigration(input)) { - throw new ClosedCliError( - "An env migration is unfinished. Resume with hack env migrate --apply (add --accept-tracked-env-changes for its tracked edits), or use its rollback manifest. No project command was run." - ); - } + await assertEnvMigrationReadable(input); } async function assertOutsideWorkingTrees( @@ -228,6 +230,7 @@ async function checkPrivateDirectory(path: string): Promise { async function writeMetadata(path: string, value: unknown): Promise { await replacePrivateFileAtomic({ path, + staging: "sibling", directories: [dirname(path)], text: `${JSON.stringify(value, null, 2)}\n`, mode: 0o600, @@ -265,9 +268,15 @@ function validPublication(entry: unknown, path: string): entry is Publication { dirname(dirname(entry.payload)) === dirname(path) && basename(dirname(entry.payload)).startsWith("payloads-") && PAYLOAD_NAME.test(basename(entry.payload)) && - ["ignore", "key", "local_layer", "layer", "config", "compose"].includes( - String(entry.kind) - ) && + [ + "staging_guard", + "ignore", + "key", + "local_layer", + "layer", + "config", + "compose", + ].includes(String(entry.kind)) && typeof entry.done === "boolean" && Array.isArray(entry.keys) && entry.keys.every((key) => typeof key === "string" && ENV_KEY.test(key)) @@ -380,6 +389,7 @@ async function previewPrepared( status: prepared.files.length ? "ready" : "complete", files, journalPath: path, + manifestPath: null, }; } @@ -458,6 +468,7 @@ export async function previewEnvMigration( status: previewStatus(journal), files, journalPath: path, + manifestPath: journal.manifestPath, }; } return await previewPrepared( @@ -474,6 +485,35 @@ function previewStatus(journal: Journal): EnvMigrationPreview["status"] { return journal.state === "applying" ? "ready" : "complete"; } +/** Cleanup's observation includes pointer repair as well as later removals. */ +export async function previewEnvCleanup( + input: EnvMigrationProject +): Promise { + const project = await canonicalProject(input); + const preview = await previewEnvMigration(project); + const journal = await readJournal(preview.journalPath, project); + const files = preview.files.filter((file) => file.action === "removable"); + if ( + journal?.state === "complete" && + journal.store && + journal.removable.includes(journal.store.path) + ) { + const repair = await prepareLegacyEnvCleanupConfig(project); + if (repair) { + files.push({ + path: repair.path, + action: "change", + keys: [], + tracked: await isGitTrackedOrUnknownProjectPath({ + ...project, + path: repair.path, + }), + }); + } + } + return { ...preview, files }; +} + async function assertConsent( project: EnvMigrationProject, paths: readonly string[], @@ -575,6 +615,26 @@ async function prepareJournal( return journal; } +async function publishStagingGuard( + entry: Publication, + text: string +): Promise { + if ( + text !== "*\n" || + basename(entry.path) !== ".gitignore" || + basename(dirname(entry.path)) !== PRIVATE_STAGING_DIRNAME + ) { + throw refuse(); + } + await preparePrivateStaging({ + target: join(dirname(dirname(entry.path)), "env-migration"), + refusal: refuse, + }); + if (!same((await readState(entry.path)).state, entry)) { + throw refuse(); + } +} + async function publishEntry( project: EnvMigrationProject, entry: Publication @@ -608,6 +668,10 @@ async function publishEntry( if (entry.kind === "layer" || entry.kind === "local_layer") { await assertProjectEnvLayerPaths(project); } + if (entry.kind === "staging_guard") { + await publishStagingGuard(entry, payload.text); + return; + } await replacePrivateFileAtomic({ path: entry.path, directories: [dirname(entry.path)], @@ -624,7 +688,9 @@ async function publishEntry( throw refuse(); } }; - if (entry.kind === "key") { + if (entry.kind === "staging_guard") { + await run(); + } else if (entry.kind === "key") { const shared = await resolveProjectEnvSharedKeyLocation(project); await withProjectsRegistryLock({ lockPath: `${shared?.path ?? entry.path}.lock`, @@ -652,6 +718,14 @@ function retainedPayloads(journal: Journal | null): readonly string[] { return files; } +function assertApplyJournalState(journal: Journal | null): void { + if (journal?.state === "rolling_back" || journal?.state === "cleaning") { + throw new ClosedCliError( + `Env migration needs recovery. Resume hack env migrate --rollback-manifest ${JSON.stringify(journal.manifestPath)} before applying again; no migration files were changed.` + ); + } +} + /** Uses recorded before/after images; an unrecorded successful rename is adopted on resume. */ export async function applyEnvMigration(input: { readonly project: EnvMigrationProject; @@ -664,27 +738,33 @@ export async function applyEnvMigration(input: { const path = envMigrationJournalPath(project.projectRoot); await assertOutsideWorkingTrees(path, project); const existing = await readJournal(path, project); - if (existing?.state === "rolling_back") { - throw new ClosedCliError( - `Env rollback is unfinished. Resume hack env migrate --rollback-manifest ${JSON.stringify(existing.manifestPath)}; no migration files were changed.` - ); - } + assertApplyJournalState(existing); + await input.hooks?.beforeLock?.(); if (existing && ["complete", "cleaned"].includes(existing.state)) { return await withProjectsRegistryLock({ lockPath: `${path}.lock`, label: "env migration", - run: async () => ({ - status: "already_complete" as const, - journalPath: path, - manifestPath: existing.manifestPath, - payloadCleanup: (await purgeMigrationPayloads({ + run: async () => { + const current = await readJournal(path, project); + assertApplyJournalState(current); + if (!(current && ["complete", "cleaned"].includes(current.state))) { + throw new ClosedCliError( + "Env migration state changed while waiting. Repeat the preview and apply command; no project files were changed." + ); + } + return { + status: "already_complete" as const, journalPath: path, - projectRoot: project.projectRoot, - retain: [], - })) - ? "complete" - : "kept", - }), + manifestPath: current.manifestPath, + payloadCleanup: (await purgeMigrationPayloads({ + journalPath: path, + projectRoot: project.projectRoot, + retain: [], + })) + ? "complete" + : "kept", + }; + }, }); } const preparation = @@ -717,6 +797,7 @@ export async function applyEnvMigration(input: { label: "env migration", run: async () => { const retained = await readJournal(path, project); + assertApplyJournalState(retained); const retain = retainedPayloads(retained); await purgeMigrationPayloads({ journalPath: path, @@ -771,11 +852,16 @@ export async function applyEnvMigration(input: { await input.hooks?.boundary?.({ phase: "recorded", index }); } await assertSources(journal); - for (const envName of [null, ...journal.overlayNames]) { - if (!(await resolveProjectEnvConfig({ ...project, envName }))) { - throw refuse(); - } - } + await withEnvMigrationVerification({ + projectRoot: project.projectRoot, + run: async () => { + for (const envName of [null, ...journal.overlayNames]) { + if (!(await resolveProjectEnvConfig({ ...project, envName }))) { + throw refuse(); + } + } + }, + }); await input.hooks?.boundary?.({ phase: "verified", index: -1 }); journal = { ...journal, state: "complete" }; await writeMetadata(path, journal); @@ -813,6 +899,32 @@ export async function cleanupEnvMigration(input: { }); } +function migrationCleanupFailure(input: { + readonly error: unknown; + readonly removed: readonly string[]; + readonly journal: Journal; +}): LegacyEnvCleanupError { + const { error, removed, journal } = input; + const failure = + error instanceof LegacyEnvCleanupError + ? error + : new LegacyEnvCleanupError({ + reason: "unverified", + removedPaths: removed, + }); + if ( + journal.cleanupConfig && + journal.store && + removed.includes(journal.store.path) && + !journal.cleanupConfig.done + ) { + failure.message += + " Could not reconcile the legacy secret store pointer; check hack.config.json before using the legacy store."; + } + failure.message += ` Cleanup or its receipts are incomplete. Restore originals with hack env migrate --rollback-manifest ${JSON.stringify(journal.manifestPath)} before retrying cleanup (values omitted).`; + return failure; +} + async function cleanupLocked( input: { readonly acceptTrackedChanges: boolean; @@ -918,32 +1030,40 @@ async function cleanupLocked( ...journal, removed: [...new Set([...journal.removed, ...removed])], }; - await writeMetadata(path, journal); + let pointerRepairFailed = false; if ( journal.cleanupConfig && journal.store && journal.removed.includes(journal.store.path) ) { - await publishEntry(project, journal.cleanupConfig); - await input.snapshots.recordAfter({ - manifestPath: journal.manifestPath, - files: [journal.cleanupConfig], - }); - journal = { - ...journal, - cleanupConfig: { ...journal.cleanupConfig, done: true }, - }; - await writeMetadata(path, journal); + try { + await publishEntry(project, journal.cleanupConfig); + journal = { + ...journal, + cleanupConfig: { ...journal.cleanupConfig, done: true }, + }; + } catch { + pointerRepairFailed = true; + } } + // Repair the pointer even if recording the removal receipt will fail. + // The cleaning journal already owns the repair's before/after images. + await writeMetadata(path, journal); await input.snapshots.recordAfter({ manifestPath: journal.manifestPath, - files: journal.removed.map((removed) => ({ - path: removed, - sha256: null, - mode: null, - size: 0, - })), + files: [ + ...journal.removed.map((removed) => ({ + path: removed, + sha256: null, + mode: null, + size: 0, + })), + ...(journal.cleanupConfig?.done ? [journal.cleanupConfig] : []), + ], }); + if (pointerRepairFailed) { + throw refuse(); + } }; let removed: readonly string[] = []; try { @@ -952,20 +1072,20 @@ async function cleanupLocked( paths: journal.removable, context, }); + await recordCleanup(removed); + await writeMetadata(path, { + ...journal, + state: "cleaned", + removed: [...new Set([...journal.removed, ...removed])], + }); } catch (error) { if (error instanceof LegacyEnvCleanupError) { removed = error.removedPaths; // Preserve the original failure and confirmed removals even if repair fails. await recordCleanup(removed).catch(() => undefined); } - throw error; + throw migrationCleanupFailure({ error, removed, journal }); } - await recordCleanup(removed); - await writeMetadata(path, { - ...journal, - state: "cleaned", - removed: [...new Set([...journal.removed, ...removed])], - }); await purgeMigrationPayloads({ journalPath: path, projectRoot: project.projectRoot, @@ -1048,7 +1168,9 @@ async function rollbackLocked( }); // Restore legacy sources first, then reverse publication: encrypted layers // leave resolution before a newly-created project key can be moved aside. - const retainedKeys = journal.writes.filter((write) => write.kind === "key"); + const retainedKeys = journal.writes.filter( + (write) => write.kind === "key" || write.kind === "staging_guard" + ); const retainedPaths = new Set(retainedKeys.map((write) => write.path)); if ( retainedKeys.some( diff --git a/src/lib/project-env-config.ts b/src/lib/project-env-config.ts index 3c8cabc7b..d10c2209b 100644 --- a/src/lib/project-env-config.ts +++ b/src/lib/project-env-config.ts @@ -18,6 +18,7 @@ import { basename, dirname, isAbsolute, + join, relative, resolve, sep, @@ -47,6 +48,10 @@ import { ClosedCliError, HackCliError } from "./cli-result.ts"; import { resolveGlobalConfigPath } from "./config-paths.ts"; import { parseDotEnv, serializeDotEnv } from "./env.ts"; import { withEnvLayerLock } from "./env-layer-lock.ts"; +import { + assertEnvMigrationReadable, + EnvMigrationPendingError, +} from "./env-migration-state.ts"; import { envParseDiagnostic } from "./env-parse-diagnostic.ts"; import { AtomicWriteConflictError, @@ -112,6 +117,7 @@ import { readReplaceablePrivateFile, replacePrivateFileAtomic, } from "./private-file.ts"; +import { PRIVATE_STAGING_DIRNAME } from "./private-staging.ts"; import { parseEnvConfigSelection, readProjectDefaultEnvConfig, @@ -1441,6 +1447,7 @@ async function readProjectEnvLayers(opts: { readonly envName?: string | null; readonly strictRead?: boolean; }) { + await assertEnvMigrationReadable(opts); const selection = await resolveProjectEnvSelection({ projectRoot: opts.projectRoot, projectDir: opts.projectDir, @@ -1481,6 +1488,7 @@ async function readProjectEnvLayers(opts: { const primaryDir = await resolvePrimaryLocalProjectDir(opts); const inheritedReads: ProjectEnvConfigReadResult[] = []; if (primaryDir !== null) { + await assertEnvMigrationReadable({ projectRoot: dirname(primaryDir) }); const primaryDefaultPath = await resolveProjectEnvEffectiveLocalConfigPath({ projectRoot: dirname(primaryDir), projectDir: primaryDir, @@ -1654,6 +1662,9 @@ export async function resolveProjectEnvMetadata( .metadata : null; } catch (error) { + if (error instanceof EnvMigrationPendingError) { + throw error; + } if ( opts.fileDiagnostics && (error instanceof ClosedCliError || error instanceof HackCliError) @@ -2050,6 +2061,7 @@ async function readNativeProjectEnvSelection( onAcquired?: NativeEnvAcquisitionObserver, inputOwner: ExplicitManagedEnvInputOwner = nativeManagedEnvInputOwner ) { + await assertEnvMigrationReadable(opts); checkNativeEnvCancellation(opts.signal); const overlayName = opts.overlay; const inheritLocal = opts.inheritLocal; @@ -2074,6 +2086,7 @@ async function readNativeProjectEnvSelection( signal: opts.signal, }); if (primaryRoot) { + await assertEnvMigrationReadable({ projectRoot: primaryRoot }); await inputOwner.assertRoot({ projectRoot: primaryRoot, signal: opts.signal, @@ -2181,6 +2194,9 @@ export async function resolveProjectEnvMetadataForNativeSelection( try { return (await readNativeProjectEnvSelection(opts)).metadata; } catch (error: unknown) { + if (error instanceof EnvMigrationPendingError) { + throw error; + } if ( error instanceof NativeConfigCompilerError && error.code === "E_COMPILER_CANCELLED" @@ -2208,6 +2224,9 @@ export async function resolveProjectEnvConfigForNativeSelection( signal: opts.signal, }); } catch (error: unknown) { + if (error instanceof EnvMigrationPendingError) { + throw error; + } if ( error instanceof NativeConfigCompilerError && error.code === "E_COMPILER_CANCELLED" @@ -6258,6 +6277,7 @@ export type PreparedLegacyEnvMigration = { readonly files: readonly { readonly path: string; readonly kind: + | "staging_guard" | "ignore" | "key" | "local_layer" @@ -6627,6 +6647,87 @@ async function prepareMigrationCompose( } } +async function prepareMigrationStagingGuards( + files: readonly PreparedLegacyEnvMigration["files"][number][], + extraSources: Set, + add: AddPreparedMigrationFile +): Promise { + const parents = [...new Set(files.map((file) => dirname(file.path)))]; + for (const parent of parents) { + const path = join(parent, PRIVATE_STAGING_DIRNAME, ".gitignore"); + extraSources.add(path); + const info = await lstat(dirname(path)).catch((error: unknown) => { + if (isRecord(error) && error.code === "ENOENT") { + return null; + } + throw error; + }); + if ( + info && + (!info.isDirectory() || + info.isSymbolicLink() || + info.uid !== process.getuid?.() || + (info.mode & 0o077) !== 0) + ) { + throw new ClosedCliError( + "Cannot prepare env migration: staging directory is unsafe or not private; values omitted." + ); + } + const previous = await readReplaceablePrivateFile({ + path, + directories: [], + refusal: () => + new ClosedCliError( + "Cannot prepare env migration staging guard; values omitted." + ), + }); + if (previous && previous.text !== "*\n") { + throw new ClosedCliError( + "Cannot prepare env migration: staging ignore file differs from the required guard; values omitted." + ); + } + if (!previous) { + await add({ path, kind: "staging_guard", keys: [], mode: 0o600 }, "*\n"); + } + } +} + +async function prepareMigrationInputs( + opts: PreparedMigrationProject, + input: { + readonly contract: Awaited>; + readonly layersExist: boolean; + readonly extraSources: Set; + readonly add: AddPreparedMigrationFile; + readonly addLayer: AddPreparedMigrationLayer; + } +) { + const { contract, layersExist, extraSources, add, addLayer } = input; + if ( + !contract.exists && + layersExist && + !(await canMigrateCheckoutLegacyDotenv(opts)) + ) { + return { + overlayNames: [], + store: undefined, + deliveredValues: new Map>(), + configCandidates: [], + }; + } + if (contract.exists) { + return await prepareContractMigration( + opts, + contract.contract, + layersExist, + extraSources, + add, + addLayer + ); + } + return await prepareUndeclaredMigration(opts, addLayer); +} + async function prepareLegacyProjectEnvMigrationInternal(opts: { readonly projectRoot: string; readonly projectDir: string; @@ -6703,35 +6804,14 @@ async function prepareLegacyProjectEnvMigrationInternal(opts: { ); }; const layersExist = await projectEnvConfigExistsForMutation(opts); - if ( - !contract.exists && - layersExist && - !(await canMigrateCheckoutLegacyDotenv(opts)) - ) { - return { - files: [], - sourcePaths: [], - removable: [], - overlayNames: [], - hasContract: false, - retainedEnvFiles: [], - blockedCleanupCandidates: [], - payload: () => { - throw new ClosedCliError("No migration publication is required."); - }, - }; - } const { overlayNames, store, deliveredValues, configCandidates } = - contract.exists - ? await prepareContractMigration( - opts, - contract.contract, - layersExist, - extraSources, - add, - addLayer - ) - : await prepareUndeclaredMigration(opts, addLayer); + await prepareMigrationInputs(opts, { + contract, + layersExist, + extraSources, + add, + addLayer, + }); if (files.some((file) => file.kind === "local_layer")) { const path = resolve(opts.projectDir, ".gitignore"); const previous = await readReplaceablePrivateFile( @@ -6788,6 +6868,16 @@ async function prepareLegacyProjectEnvMigrationInternal(opts: { } } } + const stagingTargets = [...files]; + if (store && removable.includes(store.path)) { + stagingTargets.push({ + path: resolve(opts.projectDir, PROJECT_CONFIG_FILENAME), + kind: "config", + keys: [], + mode: 0o644, + }); + } + await prepareMigrationStagingGuards(stagingTargets, extraSources, add); const sourcePaths = [ ...new Set([ contract.path, @@ -6804,6 +6894,7 @@ async function prepareLegacyProjectEnvMigrationInternal(opts: { const prepared: PreparedLegacyEnvMigration = { files: files.sort((a, b) => { const order = { + staging_guard: -1, ignore: 0, key: 1, local_layer: 2, diff --git a/tests/doctor-fix-noninteractive.test.ts b/tests/doctor-fix-noninteractive.test.ts index 518e0d3cd..b5541cc03 100644 --- a/tests/doctor-fix-noninteractive.test.ts +++ b/tests/doctor-fix-noninteractive.test.ts @@ -688,7 +688,12 @@ for (const docker of [ }); expect(await readFile(layerPath, "utf8")).not.toContain(value); const notes = noteCalls.join("\n"); - expect(notes).toContain(`Wrote ${localPath}, ${layerPath}`); + expect(notes).toContain( + "Legacy files were kept for a separate cleanup step." + ); + expect(await readFile(resolve(projectDir, ".env"), "utf8")).toBe( + `LEGACY_SETTING=${value}\n` + ); expect(notes).toContain("Plaintext env keys remain (values omitted)"); expect(notes).toContain("hack env protect [KEY...]"); expect(notes).toContain("LEGACY_SETTING"); @@ -937,11 +942,9 @@ test("doctor contract migration keeps base and untracked overlays with undeliver ] as const) { expect(await readFile(resolve(projectDir, name), "utf8")).toBe(files[name]); } - for (const key of ["UNDECLARED", "BACKUP_ONLY", "EXAMPLE_ONLY"]) { - expect(notes).toContain( - `keys not preserved in env layers: ${key} (values omitted)` - ); - } + expect(notes).toContain( + "Legacy files were kept for a separate cleanup step." + ); for (const text of Object.values(files)) { for (const line of text.trim().split("\n")) { const value = line.split("=")[1]; @@ -950,12 +953,12 @@ test("doctor contract migration keeps base and untracked overlays with undeliver } } } - expect( - await lstat(resolve(projectDir, ".env.qa")).catch(() => null) - ).toBeNull(); - expect( - await lstat(resolve(projectDir, ".env.same")).catch(() => null) - ).toBeNull(); + expect(await readFile(resolve(projectDir, ".env.qa"), "utf8")).toBe( + files[".env.qa"] + ); + expect(await readFile(resolve(projectDir, ".env.same"), "utf8")).toBe( + files[".env.same"] + ); expect( await readFile(resolve(projectDir, "hack.env.local.yaml"), "utf8") ).toContain("synthetic-delivered"); @@ -1195,9 +1198,8 @@ test("doctor keeps an encrypted store with an undeclared entry and its key file, expect(await readFile(keyPath, "utf8")).toBe(keyText); const notes = noteCalls.join("\n"); expect(notes).toContain( - `Kept ${storePath}: keys not preserved in env layers: UNDECL (values omitted).` + "Legacy files were kept for a separate cleanup step." ); - expect(notes).toContain(`Kept ${keyPath}: the kept encrypted secret store`); for (const value of [declared, undeclared, keyText.trim()]) { expect(notes).not.toContain(value); } @@ -1219,9 +1221,6 @@ test("doctor keeps an encrypted store with an undeclared entry and its key file, backend: "encrypted_file", encryptedFile: { path: "secrets.enc.json", keyPath: "store.key" }, }); - expect(notes).toContain( - `Kept the encrypted secret store pointer (controlPlane.secrets) in ${configPath} for ${storePath}, which stays: it still holds SEC, UNDECL (values omitted)` - ); const after = await resolveSecretStore({ projectName: "store-retention", projectDir, @@ -1264,7 +1263,7 @@ test("doctor keeps an encrypted store with an undeclared entry and its key file, }, 30_000); for (const trackedStore of [false, true]) { - test(`doctor removes a fully delivered encrypted store (${trackedStore ? "tracked store, untracked key" : "untracked"}), its key file and its pointer (HACK-1715)`, async () => { + test(`doctor retains a fully delivered encrypted store (${trackedStore ? "tracked store, untracked key" : "untracked"}), its key file and its pointer (HACK-1715)`, async () => { const projectRoot = await createProjectWithStaleDaemonPid(); const projectDir = resolve(projectRoot, ".hack"); const saved = { @@ -1334,17 +1333,22 @@ for (const trackedStore of [false, true]) { ]) ).toBe(0); - expect(existsSync(storePath)).toBe(false); - expect(existsSync(keyPath)).toBe(false); + expect(existsSync(storePath)).toBe(true); + expect(existsSync(keyPath)).toBe(true); const config = JSON.parse(await readFile(configPath, "utf8")) as { readonly controlPlane?: unknown; }; - expect(config.controlPlane).toBeUndefined(); + expect(config.controlPlane).toEqual({ + secrets: { + backend: "encrypted_file", + encryptedFile: { path: "secrets.enc.json", keyPath: "store.key" }, + }, + }); const notes = noteCalls.join("\n"); expect(notes).toContain( - "Removed the encrypted secret store pointer from hack.config.json" + "Legacy files were kept for a separate cleanup step." ); - expect(notes).not.toContain("Kept the encrypted secret store pointer"); + expect(notes).not.toContain("Removed the encrypted secret store pointer"); expect(notes).not.toContain(declared); } finally { restoreEnv("HACK_SECRETS_FILE_KEY", saved.fileKey); diff --git a/tests/env-migration-journal.test.ts b/tests/env-migration-journal.test.ts index 42337fefd..e0c0a95e0 100644 --- a/tests/env-migration-journal.test.ts +++ b/tests/env-migration-journal.test.ts @@ -16,6 +16,7 @@ import { cleanupEnvMigration, type EnvMigrationProject, envMigrationJournalPath, + previewEnvCleanup, previewEnvMigration, recordEnvMigrationProjectRun, rollbackEnvMigration, @@ -23,7 +24,11 @@ import { import { migrationDigest } from "../src/lib/env-migration-files.ts"; import { stageMigrationPayloads } from "../src/lib/env-migration-payloads.ts"; import { + LegacyEnvCleanupError, resolveProjectEnvConfig, + resolveProjectEnvConfigForNativeSelection, + resolveProjectEnvMetadata, + resolveProjectEnvMetadataForNativeSelection, resolveProjectEnvSharedKeyPath, setProjectEnvValue, } from "../src/lib/project-env-config.ts"; @@ -116,6 +121,19 @@ async function projectFiles(): Promise> { return files; } +function expectRestored( + actual: Map, + before: Map +): void { + for (const [path, text] of actual) { + if (!before.has(path) && path.endsWith("/.hack-staging/.gitignore")) { + expect(text).toBe(Buffer.from("*\n").toString("base64")); + actual.delete(path); + } + } + expect(actual).toEqual(before); +} + test("an already modern project without legacy inputs previews complete and applies without creating state", async () => { for (const name of ["hack.env.json", ".env", ".env.qa"]) { await rm(join(project.projectDir, name)); @@ -152,6 +170,7 @@ test("preview makes no writes and exactly names what apply publishes; completed const preview = await previewEnvMigration(project); expect(await projectFiles()).toEqual(before); expect(await Bun.file(preview.journalPath).exists()).toBe(false); + expect(preview.manifestPath).toBeNull(); expect(JSON.stringify(preview)).not.toContain(SENTINEL); const snapshots = fixtureEnvSnapshots(root); const result = await apply({ @@ -163,6 +182,9 @@ test("preview makes no writes and exactly names what apply publishes; completed cleanupEnvMigration({ project, snapshots, acceptTrackedChanges: true }) ).rejects.toThrow("hack up"); expect(result.status).toBe("applied"); + expect((await previewEnvMigration(project)).manifestPath).toBe( + result.manifestPath + ); const after = await projectFiles(); expect( [...after] @@ -280,7 +302,7 @@ test("rollback restores original bytes and refuses a later user edit", async () snapshots, manifestPath: result.manifestPath, }); - expect(await projectFiles()).toEqual(before); + expectRestored(await projectFiles(), before); }); test("a partial publication refuses implicit commands before they can use incomplete layers", async () => { @@ -298,6 +320,25 @@ test("a partial publication refuses implicit commands before they can use incomp }, }) ).rejects.toThrow("synthetic interruption"); + await expect( + resolveProjectEnvConfig({ ...project, envName: null }) + ).rejects.toThrow("unfinished"); + expect((await previewEnvMigration(project)).manifestPath).toBeString(); + await expect( + resolveProjectEnvMetadata({ ...project, envName: null }) + ).rejects.toThrow("unfinished"); + const native = { + projectRoot: project.projectRoot, + overlay: null, + inheritLocal: false, + declaredWorkloadNames: ["api"], + }; + await expect( + resolveProjectEnvConfigForNativeSelection(native) + ).rejects.toThrow("unfinished"); + await expect( + resolveProjectEnvMetadataForNativeSelection(native) + ).rejects.toThrow("unfinished"); for (const args of [ ["env", "list"], ["env", "get", "TOKEN"], @@ -433,6 +474,45 @@ test("incomplete rollback leaves its journal resumable and skips already restore }) ).status ).toBe("restored"); + expectRestored(await projectFiles(), before); +}); + +test("apply rechecks rollback state under the lock after observing a completed journal", async () => { + const snapshots = fixtureEnvSnapshots(root); + const result = await apply({ + project, + snapshots, + acceptTrackedChanges: true, + }); + const before = await projectFiles(); + const reached = Promise.withResolvers(); + const release = Promise.withResolvers(); + const retry = applyEnvMigration({ + project, + snapshots, + acceptTrackedChanges: true, + hooks: { + beforeLock: async () => { + reached.resolve(); + await release.promise; + }, + }, + }); + await reached.promise; + try { + const interrupted = { + ...snapshots, + rollback: async () => ({ status: "incomplete" as const, results: [] }), + }; + await rollbackEnvMigration({ + project, + snapshots: interrupted, + manifestPath: result.manifestPath, + }); + } finally { + release.resolve(); + } + await expect(retry).rejects.toThrow("needs recovery"); expect(await projectFiles()).toEqual(before); }); @@ -466,7 +546,7 @@ test("cleanup snapshots must still hold every original; cleanup followed by roll snapshots, manifestPath: result.manifestPath, }); - expect(await projectFiles()).toEqual(before); + expectRestored(await projectFiles(), before); }); async function addLegacySecret() { @@ -496,12 +576,74 @@ async function addLegacySecret() { await store.set({ key: "SEC", value: SENTINEL }); } +test("a failed cleanup receipt preserves confirmed removals and repairs the removed store pointer without exposing the cause", async () => { + await addLegacySecret(); + const snapshots = fixtureEnvSnapshots(root); + const result = await apply({ + project, + snapshots, + acceptTrackedChanges: true, + }); + await recordEnvMigrationProjectRun(project); + const preview = await previewEnvCleanup(project); + const expectedRemoved = preview.files + .filter((file) => file.action === "removable") + .map((file) => file.path) + .sort(); + let failure: unknown; + try { + await cleanupEnvMigration({ + project, + snapshots: { + ...snapshots, + recordAfter: () => Promise.reject(new Error(SENTINEL)), + }, + acceptTrackedChanges: true, + }); + } catch (error) { + failure = error; + } + expect(failure).toBeInstanceOf(LegacyEnvCleanupError); + if (!(failure instanceof LegacyEnvCleanupError)) { + throw new Error("Fixture expected a cleanup failure"); + } + expect([...failure.removedPaths].sort()).toEqual(expectedRemoved); + expect(failure.message).toContain("--rollback-manifest"); + expect(failure.message).toContain(result.manifestPath); + expect(failure.message).not.toContain(SENTINEL); + const journal = await readFile(result.journalPath, "utf8"); + expect(JSON.parse(journal).state).toBe("cleaning"); + expect(JSON.parse(journal).removed.sort()).toEqual(expectedRemoved); + expect(journal).not.toContain(SENTINEL); + for (const path of expectedRemoved) { + expect(await Bun.file(path).exists()).toBe(false); + expect(failure.message).toContain(JSON.stringify(path)); + } + expect( + await readFile(join(project.projectDir, "hack.config.json"), "utf8") + ).not.toContain("encryptedFile"); + await expect( + cleanupEnvMigration({ project, snapshots, acceptTrackedChanges: true }) + ).rejects.toThrow("interrupted"); + await rollbackEnvMigration({ + project, + snapshots, + manifestPath: result.manifestPath, + }); + for (const path of expectedRemoved) { + expect(await Bun.file(path).exists()).toBe(true); + } + expect( + await readFile(join(project.projectDir, "hack.config.json"), "utf8") + ).toContain("encryptedFile"); +}); + test("preview does not provision a new project key; apply encrypts with a private key and rollback retains every legacy value", async () => { await addLegacySecret(); const before = await projectFiles(); const keyPath = join(project.projectRoot, ".hack.secret.key"); const preview = await previewEnvMigration(project); - expect(await projectFiles()).toEqual(before); + expectRestored(await projectFiles(), before); expect(preview.files).toContainEqual({ path: keyPath, action: "write", @@ -520,7 +662,18 @@ test("preview does not provision a new project key; apply encrypts with a privat .api?.SEC ).toBe(SENTINEL); await recordEnvMigrationProjectRun(project); + const cleanupBefore = await projectFiles(); + const cleanupPreview = await previewEnvCleanup(project); + expect(await projectFiles()).toEqual(cleanupBefore); await cleanupEnvMigration({ project, snapshots, acceptTrackedChanges: true }); + const cleanupAfter = await projectFiles(); + const cleanupEffects = [...cleanupBefore] + .filter(([path, text]) => cleanupAfter.get(path) !== text) + .map(([path]) => path) + .sort(); + expect(cleanupEffects).toEqual( + cleanupPreview.files.map((file) => file.path).sort() + ); expect( await readFile(join(project.projectDir, "hack.config.json"), "utf8") ).not.toContain("encryptedFile"); @@ -534,7 +687,7 @@ test("preview does not provision a new project key; apply encrypts with a privat const after = await projectFiles(); after.delete(keyPath); after.delete(join(project.projectRoot, ".gitignore")); - expect(after).toEqual(before); + expectRestored(after, before); }); test("encrypted scoped values and store-only overlays survive; cleanup never removes their live project key", async () => { @@ -705,7 +858,7 @@ test("SIGKILL at every journal/publication boundary resumes without losing an ov selectedProject.projectRoot, relative(project.projectRoot, path) ); - await mkdir(dirname(target), { recursive: true }); + await mkdir(dirname(target), { recursive: true, mode: 0o700 }); await writeFile(target, Buffer.from(encoded, "base64"), { mode: (await lstat(path)).mode & 0o777, }); diff --git a/tests/env-migration-preview-cli.test.ts b/tests/env-migration-preview-cli.test.ts index 8d964ebae..15f6dfc79 100644 --- a/tests/env-migration-preview-cli.test.ts +++ b/tests/env-migration-preview-cli.test.ts @@ -9,10 +9,23 @@ import { writeFile, } from "node:fs/promises"; import { tmpdir } from "node:os"; -import { join, resolve } from "node:path"; +import { isAbsolute, join, resolve } from "node:path"; import { fixtureChildren } from "./helpers/fixture-children.ts"; import { privateCliEnv, privateCliHome } from "./helpers/private-cli-env.ts"; +const binary = process.env.HACK_TEST_ENV_MIGRATION_BINARY; +function cliPrefix(): readonly string[] { + if (binary && !isAbsolute(binary)) { + throw new Error("Compiled test executable must be absolute"); + } + return binary + ? [binary] + : [ + process.execPath, + "--no-env-file", + resolve(import.meta.dir, "..", "index.ts"), + ]; +} const sentinel = "synthetic-preview-cli-value"; const children = fixtureChildren("env-migration-preview"); let root: string | null = null; @@ -53,7 +66,7 @@ async function fixture() { return { root, project, projectDir, env, home: privateCliHome(root) }; } -test("source CLI preview JSON is versioned and value-free with no runtime, HOME state or project writes", async () => { +test("CLI preview JSON is versioned and value-free with no runtime, HOME state or project writes", async () => { const selected = await fixture(); const names = (await readdir(selected.projectDir)).sort(); const before = await Promise.all( @@ -62,9 +75,7 @@ test("source CLI preview JSON is versioned and value-free with no runtime, HOME for (const flags of [[], ["--cleanup"]]) { const result = await children.run({ cmd: [ - process.execPath, - "--no-env-file", - resolve(import.meta.dir, "..", "index.ts"), + ...cliPrefix(), "env", "migrate", "--path", @@ -85,6 +96,7 @@ test("source CLI preview JSON is versioned and value-free with no runtime, HOME status: "ready", dryRun: true, action: flags.length ? "cleanup" : "preview", + manifestPath: null, project: { projectRoot: selected.project, projectDir: selected.projectDir, @@ -102,6 +114,7 @@ test("source CLI preview JSON is versioned and value-free with no runtime, HOME "project", "files", "journalPath", + "manifestPath", ].sort() ); for (const file of output.data.files) { @@ -131,9 +144,7 @@ test("malformed legacy input cannot print its sentinel in JSON errors", async () ); const result = await children.run({ cmd: [ - process.execPath, - "--no-env-file", - resolve(import.meta.dir, "..", "index.ts"), + ...cliPrefix(), "env", "migrate", "--path", diff --git a/tests/env-migration-provider.test.ts b/tests/env-migration-provider.test.ts new file mode 100644 index 000000000..e396437f5 --- /dev/null +++ b/tests/env-migration-provider.test.ts @@ -0,0 +1,245 @@ +import { afterEach, beforeEach, expect, test } from "bun:test"; +import { + mkdir, + mkdtemp, + readFile, + realpath, + rm, + writeFile, +} from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { isAbsolute, join, resolve } from "node:path"; +import { isRecord } from "../src/lib/guards.ts"; +import { resolveProjectEnvConfig } from "../src/lib/project-env-config.ts"; +import { resolveSecretStore } from "../src/lib/secret-store.ts"; +import { restoreEnv } from "./helpers/env.ts"; +import { fixtureChildren } from "./helpers/fixture-children.ts"; +import { privateCliEnv } from "./helpers/private-cli-env.ts"; + +const keys = [ + "HOME", + "HACK_HOME", + "HACK_GLOBAL_CONFIG_PATH", + "HACK_ENV_SECRET_KEY", + "HACK_SECRETS_FILE_KEY", + "HACK_SECRETS_DISABLE_KEYCHAIN_FALLBACK", +] as const; +const sentinel = "synthetic-provider-cli-sentinel"; +const children = fixtureChildren("env-migration-provider"); +const compiled = process.env.HACK_TEST_ENV_MIGRATION_BINARY; +let root = ""; +let saved: (string | undefined)[] = []; + +beforeEach(async () => { + saved = keys.map((key) => process.env[key]); + root = await realpath(await mkdtemp(join(tmpdir(), "env-provider-"))); + for (const key of keys) { + Reflect.deleteProperty(process.env, key); + } + process.env.HOME = join(root, "home"); + process.env.HACK_HOME = join(root, "state"); + process.env.HACK_SECRETS_DISABLE_KEYCHAIN_FALLBACK = "1"; + await mkdir(process.env.HOME, { mode: 0o700 }); +}); +afterEach(async () => { + await children.stopAll(); + for (const [index, key] of keys.entries()) { + restoreEnv(key, saved[index]); + } + if (root) { + await rm(root, { recursive: true, force: true }); + } +}); + +async function fixture() { + const projectRoot = join(root, "project"); + const projectDir = join(projectRoot, ".hack"); + await mkdir(projectDir, { recursive: true }); + await writeFile( + join(projectDir, "hack.config.json"), + JSON.stringify({ + name: "provider", + controlPlane: { + secrets: { + backend: "encrypted_file", + encryptedFile: { path: "legacy-store.json", keyPath: "legacy.key" }, + }, + }, + }) + ); + await writeFile( + join(projectDir, "docker-compose.yml"), + "services:\n api:\n image: alpine:3.22\n" + ); + await writeFile( + join(projectDir, "hack.env.json"), + JSON.stringify({ + version: 1, + vars: [ + { key: "TOKEN", source: "plain_env" }, + { key: "SECRET", source: "keychain", services: ["api"] }, + ], + }) + ); + await writeFile(join(projectDir, ".env"), `TOKEN=${sentinel}\n`); + const project = { + projectRoot, + projectDir, + projectName: "provider", + serviceNames: ["api"], + }; + await (await resolveSecretStore(project)).set({ + key: "SECRET", + value: sentinel, + }); + return project; +} + +async function cli(projectRoot: string, args: readonly string[]) { + if (compiled && !isAbsolute(compiled)) { + throw new Error("Compiled test executable must be absolute"); + } + const command = compiled + ? [compiled] + : [ + process.execPath, + "--no-env-file", + resolve(import.meta.dir, "..", "index.ts"), + ]; + const result = await children.run({ + cmd: [ + ...command, + "env", + "migrate", + "--path", + projectRoot, + "--json", + ...args, + ], + cwd: projectRoot, + env: privateCliEnv({ + root, + home: join(root, "home"), + env: { + HACK_HOME: join(root, "state"), + HACK_NO_INTERACTIVE: "1", + HACK_SECRETS_DISABLE_KEYCHAIN_FALLBACK: "1", + PATH: "/nonexistent", + }, + }), + }); + expect(result.stdout + result.stderr).not.toContain(sentinel); + const raw: unknown = JSON.parse(result.stdout); + if (!(isRecord(raw) && typeof raw.ok === "boolean")) { + throw new Error("Expected CLI result envelope"); + } + if (raw.ok && !isRecord(raw.data)) { + throw new Error("Expected migration result data"); + } + return { + ...result, + envelope: { + ok: raw.ok, + data: isRecord(raw.data) ? raw.data : {}, + error: raw.error, + }, + }; +} + +function pathField(data: Record, name: string): string { + const path = data[name]; + if (typeof path !== "string") { + throw new Error("Expected migration metadata path"); + } + return path; +} + +test("production snapshot provider backs source CLI apply, no-op and guarded rollback", async () => { + const project = await fixture(); + const preview = await cli(project.projectRoot, []); + expect(preview.exitCode, preview.stderr).toBe(0); + const applied = await cli(project.projectRoot, [ + "--apply", + "--accept-tracked-env-changes", + ]); + expect(applied.exitCode, applied.stderr).toBe(0); + expect(applied.envelope).toMatchObject({ + ok: true, + data: { status: "applied" }, + }); + const data = applied.envelope.data; + const manifestPath = pathField(data, "manifestPath"); + for (const path of [manifestPath, pathField(data, "journalPath")]) { + expect(await readFile(path, "utf8")).not.toContain(sentinel); + } + expect( + (await resolveProjectEnvConfig({ ...project, envName: null }))?.serviceEnv + .api?.SECRET + ).toBe(sentinel); + expect(await readFile(join(project.projectDir, ".env"), "utf8")).toBe( + `TOKEN=${sentinel}\n` + ); + expect( + ( + await cli(project.projectRoot, [ + "--apply", + "--accept-tracked-env-changes", + ]) + ).envelope + ).toMatchObject({ ok: true, data: { status: "already_complete" } }); + const cleanup = await cli(project.projectRoot, [ + "--cleanup", + "--apply", + "--accept-tracked-env-changes", + ]); + expect(cleanup.exitCode).not.toBe(0); + expect(JSON.stringify(cleanup.envelope)).toContain("hack up"); + const rolled = await cli(project.projectRoot, [ + "--rollback-manifest", + manifestPath, + ]); + expect(rolled.exitCode, rolled.stderr).toBe(0); + expect(rolled.envelope).toMatchObject({ + ok: true, + data: { status: "restored" }, + }); + expect(await readFile(join(project.projectDir, ".env"), "utf8")).toBe( + `TOKEN=${sentinel}\n` + ); +}); + +test("production rollback maps preserved edits to retrievable private files without printing them", async () => { + const project = await fixture(); + const applied = await cli(project.projectRoot, [ + "--apply", + "--accept-tracked-env-changes", + ]); + expect(applied.exitCode, applied.stderr).toBe(0); + const layer = join(project.projectDir, "hack.env.local.yaml"); + const edited = `version: 1\n# ${sentinel}\n`; + await writeFile(layer, edited); + const manifest = pathField(applied.envelope.data, "manifestPath"); + expect( + (await cli(project.projectRoot, ["--rollback-manifest", manifest])).exitCode + ).not.toBe(0); + const recovered = await cli(project.projectRoot, [ + "--rollback-manifest", + manifest, + "--preserve-changes", + ]); + expect(recovered.exitCode, recovered.stderr).toBe(0); + const preserved = recovered.envelope.data.preserved; + if (!Array.isArray(preserved)) { + throw new Error("Expected preserved-file mapping"); + } + const file: unknown = preserved.find( + (entry: unknown) => isRecord(entry) && entry.path === layer + ); + if (!isRecord(file)) { + throw new Error("Expected retained edit path"); + } + expect(await readFile(pathField(file, "backupPath"), "utf8")).toBe(edited); + expect(await readFile(join(project.projectDir, ".env"), "utf8")).toBe( + `TOKEN=${sentinel}\n` + ); +}); From aa1c0a0a774f76dc9f69755b00a824c1eb15826b Mon Sep 17 00:00:00 2001 From: Kit <344842+roodboi@users.noreply.github.com> Date: Fri, 13 Nov 2026 23:02:37 +0000 Subject: [PATCH 04/32] docs(env): align cleanup and recovery guidance with journal flow --- docs/env.md | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/docs/env.md b/docs/env.md index d216ccc2a..3fd09452e 100644 --- a/docs/env.md +++ b/docs/env.md @@ -978,13 +978,13 @@ removing anything. A legacy store and its key use their separate owner locks, including the key lock shared by every store using that path. Changed sources or delivery refuse cleanup with reconciliation advice; migration writes already completed remain. Confirmed removals before a -failure are named. Implicit cleanup warns and lets the original command continue -even if a lock times out, a key is unavailable or a file is unreadable. -Once YAML layers exist, rerunning migration may not import changed legacy -values. Keep the copies and keys, reconcile chosen values with `env set KEY ---value-stdin` (use `--local` for machine-local values, add `--secret` for secrets -and preserve scope/overlay), check -delivery, then remove only copies confirmed obsolete. Editors and older writers +failure are named. Implicit commands retain legacy files; cleanup needs the +separate explicit invocation and its snapshot and successful-project-run checks. +A completed migration is a no-op on retry. Keep the copies and keys if legacy +values changed afterward. Restore the snapshot with `--preserve-changes` to keep +later edits privately, or reconcile chosen values with `env set KEY --value-stdin` +(use `--local` for machine-local values, add `--secret` for secrets and preserve +scope/overlay). Check delivery before previewing cleanup again. Editors and older writers that do not take Hack's locks can still race a filesystem operation. ### Concurrent writes to one env file @@ -1639,8 +1639,7 @@ An overlay that exists only in the store, such as a value from overlay file. A stored overlay whose name can't be an env layer name refuses (`E_CONFIG_INVALID`). The separate cleanup preview lists a configured store inside the project only when the env layers now deliver every entry it holds for this project and it holds none for -another; otherwise it keeps the store and names the entries that would be lost, never their -values. A store it can't list (its key isn't available, it can't be read, or the backend is +another; otherwise the store is kept. A store it can't list (its key isn't available, it can't be read, or the backend is the keychain) is kept too. A store key file inside the project is offered only when no store that needs it stays behind. Project env keys (including the shared Git key), current YAML layers and project config files are never legacy cleanup targets, even when From 98a251c2303e3b24cfad8ba1c1b1231867faebec Mon Sep 17 00:00:00 2001 From: Kit <344842+roodboi@users.noreply.github.com> Date: Fri, 13 Nov 2026 23:17:08 +0000 Subject: [PATCH 05/32] fix(env): keep pre-removal cleanup refusals retryable --- docs/reference/env-migration.md | 3 + docs/reference/env-migration.mmd | 1 + docs/reference/env-migration.svg | 4 +- docs/reference/v5-model-coverage.md | 2 +- scripts/lib/tla-runtime-models.ts | 2 +- src/lib/env-migration-state.ts | 20 +- src/lib/env-migration.ts | 224 +++++++++++++------ src/lib/project-env-config.ts | 5 +- tests/env-migration-journal.test.ts | 106 +++++++++ tests/models/tla/env-migration/COVERAGE.md | 3 +- tests/models/tla/env-migration/Migration.tla | 6 +- 11 files changed, 295 insertions(+), 81 deletions(-) diff --git a/docs/reference/env-migration.md b/docs/reference/env-migration.md index 6c3e161a2..4da3891bd 100644 --- a/docs/reference/env-migration.md +++ b/docs/reference/env-migration.md @@ -116,6 +116,9 @@ config pointer. A partial cleanup preserves confirmed removals in the journal an reports every confirmed removal even if pointer repair or snapshot receipt recording fails. Pointer repair is attempted before recording removal receipts; if repair cannot be confirmed, the error warns about a possible stale pointer. +When a delivery or live-input refusal happens before any unlink or pointer +publication, the migration stays complete and cleanup remains retryable after +correcting the inputs. Uncertain or partial removals require snapshot rollback. If cleanup is interrupted, use its rollback manifest before retrying; it does not infer fresh deletion authority from an old receipt. diff --git a/docs/reference/env-migration.mmd b/docs/reference/env-migration.mmd index 20373b65b..6ac017855 100644 --- a/docs/reference/env-migration.mmd +++ b/docs/reference/env-migration.mmd @@ -11,6 +11,7 @@ stateDiagram-v2 idle --> idle : SuccessfulRun / DamageBackup idle --> cleanup : RequestCleanup [proof] cleanup --> idle : RemoveLegacy [safe] + cleanup --> idle : RefuseCleanup [before unlink] idle --> rolled_back : Rollback [snapshot] rolled_back --> apply : Prepare note right of apply diff --git a/docs/reference/env-migration.svg b/docs/reference/env-migration.svg index fff80b11d..c6331fa07 100644 --- a/docs/reference/env-migration.svg +++ b/docs/reference/env-migration.svg @@ -1,4 +1,4 @@ -Migration · journal and separate cleanupPrepare [snapshot]Publish / RecordCrash / FinishResume [unfinished]SuccessfulRun /DamageBackupRequestCleanup [proof]RemoveLegacy [safe]Rollback [snapshot]Preparepreviewidlerolled_backapplycleanupINV legacy retainedReceipts follow publicationINV intact snapshotINV no live targetsFresh delivery required \ No newline at end of file +]]>Migration · journal and separate cleanupPrepare [snapshot]Publish / RecordCrash / FinishResume [unfinished]SuccessfulRun /DamageBackupRequestCleanup [proof]RemoveLegacy [safe]RefuseCleanup [before unlink]Rollback [snapshot]Preparepreviewidlerolled_backapplycleanupINV legacy retainedReceipts follow publicationINV intact snapshotINV no live targetsFresh delivery required \ No newline at end of file diff --git a/docs/reference/v5-model-coverage.md b/docs/reference/v5-model-coverage.md index 23b43b324..3794a214c 100644 --- a/docs/reference/v5-model-coverage.md +++ b/docs/reference/v5-model-coverage.md @@ -36,7 +36,7 @@ must be attached to the eventual exact integration revision. | Property and critical states | Model coverage | Implementation/regression boundary | Remaining qualification | | --- | --- | --- | --- | -| Env migration publication, crash/resume and later cleanup | [env-migration](../../tests/models/tla/env-migration/COVERAGE.md): 35 states, two publication steps, separate publication and receipt; unsafe same-run deletion fails `NoSameRunCleanup` | `env-migration.ts` binds immutable payloads and source checksums to a private snapshot, adopts interrupted renames, records completion, requires a successful project run and fresh snapshot/delivery/live-input proofs for explicit cleanup. Real-file tests check preview parity, sentinels, backup damage and rollback; owned child processes receive SIGKILL at every journal/publication boundary | The model assumes atomic per-file publication/journal writes and cooperative locks; it does not qualify fsync durability, real snapshot provider behavior, partial cleanup/rollback, unsynchronized writers, actual runtime environment or macOS. Implementation and dependency integration gates remain separate. | +| Env migration publication, crash/resume and later cleanup | [env-migration](../../tests/models/tla/env-migration/COVERAGE.md): 37 states, two publication steps, separate publication and receipt, safe pre-removal cleanup refusal; unsafe same-run deletion fails `NoSameRunCleanup` | `env-migration.ts` binds immutable payloads and source checksums to a private snapshot, adopts interrupted renames, records completion, requires a successful project run and fresh snapshot/delivery/live-input proofs for explicit cleanup. Real-file tests check preview parity, sentinels, backup damage, refused cleanup retry and rollback; owned child processes receive SIGKILL at every journal/publication boundary | The model assumes atomic per-file publication/journal writes and cooperative locks; it does not qualify fsync durability, real snapshot provider behavior, partial cleanup/rollback, unsynchronized writers, actual runtime environment or macOS. Implementation and dependency integration gates remain separate. | | Native status across startup lease stages; failed start with running services (HACK-2110) | [native-status-start](../../tests/models/tla/native-status-start/COVERAGE.md) | Observer-only presentation, stable journal fence, distinct authored Observe/Status replies; real-flock held-reader start test and expiry diagnostic; plain-down exact-proof recovery tests. Failure capture occurs at startup/effect verification and owned cleanup, never status; first failure wins, incomplete-rebind capture and owner-crash reconstruction are excluded | Seven acquisition stages, one reader, one start; 110 states. Negative control fails after services start. Second-mutator refusal and complete recovery proof are assumptions checked separately in code, not independently proved by this model. No new control-loop tick or lease; cleanup settle is bounded at ten seconds per exited listener. Lead Mac gate: ps every 0.5 s across ten starts, cross-project polling and failed-start plain down. | | Guest control ready before exposed bridge sockets bind; identity changes or the startup deadline expires | No new TLA+ model; deterministic clock schedules both readiness orders against real Unix socket nodes | `provider/bridge.rs` waits only in fresh `finish_boot`, before adoption, and keeps running audits immediate. `bridge_startup_tests.rs` covers two slots, late creation and mode tightening, permanent absence/unsafe modes, type/UID/link refusals, provider loss before/after observation and late success rejection. No wait mutates sockets or grants cleanup authority; the existing verified-provider stop handles failure. `lifecycle/boot_sequence_tests.rs` drives the real waiter through the production fresh-boot sequence and rejects audit/adoption on timeout, unsafe sockets or provider loss. `native-runtime-client.test.ts` checks the fixed runtime recover command and identical-root hint reach the frontend | Two sockets, finite scheduled events, injected provider verification and monotonic clock. Kernel process identity, provider scheduling, disk durability and successful recovery remain host boundaries. Require ten consecutive real-VM Mac Ctrl-C scenarios with pool stop/start; portable tests are not that evidence. | | CI cancellation must not lose merged-change coverage; only exact-head full evidence may qualify a prerelease | No maintained GitHub scheduler model. Finite fixture interleavings cover predecessor success/failure/cancellation, two competing full/scoped runs and two rerun attempts | `ci.yml` serializes next pushes in one non-cancelling group (GitHub cancels only the replaced pending run), cancels stale PR runs, and requires a successful before run for selection. `selectPrereleaseCiRun` reads current attempts; `requireBranchChecks` binds suite and job check IDs. `ci-trusted-policy`, `prerelease-plan` and `release-prepare-gate` regressions reject cancelled predecessors, green subsets, stale checks, skipped hosts and changed attempts | API snapshots and GitHub group/event semantics are assumptions; races after the last read remain possible. Publication rechecks the frozen protected head and full evidence but does not transactionally lock GitHub. Hosted dispatch/cancellation checks qualify wiring; no scheduler liveness or formal refinement claim. | diff --git a/scripts/lib/tla-runtime-models.ts b/scripts/lib/tla-runtime-models.ts index 44bf681c1..692f1b86f 100644 --- a/scripts/lib/tla-runtime-models.ts +++ b/scripts/lib/tla-runtime-models.ts @@ -144,7 +144,7 @@ const contracts: readonly ModelContract[] = [ { name: "env-migration", module: "Migration", - states: 35, + states: 37, invariant: "NoSameRunCleanup", action: "UnsafeRemoveDuringApply", fields: [ diff --git a/src/lib/env-migration-state.ts b/src/lib/env-migration-state.ts index a62b65ab0..85a9ee332 100644 --- a/src/lib/env-migration-state.ts +++ b/src/lib/env-migration-state.ts @@ -40,6 +40,12 @@ async function canonicalRoot(path: string): Promise { } } +function journalRefusal(path: string): ClosedCliError { + return new ClosedCliError( + `Env migration journal could not be verified at ${JSON.stringify(path)}. Keep this file and the upgrade snapshots for recovery; no env data was used (values omitted).` + ); +} + /** Central admission: native, Compose, daemon, metadata and value readers all use it. */ export async function assertEnvMigrationReadable(input: { readonly projectRoot: string; @@ -49,23 +55,27 @@ export async function assertEnvMigrationReadable(input: { return; } const path = migrationJournalPath(projectRoot); - const file = await readMigrationFile(path); + const file = await readMigrationFile(path).catch(() => { + throw journalRefusal(path); + }); if (file.text === null) { return; } - const info = await lstat(path); + const info = await lstat(path).catch(() => { + throw journalRefusal(path); + }); if ( info.uid !== process.getuid?.() || info.nlink !== 1 || (info.mode & 0o077) !== 0 ) { - throw migrationRefusal(); + throw journalRefusal(path); } let raw: unknown; try { raw = JSON.parse(file.text); } catch { - throw migrationRefusal(); + throw journalRefusal(path); } if ( !( @@ -84,7 +94,7 @@ export async function assertEnvMigrationReadable(input: { ].includes(String(raw.state)) ) ) { - throw migrationRefusal(); + throw journalRefusal(path); } if (raw.state === "applying" || raw.state === "rolling_back") { throw new EnvMigrationPendingError(raw.state); diff --git a/src/lib/env-migration.ts b/src/lib/env-migration.ts index d666ffceb..c21175bb8 100644 --- a/src/lib/env-migration.ts +++ b/src/lib/env-migration.ts @@ -903,15 +903,18 @@ function migrationCleanupFailure(input: { readonly error: unknown; readonly removed: readonly string[]; readonly journal: Journal; + readonly recoveryRequired: boolean; }): LegacyEnvCleanupError { const { error, removed, journal } = input; - const failure = - error instanceof LegacyEnvCleanupError - ? error - : new LegacyEnvCleanupError({ - reason: "unverified", - removedPaths: removed, - }); + const recovery = input.recoveryRequired + ? `Cleanup or its receipts are incomplete. Restore originals with hack env migrate --rollback-manifest ${JSON.stringify(journal.manifestPath)} before retrying cleanup (values omitted).` + : "No legacy files were removed. Review the refused inputs, run hack up again if env layers changed, then preview cleanup with hack env migrate --cleanup before retrying (values omitted)."; + const failure = new LegacyEnvCleanupError({ + reason: + error instanceof LegacyEnvCleanupError ? error.reason : "unverified", + removedPaths: removed, + recovery, + }); if ( journal.cleanupConfig && journal.store && @@ -921,18 +924,93 @@ function migrationCleanupFailure(input: { failure.message += " Could not reconcile the legacy secret store pointer; check hack.config.json before using the legacy store."; } - failure.message += ` Cleanup or its receipts are incomplete. Restore originals with hack env migrate --rollback-manifest ${JSON.stringify(journal.manifestPath)} before retrying cleanup (values omitted).`; return failure; } -async function cleanupLocked( - input: { - readonly acceptTrackedChanges: boolean; - readonly snapshots: EnvMigrationSnapshots; - }, +function cleanupRefusedBeforeRemoval( + error: unknown, + journal: Journal +): boolean { + return ( + error instanceof LegacyEnvCleanupError && + error.removedPaths.length === 0 && + (error.reason === "changed" || error.reason === "live_input") && + !journal.cleanupConfig?.done + ); +} + +/** A refusal before unlink can discard its staged repair and retry later. */ +async function resetRefusedCleanup(input: { + readonly path: string; + readonly selected: Journal; +}): Promise { + try { + await writeMetadata(input.path, input.selected); + } catch { + return false; + } + await purgeMigrationPayloads({ + journalPath: input.path, + projectRoot: input.selected.project.projectRoot, + retain: [], + }).catch(() => false); + return true; +} + +/** Keep confirmed effects reportable even if pointer or receipt writes fail. */ +async function recordMigrationCleanup(input: { + readonly path: string; + readonly project: EnvMigrationProject; + readonly snapshots: EnvMigrationSnapshots; + readonly journal: Journal; + readonly removed: readonly string[]; +}): Promise<{ readonly journal: Journal; readonly recorded: boolean }> { + let journal: Journal = { + ...input.journal, + removed: [...new Set([...input.journal.removed, ...input.removed])], + }; + let pointerRepairFailed = false; + if ( + journal.cleanupConfig && + journal.store && + journal.removed.includes(journal.store.path) + ) { + try { + await publishEntry(input.project, journal.cleanupConfig); + journal = { + ...journal, + cleanupConfig: { ...journal.cleanupConfig, done: true }, + }; + } catch { + pointerRepairFailed = true; + } + } + try { + // The cleaning journal already owns the repair's before/after images. + // Try pointer repair even if recording the removal receipt will fail. + await writeMetadata(input.path, journal); + await input.snapshots.recordAfter({ + manifestPath: journal.manifestPath, + files: [ + ...journal.removed.map((removed) => ({ + path: removed, + sha256: null, + mode: null, + size: 0, + })), + ...(journal.cleanupConfig?.done ? [journal.cleanupConfig] : []), + ], + }); + return { journal, recorded: !pointerRepairFailed }; + } catch { + return { journal, recorded: false }; + } +} + +async function readCleanupJournal( project: EnvMigrationProject, path: string -) { +): Promise { const selected = await readJournal(path, project); if ( !selected || @@ -944,32 +1022,47 @@ async function cleanupLocked( "Finish the env migration before requesting cleanup; no legacy files were removed." ); } - let journal: Journal = selected; - if (journal.state === "cleaned") { - return { - status: "already_cleaned" as const, - journalPath: path, - manifestPath: journal.manifestPath, - removed: journal.removed, - }; + if (selected.state === "cleaned") { + return selected; } - if (journal.state === "cleaning") { + if (selected.state === "cleaning") { throw new ClosedCliError( "Previous legacy cleanup was interrupted. Restore its snapshot with hack env migrate --rollback-manifest before retrying; remaining files were kept." ); } - if (!journal.validated) { + if (!selected.validated) { throw new ClosedCliError( "Start the complete project successfully with hack up before requesting legacy cleanup; no files were removed." ); } - for (const validated of journal.validated) { + for (const validated of selected.validated) { if (!same((await readState(validated.path)).state, validated)) { throw new ClosedCliError( "Env layers changed since the successful project run. Start the project again before cleanup; no files were removed." ); } } + return selected; +} + +async function cleanupLocked( + input: { + readonly acceptTrackedChanges: boolean; + readonly snapshots: EnvMigrationSnapshots; + }, + project: EnvMigrationProject, + path: string +) { + const selected = await readCleanupJournal(project, path); + let journal: Journal = selected; + if (journal.state === "cleaned") { + return { + status: "already_cleaned" as const, + journalPath: path, + manifestPath: journal.manifestPath, + removed: journal.removed, + }; + } const repair = journal.store && journal.removable.includes(journal.store.path) ? await prepareLegacyEnvCleanupConfig(project) @@ -1025,46 +1118,6 @@ async function cleanupLocked( await assertSources(journal, true); journal = { ...journal, state: "cleaning" }; await writeMetadata(path, journal); - const recordCleanup = async (removed: readonly string[]) => { - journal = { - ...journal, - removed: [...new Set([...journal.removed, ...removed])], - }; - let pointerRepairFailed = false; - if ( - journal.cleanupConfig && - journal.store && - journal.removed.includes(journal.store.path) - ) { - try { - await publishEntry(project, journal.cleanupConfig); - journal = { - ...journal, - cleanupConfig: { ...journal.cleanupConfig, done: true }, - }; - } catch { - pointerRepairFailed = true; - } - } - // Repair the pointer even if recording the removal receipt will fail. - // The cleaning journal already owns the repair's before/after images. - await writeMetadata(path, journal); - await input.snapshots.recordAfter({ - manifestPath: journal.manifestPath, - files: [ - ...journal.removed.map((removed) => ({ - path: removed, - sha256: null, - mode: null, - size: 0, - })), - ...(journal.cleanupConfig?.done ? [journal.cleanupConfig] : []), - ], - }); - if (pointerRepairFailed) { - throw refuse(); - } - }; let removed: readonly string[] = []; try { removed = await removeLegacyProjectEnvArtifacts({ @@ -1072,19 +1125,52 @@ async function cleanupLocked( paths: journal.removable, context, }); - await recordCleanup(removed); + const receipt = await recordMigrationCleanup({ + ...input, + project, + path, + journal, + removed, + }); + journal = receipt.journal; + if (!receipt.recorded) { + throw refuse(); + } await writeMetadata(path, { ...journal, state: "cleaned", removed: [...new Set([...journal.removed, ...removed])], }); } catch (error) { + let recoveryRequired = true; if (error instanceof LegacyEnvCleanupError) { removed = error.removedPaths; - // Preserve the original failure and confirmed removals even if repair fails. - await recordCleanup(removed).catch(() => undefined); } - throw migrationCleanupFailure({ error, removed, journal }); + if (cleanupRefusedBeforeRemoval(error, journal)) { + // These refusals precede unlink. Keep a safe retry possible without + // undoing completed migration; unknown removal outcomes still require recovery. + recoveryRequired = !(await resetRefusedCleanup({ path, selected })); + if (!recoveryRequired) { + journal = selected; + } + } else if (error instanceof LegacyEnvCleanupError) { + // Preserve confirmed removals even if repair or its receipt fails. + journal = ( + await recordMigrationCleanup({ + ...input, + project, + path, + journal, + removed, + }) + ).journal; + } + throw migrationCleanupFailure({ + error, + removed, + journal, + recoveryRequired, + }); } await purgeMigrationPayloads({ journalPath: path, diff --git a/src/lib/project-env-config.ts b/src/lib/project-env-config.ts index d10c2209b..1fccc7f85 100644 --- a/src/lib/project-env-config.ts +++ b/src/lib/project-env-config.ts @@ -10329,6 +10329,7 @@ export class LegacyEnvCleanupError extends ClosedCliError { constructor(opts: { readonly reason: "changed" | "live_input" | "unverified"; readonly removedPaths?: readonly string[]; + readonly recovery?: string; }) { const removedPaths = [...(opts.removedPaths ?? [])]; const reasons = { @@ -10343,7 +10344,9 @@ export class LegacyEnvCleanupError extends ClosedCliError { removedPaths.length > 0 ? `Confirmed removals: ${removedPaths.map((path) => JSON.stringify(path)).join(", ")}.` : "No removals were confirmed."; - super(`${reasons[opts.reason]} ${outcome} ${LEGACY_CLEANUP_RECOVERY}`); + super( + `${reasons[opts.reason]} ${outcome} ${opts.recovery ?? LEGACY_CLEANUP_RECOVERY}` + ); this.removedPaths = removedPaths; this.reason = opts.reason; } diff --git a/tests/env-migration-journal.test.ts b/tests/env-migration-journal.test.ts index e0c0a95e0..35182def7 100644 --- a/tests/env-migration-journal.test.ts +++ b/tests/env-migration-journal.test.ts @@ -426,6 +426,28 @@ test("explicit preserve-changes rollback recovers from an edited pending migrati ).toContain(edited); }); +test("a damaged journal names its recovery location without exposing its contents", async () => { + const result = await apply({ + project, + snapshots: fixtureEnvSnapshots(root), + acceptTrackedChanges: true, + }); + await writeFile(result.journalPath, SENTINEL); + let failure: unknown; + try { + await resolveProjectEnvConfig({ ...project, envName: null }); + } catch (error) { + failure = error; + } + expect(failure).toBeInstanceOf(Error); + if (!(failure instanceof Error)) { + throw new Error("Fixture expected damaged-journal refusal"); + } + expect(failure.message).toContain(result.journalPath); + expect(failure.message).toContain("Keep this file and the upgrade snapshots"); + expect(failure.message).not.toContain(SENTINEL); +}); + test("incomplete rollback leaves its journal resumable and skips already restored files", async () => { const before = await projectFiles(); const base = fixtureEnvSnapshots(root); @@ -576,6 +598,89 @@ async function addLegacySecret() { await store.set({ key: "SEC", value: SENTINEL }); } +test("fresh delivery refusal before unlink keeps migration complete and cleanup retryable", async () => { + await addLegacySecret(); + const snapshots = fixtureEnvSnapshots(root); + const result = await apply({ + project, + snapshots, + acceptTrackedChanges: true, + }); + await recordEnvMigrationProjectRun(project); + const preview = await previewEnvCleanup(project); + let failure: unknown; + try { + await cleanupEnvMigration({ + project, + acceptTrackedChanges: true, + snapshots: { + ...snapshots, + assertCoverage: async (input) => { + await snapshots.assertCoverage(input); + await setProjectEnvValue({ + ...project, + envName: null, + scope: "global", + key: "TOKEN", + value: QA, + secret: false, + local: true, + }); + }, + }, + }); + } catch (error) { + failure = error; + } + expect(failure).toBeInstanceOf(LegacyEnvCleanupError); + if (!(failure instanceof LegacyEnvCleanupError)) { + throw new Error("Fixture expected a delivery refusal"); + } + expect(failure.reason).toBe("changed"); + expect(failure.removedPaths).toEqual([]); + expect(failure.message).toContain("--cleanup"); + expect(failure.message).not.toContain("--rollback-manifest"); + expect(failure.message).not.toContain("remove only copies"); + expect(failure.message).not.toContain(SENTINEL); + expect(failure.message).not.toContain(QA); + const journal = JSON.parse(await readFile(result.journalPath, "utf8")); + expect(journal.state).toBe("complete"); + expect(journal.cleanupConfig).toBeUndefined(); + expect(journal.removed).toEqual([]); + expect( + (await readdir(dirname(result.journalPath))).filter((name) => + name.startsWith("payloads-") + ) + ).toEqual([]); + for (const file of preview.files.filter( + (entry) => entry.action === "removable" + )) { + expect(await Bun.file(file.path).exists()).toBe(true); + } + expect( + await readFile(join(project.projectDir, "hack.config.json"), "utf8") + ).toContain("encryptedFile"); + await setProjectEnvValue({ + ...project, + envName: null, + scope: "global", + key: "TOKEN", + value: SENTINEL, + secret: false, + local: true, + }); + await recordEnvMigrationProjectRun(project); + expect( + ( + await cleanupEnvMigration({ + project, + snapshots, + acceptTrackedChanges: true, + }) + ).status + ).toBe("cleaned"); +}); + test("a failed cleanup receipt preserves confirmed removals and repairs the removed store pointer without exposing the cause", async () => { await addLegacySecret(); const snapshots = fixtureEnvSnapshots(root); @@ -610,6 +715,7 @@ test("a failed cleanup receipt preserves confirmed removals and repairs the remo expect([...failure.removedPaths].sort()).toEqual(expectedRemoved); expect(failure.message).toContain("--rollback-manifest"); expect(failure.message).toContain(result.manifestPath); + expect(failure.message).not.toContain("remove only copies"); expect(failure.message).not.toContain(SENTINEL); const journal = await readFile(result.journalPath, "utf8"); expect(JSON.parse(journal).state).toBe("cleaning"); diff --git a/tests/models/tla/env-migration/COVERAGE.md b/tests/models/tla/env-migration/COVERAGE.md index cc85ceb6d..8620263f8 100644 --- a/tests/models/tla/env-migration/COVERAGE.md +++ b/tests/models/tla/env-migration/COVERAGE.md @@ -6,7 +6,7 @@ Completion, successful project startup and a later explicit cleanup request are distinct. Snapshot damage while legacy files remain blocks cleanup. Live targets cannot be removed. The unsafe control restores same-run deletion and must fail `NoSameRunCleanup` at `UnsafeRemoveDuringApply`. -The positive TLC exploration exhausts 35 distinct states (two initial states +The positive TLC exploration exhausts 37 distinct states (two initial states cover ordinary and live targets). The negative witness has zero publications, an intact snapshot, a retained applying journal, no later cleanup request and a removed legacy target. Both live-target initial states admit that witness. @@ -17,6 +17,7 @@ removed legacy target. Both live-target initial states admit that witness. | Publish, Record, Crash, Resume | `applyEnvMigration` publishes before recording provider/journal receipts; matching intended bytes are adopted on retry. `env-migration-journal.test.ts` SIGKILLs an owned child at every journal/publication boundary. | | Finish / CompleteHasAllSteps | Resolution succeeds before the complete journal is published; completed retry is a no-op. | | SuccessfulRun, RequestCleanup, RemoveLegacy / NoSameRunCleanup | `recordEnvMigrationProjectRun` and `cleanupEnvMigration` require a later request, unchanged successful-run layer revision, intact snapshot and fresh delivery. Migration, including implicit routes, retains originals. | +| RefuseCleanup | A known delivery or live-input refusal before unlink returns to completed migration; unknown or partial effects require rollback. A real-file regression changes delivery immediately before removal, checks that every legacy file stays and the journal remains complete, then corrects delivery and successfully retries. The model abstracts the pre-removal refusal only; it does not model receipt failures or partial cleanup. | | LiveInputsRetained | Existing #769 identity guard remains at candidate selection and immediately before deletion. The live-key regression uses real files. | | Rollback | Snapshot adapter guards expected current bytes, restores legacy sources before reversing publications and preserves overwritten bytes. Round-trip and later-edit refusal use real files. | diff --git a/tests/models/tla/env-migration/Migration.tla b/tests/models/tla/env-migration/Migration.tla index b14177e0b..d8b701349 100644 --- a/tests/models/tla/env-migration/Migration.tla +++ b/tests/models/tla/env-migration/Migration.tla @@ -38,6 +38,9 @@ RequestCleanup == /\ phase = "idle" /\ complete /\ runProof /\ snapshot /\ ~live RemoveLegacy == /\ phase = "cleanup" /\ snapshot /\ runProof /\ ~live /\ legacy' = FALSE /\ phase' = "idle" /\ UNCHANGED <> +RefuseCleanup == /\ phase = "cleanup" /\ legacy + /\ phase' = "idle" + /\ UNCHANGED <> UnsafeRemoveDuringApply == /\ UnsafeCleanup /\ phase = "apply" /\ legacy /\ published = 0 /\ legacy' = FALSE /\ UNCHANGED <> @@ -47,7 +50,8 @@ Rollback == /\ phase = "idle" /\ snapshot /\ separate' = FALSE /\ UNCHANGED <> Next == Prepare \/ Publish \/ Record \/ Crash \/ Resume \/ Finish \/ SuccessfulRun - \/ DamageBackup \/ RequestCleanup \/ RemoveLegacy \/ UnsafeRemoveDuringApply \/ Rollback + \/ DamageBackup \/ RequestCleanup \/ RemoveLegacy \/ RefuseCleanup + \/ UnsafeRemoveDuringApply \/ Rollback Spec == Init /\ [][Next]_vars TypeOK == /\ phase \in {"preview", "apply", "idle", "cleanup", "rolled_back"} /\ published \in 0..Steps /\ recorded \in 0..Steps From 7341b2bc8211fa98d1613ab4ccda8f2b52a1d70d Mon Sep 17 00:00:00 2001 From: Kit <344842+roodboi@users.noreply.github.com> Date: Sat, 14 Nov 2026 00:46:09 +0000 Subject: [PATCH 06/32] test(env): accept cold-home refusal during journal admission --- tests/native-compose-file-cold-home.test.ts | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/tests/native-compose-file-cold-home.test.ts b/tests/native-compose-file-cold-home.test.ts index cc0fe4d51..117aff573 100644 --- a/tests/native-compose-file-cold-home.test.ts +++ b/tests/native-compose-file-cold-home.test.ts @@ -182,11 +182,15 @@ test.each([ } else { await writeFile(home, "owned non-directory", { mode: 0o600 }); } - await store.withMutation(async (mutation) => { - const input = await selected(mutation); - await expect(input.owner.prepare(input)).rejects.toMatchObject({ - code: "E_NATIVE_COMPOSE_STATE", - }); + // Journal admission may reject a non-directory home during acquisition; + // otherwise the private material owner rejects it before preparation. + await expect( + store.withMutation(async (mutation) => { + const input = await selected(mutation); + await input.owner.prepare(input); + }) + ).rejects.toMatchObject({ + code: "E_NATIVE_COMPOSE_STATE", }); expect(await readdir(foreign)).toEqual([]); await expect(lstat(join(home, "compose-files"))).rejects.toMatchObject({ From 060119a9d536b2037a1e26b96ab21cf271f33cc5 Mon Sep 17 00:00:00 2001 From: Kit <344842+roodboi@users.noreply.github.com> Date: Sat, 14 Nov 2026 00:48:40 +0000 Subject: [PATCH 07/32] test(env): retain stage-specific unsafe-home assertions --- tests/native-compose-file-cold-home.test.ts | 26 ++++++++++++++------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/tests/native-compose-file-cold-home.test.ts b/tests/native-compose-file-cold-home.test.ts index 117aff573..c03215a28 100644 --- a/tests/native-compose-file-cold-home.test.ts +++ b/tests/native-compose-file-cold-home.test.ts @@ -182,16 +182,24 @@ test.each([ } else { await writeFile(home, "owned non-directory", { mode: 0o600 }); } - // Journal admission may reject a non-directory home during acquisition; - // otherwise the private material owner rejects it before preparation. - await expect( - store.withMutation(async (mutation) => { + if (kind === "file") { + // Journal admission rejects a non-directory home during acquisition. + await expect( + store.withMutation(async (mutation) => { + const input = await selected(mutation); + await input.owner.prepare(input); + }) + ).rejects.toMatchObject({ + code: "E_NATIVE_COMPOSE_STATE", + }); + } else { + await store.withMutation(async (mutation) => { const input = await selected(mutation); - await input.owner.prepare(input); - }) - ).rejects.toMatchObject({ - code: "E_NATIVE_COMPOSE_STATE", - }); + await expect(input.owner.prepare(input)).rejects.toMatchObject({ + code: "E_NATIVE_COMPOSE_STATE", + }); + }); + } expect(await readdir(foreign)).toEqual([]); await expect(lstat(join(home, "compose-files"))).rejects.toMatchObject({ code: kind === "file" ? "ENOTDIR" : "ENOENT", From d8169fc644f22384074c427879dec33061296802 Mon Sep 17 00:00:00 2001 From: Kit <344842+roodboi@users.noreply.github.com> Date: Sat, 14 Nov 2026 02:44:43 +0000 Subject: [PATCH 08/32] fix(env): integrate snapshot provider and qualify historical upgrades --- docs/env.md | 5 +- docs/guides/upgrade-to-v5.md | 6 +- docs/reference/env-migration.md | 10 +- src/lib/env-migration-snapshot.ts | 146 +++++++++++++----------- tests/env-migration-provider.test.ts | 35 ++++++ tests/v4-upgrade.test.ts | 164 +++++++++++++++++++++++---- 6 files changed, 271 insertions(+), 95 deletions(-) diff --git a/docs/env.md b/docs/env.md index 3fd09452e..4c4649b59 100644 --- a/docs/env.md +++ b/docs/env.md @@ -1346,7 +1346,10 @@ hack env migrate --apply --accept-tracked-env-changes Preview is read-only and lists file actions and key names, never values. Apply creates a private snapshot outside the working tree and records idempotent steps in a journal. Re-run it after interruption; a finished migration is a no-op. -`doctor --migrate-env-config` remains an applying alias. Both backends use the +`doctor --migrate-env-config` still applies migration; its existing hygiene +repairs can run before the migration snapshot and are reported separately. +Use `hack env migrate --apply` for migration-only preview and rollback. +Both backends use the same file migration; Compose users need no native runtime, and neither preview nor apply needs Docker or a daemon. diff --git a/docs/guides/upgrade-to-v5.md b/docs/guides/upgrade-to-v5.md index ff74df33e..356c6e8c2 100644 --- a/docs/guides/upgrade-to-v5.md +++ b/docs/guides/upgrade-to-v5.md @@ -123,7 +123,11 @@ file paths before applying. `hack env migrate --apply` snapshots originals outsi the working tree and records resumable steps; repeat the same command after an interruption. Add `--accept-tracked-env-changes` only when consenting to the previewed shared edits. `doctor --migrate-env-config` still applies the migration, -but now retains all legacy copies, including untracked dotenv files and tracked +alongside its existing hygiene repairs; the env journal covers migration writes, +not earlier doctor repairs. Use `hack env migrate --apply` for migration-only +preview and rollback. Snapshot storage and the target files must share a +filesystem; apply refuses before writing project files if rollback would cross one. +Every migrating route now retains all legacy copies, including untracked dotenv files and tracked `hack.env.json`. Implicit migrations retain them too. After checking a successful complete-project `hack up`, preview the separate diff --git a/docs/reference/env-migration.md b/docs/reference/env-migration.md index 4da3891bd..313b4346c 100644 --- a/docs/reference/env-migration.md +++ b/docs/reference/env-migration.md @@ -86,7 +86,11 @@ rename succeeded before its receipt. An unexpected input or destination change refuses instead of guessing. Re-run the same apply command after interruption. After completion it returns `already_complete` without rewriting the project or journal. `doctor --migrate-env-config` uses the same apply/journal path and keeps -all legacy files. Implicit migrations from env, session and startup commands also +all legacy files. Doctor can also perform its existing hygiene repairs before +migration, such as adding ignore rules or tightening permissions; those separate +doctor repairs are outside the env migration journal. Use `hack env migrate +--apply` when the preview and rollback must cover only the migration's writes. +Implicit migrations from env, session and startup commands also keep legacy files and resume an unfinished journal; they preserve existing Compose `env_file` references. Implicit commands refuse an unfinished apply or rollback journal before using @@ -97,6 +101,10 @@ staging directories. Unknown or edited entries are retained, reported as `payloadCleanup: "kept"` in the apply result. Original snapshot blobs remain for rollback. A private data home inside a Git working tree refuses apply; choose a private `HACK_HOME` outside repositories, including a home managed as dotfiles. +Snapshot storage must also share a filesystem with every migration target: +the snapshot library restores with rename and hard links. Apply refuses before +project writes if that guarantee is unavailable. Choose a private `HACK_HOME` +on the target filesystem, outside Git working trees, then retry the preview. Migration never includes cleanup. A later explicit cleanup requires: diff --git a/src/lib/env-migration-snapshot.ts b/src/lib/env-migration-snapshot.ts index cdb790655..ea59b3ee2 100644 --- a/src/lib/env-migration-snapshot.ts +++ b/src/lib/env-migration-snapshot.ts @@ -1,5 +1,5 @@ -import { lstat } from "node:fs/promises"; -import { join } from "node:path"; +import { lstat, stat } from "node:fs/promises"; +import { dirname, join } from "node:path"; import { ClosedCliError } from "./cli-result.ts"; import { migrationRefusal, @@ -7,6 +7,14 @@ import { sameMigrationFile, } from "./env-migration-files.ts"; import { isRecord } from "./guards.ts"; +import { + createUpgradeSnapshot, + readUpgradeSnapshot, + recordUpgradeSnapshotAfter, + restoreUpgradeSnapshot, + type UpgradeRestoreEntry, + type UpgradeSnapshotState, +} from "./upgrade-snapshot.ts"; /** Metadata only. An absent path has a null checksum and mode. */ export type EnvMigrationFileState = { @@ -45,61 +53,75 @@ export type EnvMigrationSnapshots = { }>; }; -/** Late binding keeps this unit independent until HACK-2189 lands; replace with a typed static import on integration. */ -export async function envMigrationSnapshots(): Promise { - const moduleUrl = new URL("./upgrade-snapshot.ts", import.meta.url).href; - let library: unknown; - try { - library = await import(moduleUrl); - } catch { - throw new ClosedCliError( - "Env migration requires the upgrade snapshot library. Preview is available; no migration files were changed." - ); +type SnapshotLibrary = { + readonly createUpgradeSnapshot: typeof createUpgradeSnapshot; + readonly readUpgradeSnapshot: typeof readUpgradeSnapshot; + readonly recordUpgradeSnapshotAfter: typeof recordUpgradeSnapshotAfter; + readonly restoreUpgradeSnapshot: typeof restoreUpgradeSnapshot; +}; + +/** Static imports keep the production provider in both source and compiled CLI routes. */ +export function envMigrationSnapshots(): EnvMigrationSnapshots { + return bindEnvMigrationSnapshots({ + createUpgradeSnapshot, + readUpgradeSnapshot, + recordUpgradeSnapshotAfter, + restoreUpgradeSnapshot, + }); +} + +/** The provider restores by rename/hard link; refuse before migration if that cannot work. */ +async function assertRestoreFilesystem( + snapshotDir: string, + files: readonly EnvMigrationFileState[] +): Promise { + const snapshot = await stat(snapshotDir); + for (const file of files) { + let parent = dirname(file.path); + for (;;) { + const info = await stat(parent).catch((error: unknown) => { + if (isRecord(error) && error.code === "ENOENT") { + return null; + } + throw migrationRefusal(); + }); + if (info) { + if (!info.isDirectory() || info.dev !== snapshot.dev) { + throw new ClosedCliError( + "Env migration needs snapshot storage on the same filesystem as every target so rollback can restore it. Choose a private HACK_HOME on that filesystem outside Git working trees; no migration files were changed (values omitted)." + ); + } + break; + } + parent = dirname(parent); + } } - return bindEnvMigrationSnapshots(library); } /** Adapter checks both the snapshot owner's verdict and the journal's exact expectations. */ export function bindEnvMigrationSnapshots( - library: unknown + library: SnapshotLibrary ): EnvMigrationSnapshots { - if (!isRecord(library)) { - throw migrationRefusal(); - } - const invoke = async (name: string, input: unknown): Promise => { - const operation = library[name]; - if (typeof operation !== "function") { - throw migrationRefusal(); - } - return await Reflect.apply(operation, undefined, [input]); - }; const load = async (manifestPath: string) => { - const loaded = await invoke("readUpgradeSnapshot", { manifestPath }); - if ( - !( - isRecord(loaded) && - loaded.status === "ok" && - isRecord(loaded.snapshot) && - isRecord(loaded.snapshot.manifest) && - Array.isArray(loaded.snapshot.manifest.entries) && - typeof loaded.snapshot.dir === "string" - ) - ) { + const loaded = await library.readUpgradeSnapshot({ manifestPath }); + if (loaded.status !== "ok") { throw migrationRefusal(); } return { dir: loaded.snapshot.dir, - entries: loaded.snapshot.manifest.entries as unknown[], + entries: loaded.snapshot.manifest.entries, }; }; - const matches = (state: unknown, expected: EnvMigrationFileState) => - isRecord(state) && - (expected.sha256 === null + const matches = ( + state: UpgradeSnapshotState, + expected: EnvMigrationFileState + ) => + expected.sha256 === null ? state.kind === "absent" : state.kind === "file" && state.sha256 === expected.sha256 && state.size === expected.size && - state.mode === expected.mode); + state.mode === expected.mode; const verifyCurrent = async (files: readonly EnvMigrationFileState[]) => { for (const expected of files) { if ( @@ -122,9 +144,9 @@ export function bindEnvMigrationSnapshots( for (const expected of expectedCurrent) { const current = (await readMigrationFile(expected.path)).state; const original = snapshot.entries.find( - (entry) => isRecord(entry) && entry.path === expected.path + (entry) => entry.path === expected.path ); - if (isRecord(original) && matches(original.before, current)) { + if (original && matches(original.before, current)) { continue; } if (!(sameMigrationFile(current, expected) || preserveChanges)) { @@ -138,21 +160,14 @@ export function bindEnvMigrationSnapshots( }; return { capture: async ({ projectRoot, files }) => { - const result = await invoke("createUpgradeSnapshot", { + const result = await library.createUpgradeSnapshot({ purpose: "env-migration", targets: files.map((file) => ({ path: file.path, label: `env migration: ${file.path}`, })), }); - if ( - !( - isRecord(result) && - result.status === "created" && - isRecord(result.snapshot) && - typeof result.snapshot.manifestPath === "string" - ) - ) { + if (result.status !== "created") { throw new ClosedCliError( `Snapshot refused for ${projectRoot}; no migration files were changed (values omitted).` ); @@ -161,11 +176,12 @@ export function bindEnvMigrationSnapshots( }, assertCoverage: async ({ manifestPath, files }) => { const snapshot = await load(manifestPath); + await assertRestoreFilesystem(snapshot.dir, files); for (const expected of files) { const entry = snapshot.entries.find( - (candidate) => isRecord(candidate) && candidate.path === expected.path + (candidate) => candidate.path === expected.path ); - if (!(isRecord(entry) && matches(entry.before, expected))) { + if (!(entry && matches(entry.before, expected))) { throw migrationRefusal(); } if (expected.sha256 !== null) { @@ -184,11 +200,11 @@ export function bindEnvMigrationSnapshots( }, recordAfter: async ({ manifestPath, files }) => { await verifyCurrent(files); - const result = await invoke("recordUpgradeSnapshotAfter", { + const result = await library.recordUpgradeSnapshotAfter({ manifestPath, paths: files.map((file) => file.path), }); - if (!(isRecord(result) && result.status === "recorded")) { + if (result.status !== "recorded") { throw migrationRefusal(); } await verifyCurrent(files); @@ -202,17 +218,17 @@ export function bindEnvMigrationSnapshots( manifestPath ); // A killed publication may predate the provider's after receipt; only journal-proven bytes may fill that gap. - const recorded = await invoke("recordUpgradeSnapshotAfter", { + const recorded = await library.recordUpgradeSnapshotAfter({ manifestPath, paths: pending.map((file) => file.path), }); - if (!(isRecord(recorded) && recorded.status === "recorded")) { + if (recorded.status !== "recorded") { throw migrationRefusal(); } - const results: unknown[] = []; + const results: UpgradeRestoreEntry[] = []; let incomplete = false; for (const target of expectedCurrent) { - const result = await invoke("restoreUpgradeSnapshot", { + const result = await library.restoreUpgradeSnapshot({ manifestPath, paths: [target.path], beforeReplace: async (path: string) => { @@ -222,13 +238,7 @@ export function bindEnvMigrationSnapshots( } }, }); - if ( - !( - isRecord(result) && - (result.status === "restored" || result.status === "incomplete") && - Array.isArray(result.results) - ) - ) { + if (result.status === "invalid-manifest") { throw migrationRefusal(); } results.push(...result.results); @@ -238,9 +248,7 @@ export function bindEnvMigrationSnapshots( } } const preserved = results.flatMap((result) => - isRecord(result) && - typeof result.path === "string" && - typeof result.keptAs === "string" + result.keptAs !== undefined ? [{ path: result.path, backupPath: result.keptAs }] : [] ); diff --git a/tests/env-migration-provider.test.ts b/tests/env-migration-provider.test.ts index e396437f5..43b22c1fb 100644 --- a/tests/env-migration-provider.test.ts +++ b/tests/env-migration-provider.test.ts @@ -1,5 +1,6 @@ import { afterEach, beforeEach, expect, test } from "bun:test"; import { + lstat, mkdir, mkdtemp, readFile, @@ -9,9 +10,12 @@ import { } from "node:fs/promises"; import { tmpdir } from "node:os"; import { isAbsolute, join, resolve } from "node:path"; +import { applyEnvMigration } from "../src/lib/env-migration.ts"; +import { bindEnvMigrationSnapshots } from "../src/lib/env-migration-snapshot.ts"; import { isRecord } from "../src/lib/guards.ts"; import { resolveProjectEnvConfig } from "../src/lib/project-env-config.ts"; import { resolveSecretStore } from "../src/lib/secret-store.ts"; +import * as snapshotLibrary from "../src/lib/upgrade-snapshot.ts"; import { restoreEnv } from "./helpers/env.ts"; import { fixtureChildren } from "./helpers/fixture-children.ts"; import { privateCliEnv } from "./helpers/private-cli-env.ts"; @@ -243,3 +247,34 @@ test("production rollback maps preserved edits to retrievable private files with `TOKEN=${sentinel}\n` ); }); + +test("apply refuses cross-filesystem snapshot storage before any project publication", async () => { + const sharedMemory = await lstat("/dev/shm").catch(() => null); + if (!sharedMemory || sharedMemory.dev === (await lstat(root)).dev) { + return; // This host has no second filesystem; Linux CI exercises the boundary. + } + const snapshotRoot = await mkdtemp(join("/dev/shm", "env-snapshot-")); + try { + const project = await fixture(); + const before = await readFile(join(project.projectDir, ".env")); + const snapshots = bindEnvMigrationSnapshots({ + ...snapshotLibrary, + createUpgradeSnapshot: (input) => + snapshotLibrary.createUpgradeSnapshot({ ...input, root: snapshotRoot }), + }); + await expect( + applyEnvMigration({ project, snapshots, acceptTrackedChanges: true }) + ).rejects.toThrow("same filesystem"); + expect( + (await readFile(join(project.projectDir, ".env"))).equals(before) + ).toBe(true); + expect( + await Bun.file(join(project.projectDir, "hack.env.default.yaml")).exists() + ).toBe(false); + expect( + await Bun.file(join(project.projectRoot, ".hack.secret.key")).exists() + ).toBe(false); + } finally { + await rm(snapshotRoot, { recursive: true, force: true }); + } +}); diff --git a/tests/v4-upgrade.test.ts b/tests/v4-upgrade.test.ts index 68f95711b..85a718d7b 100644 --- a/tests/v4-upgrade.test.ts +++ b/tests/v4-upgrade.test.ts @@ -1,8 +1,10 @@ import { afterAll, beforeAll, expect, test } from "bun:test"; +import { createHash } from "node:crypto"; import { mkdtemp, readdir, readFile, realpath, rm } from "node:fs/promises"; import { tmpdir } from "node:os"; -import { join, resolve } from "node:path"; +import { join, relative, resolve } from "node:path"; import packageJson from "../package.json"; +import { isRecord } from "../src/lib/guards.ts"; import { verifyStagedReleaseVersion } from "../src/lib/release-version.ts"; import { privateCliEnv } from "./helpers/private-cli-env.ts"; import { createV4UpgradeFixture } from "./helpers/v4-upgrade-fixture.ts"; @@ -127,6 +129,7 @@ for (const globalKeyPresent of [true, false]) { expect(refused.code).toBe(1); expect(refused.output).toContain("--accept-tracked-env-changes"); expect(await f.snapshot()).toEqual(beforeRefusal); + const beforeApply = new Map([[f.tracked.path, beforeRefusal]]); const apply = await f.run([ "doctor", "--path", @@ -137,13 +140,13 @@ for (const globalKeyPresent of [true, false]) { expect(apply.output).not.toContain("E_INTERACTIVE_REQUIRED"); // Assertions carry booleans/digests only, including their failure diagnostics. expect((await readFile(f.keyPath)).equals(keyBefore)).toBe(true); - expect(await Bun.file(f.storePath).exists()).toBe(false); + expect(await Bun.file(f.storePath).exists()).toBe(true); expect( await Bun.file(join(f.tracked.path, ".hack/hack.env.json")).exists(), apply.output - ).toBe(false); + ).toBe(true); expect(await Bun.file(join(f.tracked.path, ".hack/.env")).exists()).toBe( - false + true ); expect( await Bun.file( @@ -156,9 +159,9 @@ for (const globalKeyPresent of [true, false]) { ).exists() ).toBe(true); expect((await f.resolveValues(f.tracked)).code).toBe(0); - // The linked checkout still resolves its independently declared plaintext value. - expect((await f.resolveValues(f.worktree)).code).toBe(0); + // Keep explicit consent inputs intact; value probes can implicitly migrate. for (const project of [f.worktree, f.untracked]) { + beforeApply.set(project.path, await f.snapshot()); const flags = project === f.worktree ? ["--accept-tracked-env-changes"] : []; const result = await f.run([ @@ -171,6 +174,9 @@ for (const globalKeyPresent of [true, false]) { expect(result.output).not.toContain("E_INTERACTIVE_REQUIRED"); expect((await f.resolveValues(project)).code).toBe(0); } + // Re-probe the linked value after explicit migration, rather than + // implicitly migrating it before its consent path is exercised. + expect((await f.resolveValues(f.worktree)).code).toBe(0); expect((await readFile(f.keyPath)).equals(keyBefore)).toBe(true); expect((await readFile(f.globalConfig)).equals(configBefore)).toBe(true); if (globalKeyPresent) { @@ -190,15 +196,134 @@ for (const globalKeyPresent of [true, false]) { false ); } + const migrationFiles: string[] = []; + for (const project of [f.tracked, f.untracked, f.worktree]) { + const id = createHash("sha256").update(project.path).digest("hex"); + const originals = beforeApply.get(project.path); + if (!originals) { + throw new Error("Missing pre-apply fixture observation"); + } + const journalName = `home/.hack/env-migrations/${id}/journal.json`; + const journalText = await readFile(join(f.root, journalName), "utf8"); + expect(f.sentinels.some((value) => journalText.includes(value))).toBe( + false + ); + const journal: unknown = JSON.parse(journalText); + if (!(isRecord(journal) && typeof journal.manifestPath === "string")) { + throw new Error("Expected migration journal metadata"); + } + expect(journal.state).toBe("complete"); + expect(journal.removed).toEqual([]); + const manifestName = relative(f.root, journal.manifestPath); + expect(manifestName).toMatch( + /^home\/\.hack\/upgrade\/snapshots\/[A-Za-z0-9._-]+\/manifest\.json$/ + ); + const text = await readFile(journal.manifestPath, "utf8"); + expect(f.sentinels.some((value) => text.includes(value))).toBe(false); + const manifest: unknown = JSON.parse(text); + if (!(isRecord(manifest) && Array.isArray(manifest.entries))) { + throw new Error("Expected private snapshot metadata"); + } + migrationFiles.push(journalName, manifestName); + for (const entry of manifest.entries) { + if ( + !( + isRecord(entry) && + typeof entry.path === "string" && + isRecord(entry.before) + ) + ) { + throw new Error("Expected snapshot original metadata"); + } + const source = relative(f.root, entry.path); + let original = originals[source]; + if ( + original === undefined && + /^(tracked|untracked|linked)\/(?:\.hack\/)?\.hack-staging\/\.gitignore$/.test( + source + ) && + entry.before.kind === "file" + ) { + original = createHash("sha256").update("*\n").digest("hex"); + } + // Doctor's secrets hygiene runs before migration's snapshot and + // creates ignore files. Those fixed housekeeping writes are outside + // the env migration journal; all other originals match pre-doctor bytes. + if ( + originals[source] === undefined && + [ + "tracked/.gitignore", + "tracked/.hack/.gitignore", + "untracked/.hack/.gitignore", + "linked/.hack/.gitignore", + ].includes(source) + ) { + if (typeof entry.before.sha256 !== "string") { + throw new Error( + "Expected snapshot of pre-migration ignore rules" + ); + } + const ignore = await readFile( + join(journal.manifestPath, "..", "blobs", entry.before.sha256) + ); + const ignoreText = ignore.toString("utf8"); + expect( + f.sentinels.some((value) => ignoreText.includes(value)) + ).toBe(false); + const rules = ignoreText + .split("\n") + .filter((line) => line !== "" && !line.startsWith("#")); + if (source === "tracked/.gitignore") { + expect(rules).toContain(".hack.secret.key"); + expect(rules).toContain("/legacy-store.json"); + expect( + rules.every((rule) => + [ + ".hack.secret.key", + "/.hack.secret.key", + "/legacy-store.json", + ].includes(rule) + ) + ).toBe(true); + } else { + expect(ignoreText).toContain(".env"); + expect( + ignoreText === + "# machine-local secrets (hack doctor --fix)\n.env\n" || + ignoreText.includes("hack.env.local.yaml") + ).toBe(true); + } + original = createHash("sha256").update(ignore).digest("hex"); + } + if (entry.before.kind === "absent") { + expect(original, source).toBeUndefined(); + continue; + } + expect(entry.before.sha256, source).toBe(original); + if (typeof entry.before.sha256 !== "string") { + throw new Error("Expected snapshot original digest"); + } + const blobName = join( + manifestName, + "..", + "blobs", + entry.before.sha256 + ); + expect(stable[blobName]).toBe(entry.before.sha256); + migrationFiles.push(blobName); + } + } + expect( + changed.filter((name) => migrationFiles.includes(name)).sort() + ).toEqual([...new Set(migrationFiles)].sort()); // Private writes keep an empty, self-ignoring stage directory (HACK-2165). // Pin these exact effects as well as the migration's payload changes. const stagingMarkers = [ "linked/.hack/.hack-staging/.gitignore", "tracked/.hack/.hack-staging/.gitignore", "untracked/.hack/.hack-staging/.gitignore", - globalKeyPresent - ? "untracked/.hack-staging/.gitignore" - : "home/.hack/.hack-staging/.gitignore", + "tracked/.hack-staging/.gitignore", + ...(globalKeyPresent ? ["untracked/.hack-staging/.gitignore"] : []), ]; for (const marker of stagingMarkers) { expect(await readFile(join(f.root, marker), "utf8")).toBe("*\n"); @@ -218,18 +343,12 @@ for (const globalKeyPresent of [true, false]) { ...stagingMarkers, "linked/.hack/docker-compose.yml", "linked/.hack/.gitignore", - "tracked/.hack/.env", "tracked/.hack/docker-compose.yml", - "tracked/.hack/hack.config.json", - "tracked/.hack/hack.env.json", - "tracked/legacy-store.json", "tracked/.gitignore", "tracked/.hack/.gitignore", "tracked/.hack/hack.env.default.yaml", "tracked/.hack/hack.env.local.yaml", - "untracked/.hack/.env", "untracked/.hack/docker-compose.yml", - "untracked/.hack/hack.env.json", "untracked/.hack/.gitignore", "untracked/.hack/hack.env.default.yaml", "untracked/.hack/hack.env.local.yaml", @@ -237,13 +356,15 @@ for (const globalKeyPresent of [true, false]) { ? ["untracked/.gitignore", "untracked/.hack.secret.key"] : [ "home/.hack/secrets.enc.json", - "home/.hack/secrets-file.key", "home/.hack//secrets.enc.json", ]), ]; expect( changed - .filter((name) => !commands.includes(name)) + .filter( + (name) => + !(commands.includes(name) || migrationFiles.includes(name)) + ) .map((name) => name.startsWith("home/.hack/secrets.enc.json.retired-") ? "home/.hack//secrets.enc.json" @@ -289,12 +410,9 @@ for (const globalKeyPresent of [true, false]) { if (globalKeyPresent) { expect(await Bun.file(f.globalKeyPath).exists()).toBe(true); } - // Migration may initialize a fresh global backend after the unused lost-key - // store is retired. The retired ciphertext must remain byte-identical. - else if (await Bun.file(f.globalKeyPath).exists()) { - expect( - (await readFile(f.globalKeyPath, "utf8")).trim().length > 0 - ).toBe(true); + // Migration reads legacy backends but never provisions a replacement key. + else { + expect(await Bun.file(f.globalKeyPath).exists()).toBe(false); } for (const project of [f.tracked, f.untracked, f.worktree]) { expect((await f.resolveValues(project)).code).toBe(0); From 0b43a4bb9d090009dff7ad84eaed7664f9f75073 Mon Sep 17 00:00:00 2001 From: Kit <344842+roodboi@users.noreply.github.com> Date: Sat, 14 Nov 2026 03:00:59 +0000 Subject: [PATCH 09/32] fix(env): probe rollback links before migration publication --- docs/reference/env-migration.md | 14 +++-- src/lib/env-migration-snapshot.ts | 77 +++++++++++++++++++++++++--- tests/env-migration-provider.test.ts | 39 ++++++++++++++ tests/v4-upgrade.test.ts | 14 +++++ 4 files changed, 132 insertions(+), 12 deletions(-) diff --git a/docs/reference/env-migration.md b/docs/reference/env-migration.md index 313b4346c..c4eda960b 100644 --- a/docs/reference/env-migration.md +++ b/docs/reference/env-migration.md @@ -101,10 +101,16 @@ staging directories. Unknown or edited entries are retained, reported as `payloadCleanup: "kept"` in the apply result. Original snapshot blobs remain for rollback. A private data home inside a Git working tree refuses apply; choose a private `HACK_HOME` outside repositories, including a home managed as dotfiles. -Snapshot storage must also share a filesystem with every migration target: -the snapshot library restores with rename and hard links. Apply refuses before -project writes if that guarantee is unavailable. Choose a private `HACK_HOME` -on the target filesystem, outside Git working trees, then retry the preview. +Snapshot storage must also share a filesystem and mount with every migration +target, with hard links permitted: the snapshot library restores with rename and +hard links. Apply checks this before publishing any migration target. It briefly +links an exclusively created, empty `.hack-env-restore-probe-*` file from the +snapshot to each existing target parent, then removes both names. These temporary +probes contain no input bytes and are not migration targets in `files`; preview +never creates them. A process killed during this check can leave an empty probe. +If admission refuses, choose a private `HACK_HOME` on the target mount outside +Git working trees, then retry the preview. Matching device numbers alone do not +prove that two paths share a mount, for example with Linux bind mounts. Migration never includes cleanup. A later explicit cleanup requires: diff --git a/src/lib/env-migration-snapshot.ts b/src/lib/env-migration-snapshot.ts index ea59b3ee2..4b22c54f1 100644 --- a/src/lib/env-migration-snapshot.ts +++ b/src/lib/env-migration-snapshot.ts @@ -1,4 +1,6 @@ -import { lstat, stat } from "node:fs/promises"; +import { randomUUID } from "node:crypto"; +import type { Stats } from "node:fs"; +import { link, lstat, open, stat, unlink } from "node:fs/promises"; import { dirname, join } from "node:path"; import { ClosedCliError } from "./cli-result.ts"; import { @@ -70,12 +72,68 @@ export function envMigrationSnapshots(): EnvMigrationSnapshots { }); } -/** The provider restores by rename/hard link; refuse before migration if that cannot work. */ +function restoreFilesystemRefusal(): ClosedCliError { + return new ClosedCliError( + "Env migration needs snapshot storage on the same filesystem and mount as every target, with hard links permitted, so rollback can restore it. Choose a private HACK_HOME on that filesystem outside Git working trees; no migration files were changed (values omitted)." + ); +} + +async function removeRestoreProbe( + path: string, + identity: Stats +): Promise { + const current = await lstat(path); + if (current.dev !== identity.dev || current.ino !== identity.ino) { + throw restoreFilesystemRefusal(); + } + await unlink(path); +} + +/** Empty, exclusively created probes contain no input bytes and never replace an existing name. */ +async function probeRestoreLink(input: { + readonly snapshotDir: string; + readonly parent: string; + readonly linkFile: typeof link; +}): Promise { + const name = `.hack-env-restore-probe-${randomUUID()}`; + const source = join(input.snapshotDir, name); + const destination = join(input.parent, name); + let identity: Stats | null = null; + let linked = false; + try { + try { + const handle = await open(source, "wx", 0o600); + try { + identity = await handle.stat(); + } finally { + await handle.close(); + } + await input.linkFile(source, destination); + linked = true; + } finally { + if (identity) { + try { + if (linked) { + await removeRestoreProbe(destination, identity); + } + } finally { + await removeRestoreProbe(source, identity); + } + } + } + } catch { + throw restoreFilesystemRefusal(); + } +} + +/** Device numbers alone miss distinct bind mounts on the same filesystem. */ async function assertRestoreFilesystem( snapshotDir: string, - files: readonly EnvMigrationFileState[] + files: readonly EnvMigrationFileState[], + linkFile: typeof link ): Promise { const snapshot = await stat(snapshotDir); + const checked = new Set(); for (const file of files) { let parent = dirname(file.path); for (;;) { @@ -87,9 +145,11 @@ async function assertRestoreFilesystem( }); if (info) { if (!info.isDirectory() || info.dev !== snapshot.dev) { - throw new ClosedCliError( - "Env migration needs snapshot storage on the same filesystem as every target so rollback can restore it. Choose a private HACK_HOME on that filesystem outside Git working trees; no migration files were changed (values omitted)." - ); + throw restoreFilesystemRefusal(); + } + if (!checked.has(parent)) { + await probeRestoreLink({ snapshotDir, parent, linkFile }); + checked.add(parent); } break; } @@ -100,7 +160,8 @@ async function assertRestoreFilesystem( /** Adapter checks both the snapshot owner's verdict and the journal's exact expectations. */ export function bindEnvMigrationSnapshots( - library: SnapshotLibrary + library: SnapshotLibrary, + { linkRestoreProbe = link }: { readonly linkRestoreProbe?: typeof link } = {} ): EnvMigrationSnapshots { const load = async (manifestPath: string) => { const loaded = await library.readUpgradeSnapshot({ manifestPath }); @@ -176,7 +237,7 @@ export function bindEnvMigrationSnapshots( }, assertCoverage: async ({ manifestPath, files }) => { const snapshot = await load(manifestPath); - await assertRestoreFilesystem(snapshot.dir, files); + await assertRestoreFilesystem(snapshot.dir, files, linkRestoreProbe); for (const expected of files) { const entry = snapshot.entries.find( (candidate) => candidate.path === expected.path diff --git a/tests/env-migration-provider.test.ts b/tests/env-migration-provider.test.ts index 43b22c1fb..c5e1b04cb 100644 --- a/tests/env-migration-provider.test.ts +++ b/tests/env-migration-provider.test.ts @@ -3,6 +3,7 @@ import { lstat, mkdir, mkdtemp, + readdir, readFile, realpath, rm, @@ -278,3 +279,41 @@ test("apply refuses cross-filesystem snapshot storage before any project publica await rm(snapshotRoot, { recursive: true, force: true }); } }); + +for (const code of ["EXDEV", "EPERM"]) { + test(`apply refuses a same-device restore link failure (${code}) before publication`, async () => { + const project = await fixture(); + const names = ( + await readdir(project.projectRoot, { recursive: true }) + ).sort(); + let probed = false; + const snapshots = bindEnvMigrationSnapshots(snapshotLibrary, { + linkRestoreProbe: async (source, destination) => { + // Unlike the second-filesystem case, admission reached the real link + // boundary after proving matching devices and creating an empty probe. + expect((await lstat(source)).dev).toBe( + (await lstat(project.projectRoot)).dev + ); + expect((await readFile(source)).length).toBe(0); + expect(String(destination)).toContain(".hack-env-restore-probe-"); + probed = true; + throw Object.assign(new Error("synthetic link refusal"), { code }); + }, + }); + await expect( + applyEnvMigration({ project, snapshots, acceptTrackedChanges: true }) + ).rejects.toThrow("same filesystem and mount"); + expect(probed).toBe(true); + expect( + (await readdir(project.projectRoot, { recursive: true })).sort() + ).toEqual(names); + expect(await readFile(join(project.projectDir, ".env"), "utf8")).toBe( + `TOKEN=${sentinel}\n` + ); + expect( + (await readdir(process.env.HACK_HOME ?? "", { recursive: true })).some( + (name) => name.includes(".hack-env-restore-probe-") + ) + ).toBe(false); + }); +} diff --git a/tests/v4-upgrade.test.ts b/tests/v4-upgrade.test.ts index 85a718d7b..e4f9da327 100644 --- a/tests/v4-upgrade.test.ts +++ b/tests/v4-upgrade.test.ts @@ -111,6 +111,20 @@ for (const globalKeyPresent of [true, false]) { baseline.untracked, baseline.worktree, ]) { + if (project === baseline.worktree) { + // This route starts with a migrated primary and a legacy linked + // checkout, then resolves through the normal implicit migration. + expect( + await Bun.file( + join(baseline.tracked.path, ".hack/hack.env.default.yaml") + ).exists() + ).toBe(true); + expect( + await Bun.file( + join(baseline.worktree.path, ".hack/hack.env.default.yaml") + ).exists() + ).toBe(false); + } const resolution = await baseline.resolveValues(project); expect(resolution.code, `${project.path}: ${resolution.output}`).toBe( 0 From 7b3fae28705ab2f9e54bb4617de28b2e76e6abe7 Mon Sep 17 00:00:00 2001 From: Kit <344842+roodboi@users.noreply.github.com> Date: Sat, 14 Nov 2026 03:16:00 +0000 Subject: [PATCH 10/32] fix(env): admit snapshot storage before copying inputs --- docs/reference/env-migration.md | 24 +++++++--- src/lib/env-migration-files.ts | 38 ++++++++++++++++ src/lib/env-migration-snapshot.ts | 49 +++++++++++++++++++-- src/lib/env-migration.ts | 47 +------------------- src/lib/project-env-config.ts | 27 +++++++----- tests/env-migration-provider.test.ts | 66 ++++++++++++++++++++++------ 6 files changed, 173 insertions(+), 78 deletions(-) diff --git a/docs/reference/env-migration.md b/docs/reference/env-migration.md index c4eda960b..b8126934e 100644 --- a/docs/reference/env-migration.md +++ b/docs/reference/env-migration.md @@ -101,14 +101,26 @@ staging directories. Unknown or edited entries are retained, reported as `payloadCleanup: "kept"` in the apply result. Original snapshot blobs remain for rollback. A private data home inside a Git working tree refuses apply; choose a private `HACK_HOME` outside repositories, including a home managed as dotfiles. + Snapshot storage must also share a filesystem and mount with every migration target, with hard links permitted: the snapshot library restores with rename and -hard links. Apply checks this before publishing any migration target. It briefly -links an exclusively created, empty `.hack-env-restore-probe-*` file from the -snapshot to each existing target parent, then removes both names. These temporary -probes contain no input bytes and are not migration targets in `files`; preview -never creates them. A process killed during this check can leave an empty probe. -If admission refuses, choose a private `HACK_HOME` on the target mount outside +hard links. Apply admits the storage location before copying original inputs, +then checks the completed snapshot again before publishing any migration target. +Cleanup and rollback repeat the snapshot check. Admission briefly links an +exclusively created, empty `.hack-env-restore-probe-*` file from the +storage directory (or its nearest existing parent before capture) to each +existing target parent, then removes both names. These temporary probes contain +no input bytes and are not migration targets in `files`; preview +never creates them. A process killed during an apply, cleanup or rollback check +can leave an empty probe. It is retained rather than removed by matching its +name alone. + +An interruption after snapshot capture but before journal publication +can leave a private original-file snapshot; `hack upgrade rollback` without a +manifest lists retained snapshots. The snapshot can contain original values and +keys in private blobs, although its manifest contains metadata only. Keep it +private even when no migration target was published. If admission refuses, +choose a private `HACK_HOME` on the target mount outside Git working trees, then retry the preview. Matching device numbers alone do not prove that two paths share a mount, for example with Linux bind mounts. diff --git a/src/lib/env-migration-files.ts b/src/lib/env-migration-files.ts index b6f9df5bc..d51698662 100644 --- a/src/lib/env-migration-files.ts +++ b/src/lib/env-migration-files.ts @@ -1,6 +1,7 @@ import { createHash } from "node:crypto"; import { constants } from "node:fs"; import { type FileHandle, lstat, open } from "node:fs/promises"; +import { dirname, isAbsolute, join, relative, sep } from "node:path"; import { ClosedCliError } from "./cli-result.ts"; import type { EnvMigrationFileState } from "./env-migration-snapshot.ts"; import { isRecord } from "./guards.ts"; @@ -81,3 +82,40 @@ export function sameMigrationFile( left.size === right.size ); } + +/** Admit metadata storage before copying inputs, shared by journals and snapshots. */ +export async function assertMigrationStorageOutsideWorkingTrees( + path: string, + project: { readonly projectRoot: string } +): Promise { + if (!isAbsolute(path)) { + throw new ClosedCliError( + "Env migration requires private journal and snapshot storage outside Git working trees. Choose a HACK_HOME outside your repositories; preview remains read-only." + ); + } + const inside = relative(project.projectRoot, path); + if ( + !(inside === ".." || inside.startsWith(`..${sep}`) || isAbsolute(inside)) + ) { + throw new ClosedCliError( + "Env migration requires private journal and snapshot storage outside Git working trees. Choose a HACK_HOME outside your repositories; preview remains read-only." + ); + } + for (let parent = dirname(path); ; parent = dirname(parent)) { + for (const name of [".git"]) { + try { + await lstat(join(parent, name)); + throw new ClosedCliError( + "Env migration requires private journal and snapshot storage outside Git working trees. Choose a HACK_HOME outside your repositories; preview remains read-only." + ); + } catch (error) { + if (!(isRecord(error) && error.code === "ENOENT")) { + throw error; + } + } + } + if (dirname(parent) === parent) { + break; + } + } +} diff --git a/src/lib/env-migration-snapshot.ts b/src/lib/env-migration-snapshot.ts index 4b22c54f1..8148f022c 100644 --- a/src/lib/env-migration-snapshot.ts +++ b/src/lib/env-migration-snapshot.ts @@ -1,9 +1,10 @@ import { randomUUID } from "node:crypto"; import type { Stats } from "node:fs"; -import { link, lstat, open, stat, unlink } from "node:fs/promises"; -import { dirname, join } from "node:path"; +import { link, lstat, open, realpath, stat, unlink } from "node:fs/promises"; +import { dirname, join, relative, resolve } from "node:path"; import { ClosedCliError } from "./cli-result.ts"; import { + assertMigrationStorageOutsideWorkingTrees, migrationRefusal, readMigrationFile, sameMigrationFile, @@ -11,6 +12,7 @@ import { import { isRecord } from "./guards.ts"; import { createUpgradeSnapshot, + defaultUpgradeSnapshotRoot, readUpgradeSnapshot, recordUpgradeSnapshotAfter, restoreUpgradeSnapshot, @@ -57,6 +59,7 @@ export type EnvMigrationSnapshots = { type SnapshotLibrary = { readonly createUpgradeSnapshot: typeof createUpgradeSnapshot; + readonly defaultUpgradeSnapshotRoot: typeof defaultUpgradeSnapshotRoot; readonly readUpgradeSnapshot: typeof readUpgradeSnapshot; readonly recordUpgradeSnapshotAfter: typeof recordUpgradeSnapshotAfter; readonly restoreUpgradeSnapshot: typeof restoreUpgradeSnapshot; @@ -66,6 +69,7 @@ type SnapshotLibrary = { export function envMigrationSnapshots(): EnvMigrationSnapshots { return bindEnvMigrationSnapshots({ createUpgradeSnapshot, + defaultUpgradeSnapshotRoot, readUpgradeSnapshot, recordUpgradeSnapshotAfter, restoreUpgradeSnapshot, @@ -97,7 +101,7 @@ async function probeRestoreLink(input: { }): Promise { const name = `.hack-env-restore-probe-${randomUUID()}`; const source = join(input.snapshotDir, name); - const destination = join(input.parent, name); + const destination = join(input.parent, `${name}.link`); let identity: Stats | null = null; let linked = false; try { @@ -126,6 +130,25 @@ async function probeRestoreLink(input: { } } +async function existingRestoreDirectory(path: string): Promise { + let parent = path; + for (;;) { + const info = await stat(parent).catch((error: unknown) => { + if (isRecord(error) && error.code === "ENOENT") { + return null; + } + throw migrationRefusal(); + }); + if (info) { + if (!info.isDirectory()) { + throw restoreFilesystemRefusal(); + } + return parent; + } + parent = dirname(parent); + } +} + /** Device numbers alone miss distinct bind mounts on the same filesystem. */ async function assertRestoreFilesystem( snapshotDir: string, @@ -221,7 +244,27 @@ export function bindEnvMigrationSnapshots( }; return { capture: async ({ projectRoot, files }) => { + const root = resolve(library.defaultUpgradeSnapshotRoot()); + await assertMigrationStorageOutsideWorkingTrees( + join(root, "manifest.json"), + { + projectRoot, + } + ); + const existing = await existingRestoreDirectory(root); + const canonical = resolve( + await realpath(existing), + relative(existing, root) + ); + await assertMigrationStorageOutsideWorkingTrees( + join(canonical, "manifest.json"), + { projectRoot } + ); + // Refuse ordinary storage failures before the provider copies any input + // bytes. The completed snapshot is checked again by assertCoverage. + await assertRestoreFilesystem(existing, files, linkRestoreProbe); const result = await library.createUpgradeSnapshot({ + root, purpose: "env-migration", targets: files.map((file) => ({ path: file.path, diff --git a/src/lib/env-migration.ts b/src/lib/env-migration.ts index c21175bb8..b69fbaa1e 100644 --- a/src/lib/env-migration.ts +++ b/src/lib/env-migration.ts @@ -1,16 +1,9 @@ import { lstat, realpath } from "node:fs/promises"; -import { - basename, - dirname, - isAbsolute, - join, - relative, - resolve, - sep, -} from "node:path"; +import { basename, dirname, isAbsolute, join, resolve } from "node:path"; import { ClosedCliError } from "./cli-result.ts"; import { withEnvLayerLock } from "./env-layer-lock.ts"; import { + assertMigrationStorageOutsideWorkingTrees as assertOutsideWorkingTrees, migrationDigest as digest, readMigrationFile as readState, migrationRefusal as refuse, @@ -173,42 +166,6 @@ export async function assertEnvMigrationFinished(input: { await assertEnvMigrationReadable(input); } -async function assertOutsideWorkingTrees( - path: string, - project: EnvMigrationProject -): Promise { - if (!isAbsolute(path)) { - throw new ClosedCliError( - "Env migration requires private journal and snapshot storage outside Git working trees. Choose a HACK_HOME outside your repositories; preview remains read-only." - ); - } - const inside = relative(project.projectRoot, path); - if ( - !(inside === ".." || inside.startsWith(`..${sep}`) || isAbsolute(inside)) - ) { - throw new ClosedCliError( - "Env migration requires private journal and snapshot storage outside Git working trees. Choose a HACK_HOME outside your repositories; preview remains read-only." - ); - } - for (let parent = dirname(path); ; parent = dirname(parent)) { - for (const name of [".git"]) { - try { - await lstat(join(parent, name)); - throw new ClosedCliError( - "Env migration requires private journal and snapshot storage outside Git working trees. Choose a HACK_HOME outside your repositories; preview remains read-only." - ); - } catch (error) { - if (!(isRecord(error) && error.code === "ENOENT")) { - throw error; - } - } - } - if (dirname(parent) === parent) { - break; - } - } -} - async function privateDirectory(path: string): Promise { await ensurePrivateDirectory(path); await checkPrivateDirectory(path); diff --git a/src/lib/project-env-config.ts b/src/lib/project-env-config.ts index 8ddec7423..0c79ea7e6 100644 --- a/src/lib/project-env-config.ts +++ b/src/lib/project-env-config.ts @@ -6114,12 +6114,28 @@ function removeLegacyComposeEnvFileEntries(opts: { return kept.length > 0 ? kept : undefined; } +function assertLegacyEnvMigrationOverlayNames( + overlayNames: readonly string[] +): void { + if (overlayNames.includes("default")) { + throw new ClosedCliError( + "Refusing legacy env migration: .env.default would overwrite the base hack.env.default.yaml layer. Rename the legacy default overlay and update its project references before retrying. No env migration files were written." + ); + } + if (overlayNames.includes("local")) { + throw new ClosedCliError( + "Refusing legacy env migration: .env.local would collide with the machine-local hack.env.local.yaml layer. Rename the legacy local overlay and update its project references before retrying. No env migration files were written." + ); + } +} + /** Reject aliases between tracked and effective local destinations before any writes. */ async function preflightLegacyEnvMigrationPaths(opts: { readonly projectRoot: string; readonly projectDir: string; readonly overlayNames: readonly string[]; }): Promise { + assertLegacyEnvMigrationOverlayNames(opts.overlayNames); const destinations = new Set(); const localPaths: string[] = []; for (const envName of [null, ...opts.overlayNames]) { @@ -7129,16 +7145,7 @@ async function migrateLegacyProjectEnvWithSources(opts: { overlays: overlaysResolved, }); - if (overlayNames.includes("default")) { - throw new ClosedCliError( - "Refusing legacy env migration: .env.default would overwrite the base hack.env.default.yaml layer. Rename the legacy default overlay and update its project references before retrying. No env migration files were written." - ); - } - if (overlayNames.includes("local")) { - throw new ClosedCliError( - "Refusing legacy env migration: .env.local would collide with the machine-local hack.env.local.yaml layer. Rename the legacy local overlay and update its project references before retrying. No env migration files were written." - ); - } + assertLegacyEnvMigrationOverlayNames(overlayNames); const localPaths = await preflightLegacyEnvMigrationPaths({ projectRoot: opts.projectRoot, projectDir: opts.projectDir, diff --git a/tests/env-migration-provider.test.ts b/tests/env-migration-provider.test.ts index c5e1b04cb..51dc295bb 100644 --- a/tests/env-migration-provider.test.ts +++ b/tests/env-migration-provider.test.ts @@ -260,8 +260,7 @@ test("apply refuses cross-filesystem snapshot storage before any project publica const before = await readFile(join(project.projectDir, ".env")); const snapshots = bindEnvMigrationSnapshots({ ...snapshotLibrary, - createUpgradeSnapshot: (input) => - snapshotLibrary.createUpgradeSnapshot({ ...input, root: snapshotRoot }), + defaultUpgradeSnapshotRoot: () => snapshotRoot, }); await expect( applyEnvMigration({ project, snapshots, acceptTrackedChanges: true }) @@ -275,6 +274,7 @@ test("apply refuses cross-filesystem snapshot storage before any project publica expect( await Bun.file(join(project.projectRoot, ".hack.secret.key")).exists() ).toBe(false); + expect(await readdir(snapshotRoot)).toEqual([]); } finally { await rm(snapshotRoot, { recursive: true, force: true }); } @@ -287,23 +287,34 @@ for (const code of ["EXDEV", "EPERM"]) { await readdir(project.projectRoot, { recursive: true }) ).sort(); let probed = false; - const snapshots = bindEnvMigrationSnapshots(snapshotLibrary, { - linkRestoreProbe: async (source, destination) => { - // Unlike the second-filesystem case, admission reached the real link - // boundary after proving matching devices and creating an empty probe. - expect((await lstat(source)).dev).toBe( - (await lstat(project.projectRoot)).dev - ); - expect((await readFile(source)).length).toBe(0); - expect(String(destination)).toContain(".hack-env-restore-probe-"); - probed = true; - throw Object.assign(new Error("synthetic link refusal"), { code }); + let captures = 0; + const snapshots = bindEnvMigrationSnapshots( + { + ...snapshotLibrary, + createUpgradeSnapshot: (input) => { + captures++; + return snapshotLibrary.createUpgradeSnapshot(input); + }, }, - }); + { + linkRestoreProbe: async (source, destination) => { + // Unlike the second-filesystem case, admission reached the real link + // boundary after proving matching devices and creating an empty probe. + expect((await lstat(source)).dev).toBe( + (await lstat(project.projectRoot)).dev + ); + expect((await readFile(source)).length).toBe(0); + expect(String(destination)).toContain(".hack-env-restore-probe-"); + probed = true; + throw Object.assign(new Error("synthetic link refusal"), { code }); + }, + } + ); await expect( applyEnvMigration({ project, snapshots, acceptTrackedChanges: true }) ).rejects.toThrow("same filesystem and mount"); expect(probed).toBe(true); + expect(captures).toBe(0); expect( (await readdir(project.projectRoot, { recursive: true })).sort() ).toEqual(names); @@ -315,5 +326,32 @@ for (const code of ["EXDEV", "EPERM"]) { (name) => name.includes(".hack-env-restore-probe-") ) ).toBe(false); + expect(await snapshotLibrary.listUpgradeSnapshots()).toEqual([]); }); } + +test("snapshot storage inside the project refuses before copying any originals", async () => { + const project = await fixture(); + const names = ( + await readdir(project.projectRoot, { recursive: true }) + ).sort(); + let captures = 0; + const snapshots = bindEnvMigrationSnapshots({ + ...snapshotLibrary, + defaultUpgradeSnapshotRoot: () => join(project.projectRoot, "snapshots"), + createUpgradeSnapshot: (input) => { + captures++; + return snapshotLibrary.createUpgradeSnapshot(input); + }, + }); + await expect( + applyEnvMigration({ project, snapshots, acceptTrackedChanges: true }) + ).rejects.toThrow("outside Git working trees"); + expect(captures).toBe(0); + expect( + (await readdir(project.projectRoot, { recursive: true })).sort() + ).toEqual(names); + expect(await readFile(join(project.projectDir, ".env"), "utf8")).toBe( + `TOKEN=${sentinel}\n` + ); +}); From 920d98168522a5ad2f258e6fb8238c25a85d6d33 Mon Sep 17 00:00:00 2001 From: Kit <344842+roodboi@users.noreply.github.com> Date: Sat, 14 Nov 2026 03:40:45 +0000 Subject: [PATCH 11/32] fix(env): refuse tracked absent keys at migration publication --- docs/reference/env-migration.md | 13 +++- src/lib/env-migration.ts | 7 ++ src/lib/project-env-config.ts | 8 +- tests/env-migration-journal.test.ts | 111 ++++++++++++++++++++++++++++ 4 files changed, 131 insertions(+), 8 deletions(-) diff --git a/docs/reference/env-migration.md b/docs/reference/env-migration.md index b8126934e..4a58c9b87 100644 --- a/docs/reference/env-migration.md +++ b/docs/reference/env-migration.md @@ -73,8 +73,8 @@ Hack does not bypass that prompt or include the values in the preview. ## Apply and recovery -Apply takes a private snapshot of originals and absent destinations before any -project write. The journal and staged payloads live outside working trees under +Apply takes a private snapshot of originals and absent destinations before +publishing any migration target. The journal and staged payloads live outside working trees under Hack's private data directory. The snapshot manifest and journal hold only names, paths, statuses, modes, sizes and SHA-256 checksums. Original bytes and staged payloads are separate private files; they can contain values and keys and must @@ -90,12 +90,17 @@ all legacy files. Doctor can also perform its existing hygiene repairs before migration, such as adding ignore rules or tightening permissions; those separate doctor repairs are outside the env migration journal. Use `hack env migrate --apply` when the preview and rollback must cover only the migration's writes. -Implicit migrations from env, session and startup commands also -keep legacy files and resume an unfinished journal; they preserve existing +Implicit migrations from env, session and startup commands also use this journal +and keep legacy files; they preserve existing Compose `env_file` references. Implicit commands refuse an unfinished apply or rollback journal before using partially published layers. Resume the explicit apply or rollback command first. +Migration refuses a Git-tracked project key path even when its file is absent +from the worktree. Tracked-change consent does not allow publishing key bytes at +that path. Git tracking is checked again under the key lock before publication; +keep the key out of the index before resuming a refused migration. + Completed publication payloads are removed; retries also sweep receipted orphan staging directories. Unknown or edited entries are retained, reported as `payloadCleanup: "kept"` in the apply result. Original snapshot blobs remain for diff --git a/src/lib/env-migration.ts b/src/lib/env-migration.ts index b69fbaa1e..39af5be7e 100644 --- a/src/lib/env-migration.ts +++ b/src/lib/env-migration.ts @@ -35,6 +35,7 @@ import { import { acquireLegacyEnvCleanupContext, assertMachineLocalFileGitPrivate, + assertProjectEnvKeyUntracked, assertProjectEnvLayerPaths, isGitTrackedOrUnknownProjectPath, LegacyEnvCleanupError, @@ -597,6 +598,9 @@ async function publishEntry( entry: Publication ): Promise { const run = async () => { + if (entry.kind === "key") { + await assertProjectEnvKeyUntracked({ keyPath: entry.path }); + } await checkPrivateDirectory(dirname(entry.payload)); const current = await readState(entry.path); if (same(current.state, entry)) { @@ -636,6 +640,9 @@ async function publishEntry( mode: entry.mode ?? 0o600, refusal: refuse, beforeRename: async () => { + if (entry.kind === "key") { + await assertProjectEnvKeyUntracked({ keyPath: entry.path }); + } if (!same((await readState(entry.path)).state, entry.before)) { throw refuse(); } diff --git a/src/lib/project-env-config.ts b/src/lib/project-env-config.ts index 0c79ea7e6..5eedeef6d 100644 --- a/src/lib/project-env-config.ts +++ b/src/lib/project-env-config.ts @@ -5309,8 +5309,10 @@ function projectEnvTrackedKeyError(): ProjectEnvKeyRefusalError { * stays allowed for v4 compatibility; `hack doctor` reports the tracked key. * An undeterminable status (git unavailable, path outside a work tree) is not * treated as tracked. + * The index is checked even if the path is absent. Key publishers must call + * this again under the key owner lock immediately before publication. */ -async function assertProjectEnvKeyUntracked(opts: { +export async function assertProjectEnvKeyUntracked(opts: { readonly keyPath: string; }): Promise { const tracked = await isGitTrackedProjectEnvPath({ @@ -6421,9 +6423,7 @@ async function prepareContractMigrationKey( } const keyPath = existingPath ?? shared ?? localPath; extraSources.add(keyPath); - if (existingPath) { - await assertProjectEnvKeyUntracked({ keyPath }); - } + await assertProjectEnvKeyUntracked({ keyPath }); const keyText = existingPath ? await readLegacyProjectEnvKeyFile({ path: keyPath }) : randomBytes(32).toString("base64url"); diff --git a/tests/env-migration-journal.test.ts b/tests/env-migration-journal.test.ts index 35182def7..8978e9789 100644 --- a/tests/env-migration-journal.test.ts +++ b/tests/env-migration-journal.test.ts @@ -29,9 +29,11 @@ import { resolveProjectEnvConfigForNativeSelection, resolveProjectEnvMetadata, resolveProjectEnvMetadataForNativeSelection, + resolveProjectEnvSharedKeyLocation, resolveProjectEnvSharedKeyPath, setProjectEnvValue, } from "../src/lib/project-env-config.ts"; +import { withProjectsRegistryLock } from "../src/lib/projects-registry-lock.ts"; import { resolveSecretStore } from "../src/lib/secret-store.ts"; import { restoreEnv } from "./helpers/env.ts"; import { fixtureEnvSnapshots } from "./helpers/env-migration-fixture.ts"; @@ -598,6 +600,115 @@ async function addLegacySecret() { await store.set({ key: "SEC", value: SENTINEL }); } +async function migrationFixtureGit(args: readonly string[]): Promise { + const result = await runFixtureGit({ cwd: project.projectRoot, args }); + expect(result.exitCode, result.stderr).toBe(0); +} + +test("secret migration refuses a git-tracked absent project key before planning or writes", async () => { + await addLegacySecret(); + await migrationFixtureGit(["init", "--quiet"]); + const keyPath = join(project.projectRoot, ".hack.secret.key"); + await writeFile(keyPath, "synthetic indexed placeholder\n"); + await migrationFixtureGit(["add", "--force", "--", ".hack.secret.key"]); + await rm(keyPath); + const before = await projectFiles(); + await expect(previewEnvMigration(project)).rejects.toThrow("git-tracked"); + await expect( + applyEnvMigration({ + project, + snapshots: fixtureEnvSnapshots(root), + acceptTrackedChanges: true, + }) + ).rejects.toThrow("git-tracked"); + expect(await projectFiles()).toEqual(before); + expect(await Bun.file(keyPath).exists()).toBe(false); + expect( + await Bun.file(envMigrationJournalPath(project.projectRoot)).exists() + ).toBe(false); +}); + +test("key publication rechecks Git after waiting for its lock and can resume after untracking", async () => { + await addLegacySecret(); + await migrationFixtureGit(["init", "--quiet"]); + const keyPath = join(project.projectRoot, ".hack.secret.key"); + const preview = await previewEnvMigration(project); + const keyIndex = preview.files.findIndex((file) => file.path === keyPath); + expect(keyIndex).toBeGreaterThanOrEqual(0); + const location = await resolveProjectEnvSharedKeyLocation(project); + if (!location) { + throw new Error("Fixture expected a shared Git key lock"); + } + const entered = Promise.withResolvers(); + const release = Promise.withResolvers(); + const reachedKey = Promise.withResolvers(); + const holder = withProjectsRegistryLock({ + lockPath: `${location.path}.lock`, + run: async () => { + entered.resolve(); + await release.promise; + }, + }); + await entered.promise; + const snapshots = fixtureEnvSnapshots(root); + const applying = applyEnvMigration({ + project, + snapshots, + acceptTrackedChanges: true, + hooks: { + boundary: async ({ phase, index }) => { + if (phase === "before_publish" && index === keyIndex) { + reachedKey.resolve(); + } + }, + }, + }).then( + () => ({ kind: "unexpected-success" as const }), + (error: unknown) => ({ kind: "refused" as const, error }) + ); + try { + await Promise.race([ + reachedKey.promise, + applying.then(() => { + throw new Error("Migration settled before reaching key publication"); + }), + ]); + // The worktree returns to the planned absent state, but the index now + // tracks that name while the actual key owner lock is still held. + await writeFile(keyPath, "synthetic indexed placeholder\n"); + await migrationFixtureGit(["add", "--force", "--", ".hack.secret.key"]); + await rm(keyPath); + } finally { + release.resolve(); + await holder; + await applying; + } + const result = await applying; + if (!(result.kind === "refused" && result.error instanceof Error)) { + throw new Error("Expected a tracked-key refusal after lock acquisition"); + } + expect(result.error.message).toContain("git-tracked"); + expect(result.error.message).not.toContain(SENTINEL); + expect(await Bun.file(keyPath).exists()).toBe(false); + expect( + await Bun.file(join(project.projectDir, "hack.env.default.yaml")).exists() + ).toBe(false); + await migrationFixtureGit([ + "rm", + "--cached", + "--force", + "--", + ".hack.secret.key", + ]); + expect( + (await apply({ project, snapshots, acceptTrackedChanges: true })).status + ).toBe("applied"); + expect( + (await resolveProjectEnvConfig({ ...project, envName: null }))?.serviceEnv + .api?.SEC + ).toBe(SENTINEL); +}, 20_000); + test("fresh delivery refusal before unlink keeps migration complete and cleanup retryable", async () => { await addLegacySecret(); const snapshots = fixtureEnvSnapshots(root); From 62224b75f6419b1cd90e783f101c7f542fab4344 Mon Sep 17 00:00:00 2001 From: Kit <344842+roodboi@users.noreply.github.com> Date: Sat, 14 Nov 2026 03:53:40 +0000 Subject: [PATCH 12/32] fix(env): fail incomplete migration rollback commands --- docs/reference/env-migration.md | 8 +++- src/commands/env-migrate.ts | 9 ++-- tests/env-migration-provider.test.ts | 65 ++++++++++++++++++++++++++-- 3 files changed, 74 insertions(+), 8 deletions(-) diff --git a/docs/reference/env-migration.md b/docs/reference/env-migration.md index 4a58c9b87..07a5afd67 100644 --- a/docs/reference/env-migration.md +++ b/docs/reference/env-migration.md @@ -160,8 +160,12 @@ root key ignore entry is retained with a checkout-local key. The rollback result names these paths in `retained`; key removal is not part of rollback. Ordinary rollback refuses later edits. Explicit `--preserve-changes` restores the originals and keeps replaced bytes privately in the snapshot instead; it does not -select or merge values. Rollback can resume an incomplete restoration. Keep the manifest, blobs and journal together -until the upgrade is accepted. Neither file-level atomic writes nor cooperative +select or merge values. Rollback can resume an incomplete restoration. An +incomplete rollback exits 1 in both JSON and plain modes, preserving its per-file +restoration details and leaving the journal in `rolling_back`. Project commands +remain blocked until the same rollback completes; a restored result exits 0. +Keep the manifest, blobs and journal together until the upgrade is accepted. +Neither file-level atomic writes nor cooperative locks protect against an unrelated process replacing files outside Hack's locks. ### Recovering retained edits diff --git a/src/commands/env-migrate.ts b/src/commands/env-migrate.ts index bf0bb0d8f..e1f430619 100644 --- a/src/commands/env-migrate.ts +++ b/src/commands/env-migrate.ts @@ -129,13 +129,16 @@ export async function handleEnvMigrate({ action = "rollback"; } let data: unknown; + let exitCode = 0; if (opts.rollbackManifest) { - data = await rollbackEnvMigration({ + const result = await rollbackEnvMigration({ project, manifestPath: resolve(ctx.cwd, opts.rollbackManifest), snapshots: await envMigrationSnapshots(), preserveChanges: opts.preserveChanges === true, }); + data = result; + exitCode = result.status === "restored" ? 0 : 1; } else if (opts.apply) { data = await (opts.cleanup ? cleanupEnvMigration : applyEnvMigration)({ project, @@ -155,10 +158,10 @@ export async function handleEnvMigrate({ } if (opts.json) { emitCliResult({ result: okResult({ data }) }); - return 0; + return exitCode; } process.stdout.write(`${JSON.stringify(data, null, 2)}\n`); - return 0; + return exitCode; } export const envMigrateCommand = withHandler(spec, handleEnvMigrate); diff --git a/tests/env-migration-provider.test.ts b/tests/env-migration-provider.test.ts index 51dc295bb..d1a391862 100644 --- a/tests/env-migration-provider.test.ts +++ b/tests/env-migration-provider.test.ts @@ -10,7 +10,7 @@ import { writeFile, } from "node:fs/promises"; import { tmpdir } from "node:os"; -import { isAbsolute, join, resolve } from "node:path"; +import { dirname, isAbsolute, join, resolve } from "node:path"; import { applyEnvMigration } from "../src/lib/env-migration.ts"; import { bindEnvMigrationSnapshots } from "../src/lib/env-migration-snapshot.ts"; import { isRecord } from "../src/lib/guards.ts"; @@ -100,7 +100,11 @@ async function fixture() { return project; } -async function cli(projectRoot: string, args: readonly string[]) { +async function runMigrationCli( + projectRoot: string, + args: readonly string[], + { json = true }: { readonly json?: boolean } = {} +) { if (compiled && !isAbsolute(compiled)) { throw new Error("Compiled test executable must be absolute"); } @@ -118,7 +122,7 @@ async function cli(projectRoot: string, args: readonly string[]) { "migrate", "--path", projectRoot, - "--json", + ...(json ? ["--json"] : []), ...args, ], cwd: projectRoot, @@ -134,6 +138,11 @@ async function cli(projectRoot: string, args: readonly string[]) { }), }); expect(result.stdout + result.stderr).not.toContain(sentinel); + return result; +} + +async function cli(projectRoot: string, args: readonly string[]) { + const result = await runMigrationCli(projectRoot, args); const raw: unknown = JSON.parse(result.stdout); if (!(isRecord(raw) && typeof raw.ok === "boolean")) { throw new Error("Expected CLI result envelope"); @@ -213,6 +222,56 @@ test("production snapshot provider backs source CLI apply, no-op and guarded rol ); }); +for (const json of [true, false]) { + test(`incomplete production rollback exits 1 with details (${json ? "JSON" : "plain"})`, async () => { + const project = await fixture(); + const applied = await cli(project.projectRoot, [ + "--apply", + "--accept-tracked-env-changes", + ]); + expect(applied.exitCode, applied.stderr).toBe(0); + const manifestPath = pathField(applied.envelope.data, "manifestPath"); + const journalPath = pathField(applied.envelope.data, "journalPath"); + // A real malformed provider work directory refuses restoration on every + // platform, including root. No permission-based skip or mock is needed. + const blocked = join(dirname(manifestPath), "staging"); + const marker = "synthetic restore-work-directory blocker"; + await writeFile(blocked, marker, { mode: 0o600 }); + const incomplete = await runMigrationCli( + project.projectRoot, + ["--rollback-manifest", manifestPath], + { json } + ); + expect(incomplete.exitCode).toBe(1); + expect(incomplete.stdout + incomplete.stderr).not.toContain(marker); + const raw: unknown = JSON.parse(incomplete.stdout); + const data = json && isRecord(raw) ? raw.data : raw; + if (!isRecord(data)) { + throw new Error("Expected rollback restoration details"); + } + expect(data.status).toBe("incomplete"); + expect(data.results).toContainEqual( + expect.objectContaining({ status: "error" }) + ); + expect(JSON.parse(await readFile(journalPath, "utf8")).state).toBe( + "rolling_back" + ); + await expect( + resolveProjectEnvConfig({ ...project, envName: null }) + ).rejects.toThrow("unfinished"); + await rm(blocked); + const restored = await cli(project.projectRoot, [ + "--rollback-manifest", + manifestPath, + ]); + expect(restored.exitCode, restored.stderr).toBe(0); + expect(restored.envelope.data.status).toBe("restored"); + expect(await readFile(join(project.projectDir, ".env"), "utf8")).toBe( + `TOKEN=${sentinel}\n` + ); + }); +} + test("production rollback maps preserved edits to retrievable private files without printing them", async () => { const project = await fixture(); const applied = await cli(project.projectRoot, [ From d43cee8c26f6360c3ecf6da1cd0256b01a0112c9 Mon Sep 17 00:00:00 2001 From: Kit <344842+roodboi@users.noreply.github.com> Date: Sat, 14 Nov 2026 03:57:27 +0000 Subject: [PATCH 13/32] fix(env): reject case-folded tracked project keys --- docs/reference/env-migration.md | 5 +- src/lib/project-env-config.ts | 7 +- tests/env-migration-journal.test.ts | 202 ++++++++++++++-------------- 3 files changed, 109 insertions(+), 105 deletions(-) diff --git a/docs/reference/env-migration.md b/docs/reference/env-migration.md index 07a5afd67..8b6c688fe 100644 --- a/docs/reference/env-migration.md +++ b/docs/reference/env-migration.md @@ -97,8 +97,9 @@ Implicit commands refuse an unfinished apply or rollback journal before using partially published layers. Resume the explicit apply or rollback command first. Migration refuses a Git-tracked project key path even when its file is absent -from the worktree. Tracked-change consent does not allow publishing key bytes at -that path. Git tracking is checked again under the key lock before publication; +from the worktree, including case-folded aliases such as `.Hack.Secret.Key`. +Tracked-change consent does not allow publishing key bytes at that path. +Git tracking is checked again under the key lock before publication; keep the key out of the index before resuming a refused migration. Completed publication payloads are removed; retries also sweep receipted orphan diff --git a/src/lib/project-env-config.ts b/src/lib/project-env-config.ts index 5eedeef6d..d16a36a6e 100644 --- a/src/lib/project-env-config.ts +++ b/src/lib/project-env-config.ts @@ -5309,13 +5309,14 @@ function projectEnvTrackedKeyError(): ProjectEnvKeyRefusalError { * stays allowed for v4 compatibility; `hack doctor` reports the tracked key. * An undeterminable status (git unavailable, path outside a work tree) is not * treated as tracked. - * The index is checked even if the path is absent. Key publishers must call - * this again under the key owner lock immediately before publication. + * The index is checked ignoring filename case even if the path is absent, + * protecting portable checkouts whose filesystem folds case. Key publishers + * must call this again under the key owner lock before publication. */ export async function assertProjectEnvKeyUntracked(opts: { readonly keyPath: string; }): Promise { - const tracked = await isGitTrackedProjectEnvPath({ + const tracked = await isGitTrackedProjectEnvPathAnyCase({ projectRoot: dirname(opts.keyPath), path: opts.keyPath, }); diff --git a/tests/env-migration-journal.test.ts b/tests/env-migration-journal.test.ts index 8978e9789..f3d7977ea 100644 --- a/tests/env-migration-journal.test.ts +++ b/tests/env-migration-journal.test.ts @@ -605,109 +605,111 @@ async function migrationFixtureGit(args: readonly string[]): Promise { expect(result.exitCode, result.stderr).toBe(0); } -test("secret migration refuses a git-tracked absent project key before planning or writes", async () => { - await addLegacySecret(); - await migrationFixtureGit(["init", "--quiet"]); - const keyPath = join(project.projectRoot, ".hack.secret.key"); - await writeFile(keyPath, "synthetic indexed placeholder\n"); - await migrationFixtureGit(["add", "--force", "--", ".hack.secret.key"]); - await rm(keyPath); - const before = await projectFiles(); - await expect(previewEnvMigration(project)).rejects.toThrow("git-tracked"); - await expect( - applyEnvMigration({ +for (const trackedName of [".hack.secret.key", ".Hack.Secret.Key"]) { + test(`secret migration refuses an absent key tracked as ${trackedName} before planning or writes`, async () => { + await addLegacySecret(); + await migrationFixtureGit(["init", "--quiet"]); + const keyPath = join(project.projectRoot, ".hack.secret.key"); + await writeFile( + join(project.projectRoot, trackedName), + "synthetic indexed placeholder\n" + ); + await migrationFixtureGit(["add", "--force", "--", trackedName]); + await rm(join(project.projectRoot, trackedName)); + const before = await projectFiles(); + await expect(previewEnvMigration(project)).rejects.toThrow("git-tracked"); + await expect( + applyEnvMigration({ + project, + snapshots: fixtureEnvSnapshots(root), + acceptTrackedChanges: true, + }) + ).rejects.toThrow("git-tracked"); + expect(await projectFiles()).toEqual(before); + expect(await Bun.file(keyPath).exists()).toBe(false); + expect( + await Bun.file(envMigrationJournalPath(project.projectRoot)).exists() + ).toBe(false); + }); + + test(`key publication rechecks ${trackedName} tracking after waiting for its lock and can resume after untracking`, async () => { + await addLegacySecret(); + await migrationFixtureGit(["init", "--quiet"]); + const keyPath = join(project.projectRoot, ".hack.secret.key"); + const preview = await previewEnvMigration(project); + const keyIndex = preview.files.findIndex((file) => file.path === keyPath); + expect(keyIndex).toBeGreaterThanOrEqual(0); + const location = await resolveProjectEnvSharedKeyLocation(project); + if (!location) { + throw new Error("Fixture expected a shared Git key lock"); + } + const entered = Promise.withResolvers(); + const release = Promise.withResolvers(); + const reachedKey = Promise.withResolvers(); + const holder = withProjectsRegistryLock({ + lockPath: `${location.path}.lock`, + run: async () => { + entered.resolve(); + await release.promise; + }, + }); + await entered.promise; + const snapshots = fixtureEnvSnapshots(root); + const applying = applyEnvMigration({ project, - snapshots: fixtureEnvSnapshots(root), + snapshots, acceptTrackedChanges: true, - }) - ).rejects.toThrow("git-tracked"); - expect(await projectFiles()).toEqual(before); - expect(await Bun.file(keyPath).exists()).toBe(false); - expect( - await Bun.file(envMigrationJournalPath(project.projectRoot)).exists() - ).toBe(false); -}); - -test("key publication rechecks Git after waiting for its lock and can resume after untracking", async () => { - await addLegacySecret(); - await migrationFixtureGit(["init", "--quiet"]); - const keyPath = join(project.projectRoot, ".hack.secret.key"); - const preview = await previewEnvMigration(project); - const keyIndex = preview.files.findIndex((file) => file.path === keyPath); - expect(keyIndex).toBeGreaterThanOrEqual(0); - const location = await resolveProjectEnvSharedKeyLocation(project); - if (!location) { - throw new Error("Fixture expected a shared Git key lock"); - } - const entered = Promise.withResolvers(); - const release = Promise.withResolvers(); - const reachedKey = Promise.withResolvers(); - const holder = withProjectsRegistryLock({ - lockPath: `${location.path}.lock`, - run: async () => { - entered.resolve(); - await release.promise; - }, - }); - await entered.promise; - const snapshots = fixtureEnvSnapshots(root); - const applying = applyEnvMigration({ - project, - snapshots, - acceptTrackedChanges: true, - hooks: { - boundary: async ({ phase, index }) => { - if (phase === "before_publish" && index === keyIndex) { - reachedKey.resolve(); - } + hooks: { + boundary: async ({ phase, index }) => { + if (phase === "before_publish" && index === keyIndex) { + reachedKey.resolve(); + } + }, }, - }, - }).then( - () => ({ kind: "unexpected-success" as const }), - (error: unknown) => ({ kind: "refused" as const, error }) - ); - try { - await Promise.race([ - reachedKey.promise, - applying.then(() => { - throw new Error("Migration settled before reaching key publication"); - }), - ]); - // The worktree returns to the planned absent state, but the index now - // tracks that name while the actual key owner lock is still held. - await writeFile(keyPath, "synthetic indexed placeholder\n"); - await migrationFixtureGit(["add", "--force", "--", ".hack.secret.key"]); - await rm(keyPath); - } finally { - release.resolve(); - await holder; - await applying; - } - const result = await applying; - if (!(result.kind === "refused" && result.error instanceof Error)) { - throw new Error("Expected a tracked-key refusal after lock acquisition"); - } - expect(result.error.message).toContain("git-tracked"); - expect(result.error.message).not.toContain(SENTINEL); - expect(await Bun.file(keyPath).exists()).toBe(false); - expect( - await Bun.file(join(project.projectDir, "hack.env.default.yaml")).exists() - ).toBe(false); - await migrationFixtureGit([ - "rm", - "--cached", - "--force", - "--", - ".hack.secret.key", - ]); - expect( - (await apply({ project, snapshots, acceptTrackedChanges: true })).status - ).toBe("applied"); - expect( - (await resolveProjectEnvConfig({ ...project, envName: null }))?.serviceEnv - .api?.SEC - ).toBe(SENTINEL); -}, 20_000); + }).then( + () => ({ kind: "unexpected-success" as const }), + (error: unknown) => ({ kind: "refused" as const, error }) + ); + try { + await Promise.race([ + reachedKey.promise, + applying.then(() => { + throw new Error("Migration settled before reaching key publication"); + }), + ]); + // The worktree returns to the planned absent state, but the index now + // tracks that name while the actual key owner lock is still held. + await writeFile( + join(project.projectRoot, trackedName), + "synthetic indexed placeholder\n" + ); + await migrationFixtureGit(["add", "--force", "--", trackedName]); + await rm(join(project.projectRoot, trackedName)); + } finally { + release.resolve(); + await holder; + await applying; + } + const result = await applying; + if (!(result.kind === "refused" && result.error instanceof Error)) { + throw new Error("Expected a tracked-key refusal after lock acquisition"); + } + expect(result.error.message).toContain("git-tracked"); + expect(result.error.message).not.toContain(SENTINEL); + expect(await Bun.file(keyPath).exists()).toBe(false); + expect( + await Bun.file(join(project.projectDir, "hack.env.default.yaml")).exists() + ).toBe(false); + await migrationFixtureGit(["rm", "--cached", "--force", "--", trackedName]); + expect( + (await apply({ project, snapshots, acceptTrackedChanges: true })).status + ).toBe("applied"); + expect( + (await resolveProjectEnvConfig({ ...project, envName: null }))?.serviceEnv + .api?.SEC + ).toBe(SENTINEL); + }, 20_000); +} test("fresh delivery refusal before unlink keeps migration complete and cleanup retryable", async () => { await addLegacySecret(); From eb704986515fe2cf32b6413b543a7cdd6d47e3d8 Mon Sep 17 00:00:00 2001 From: Kit <344842+roodboi@users.noreply.github.com> Date: Sat, 14 Nov 2026 04:00:11 +0000 Subject: [PATCH 14/32] fix(env): preserve implicit Compose overlay compatibility --- src/lib/project-env-config.ts | 2 +- tests/env-migration-journal.test.ts | 33 +++++++++++++++++++++++++++++ 2 files changed, 34 insertions(+), 1 deletion(-) diff --git a/src/lib/project-env-config.ts b/src/lib/project-env-config.ts index d16a36a6e..6e2fc7236 100644 --- a/src/lib/project-env-config.ts +++ b/src/lib/project-env-config.ts @@ -6533,7 +6533,7 @@ async function prepareContractMigration( const selected = await selectContractMigrationOverlays({ ...opts, contract, - composeRepairFollows: true, + composeRepairFollows: opts.repairCompose !== false, }); const overlayNames = selected.overlayNames; const store = { diff --git a/tests/env-migration-journal.test.ts b/tests/env-migration-journal.test.ts index f3d7977ea..e85932d14 100644 --- a/tests/env-migration-journal.test.ts +++ b/tests/env-migration-journal.test.ts @@ -16,6 +16,7 @@ import { cleanupEnvMigration, type EnvMigrationProject, envMigrationJournalPath, + migrateLegacyEnvForCommand, previewEnvCleanup, previewEnvMigration, recordEnvMigrationProjectRun, @@ -232,6 +233,38 @@ test("preview makes no writes and exactly names what apply publishes; completed expect(await readFile(result.journalPath, "utf8")).toBe(journal); }); +test("journalled implicit migration keeps undeclared Compose overlay inputs; explicit repair still refuses their loss", async () => { + const composePath = join(project.projectDir, "docker-compose.yml"); + const compose = + "services:\n api:\n image: alpine:3.22\n env_file: .env.staging\n"; + await writeFile(composePath, compose); + const overlayPath = join(project.projectDir, ".env.staging"); + await writeFile(overlayPath, `UNDECLARED=${QA}\n`); + const before = await projectFiles(); + // Explicit migration repairs Compose references, so it must still refuse + // dropping an overlay value that the contract does not carry into layers. + await expect(previewEnvMigration(project)).rejects.toThrow(".env.staging"); + expect(await projectFiles()).toEqual(before); + const result = await migrateLegacyEnvForCommand({ + ...project, + materialize: false, + }); + expect(result.legacyDetected).toBe(true); + expect(JSON.stringify(result)).not.toContain(SENTINEL); + expect(JSON.stringify(result)).not.toContain(QA); + expect(await readFile(composePath, "utf8")).toBe(compose); + expect(await readFile(overlayPath, "utf8")).toBe(`UNDECLARED=${QA}\n`); + expect( + (await resolveProjectEnvConfig({ ...project, envName: null }))?.globalEnv + .TOKEN + ).toBe(SENTINEL); + expect( + JSON.parse( + await readFile(envMigrationJournalPath(project.projectRoot), "utf8") + ).state + ).toBe("complete"); +}); + test("retry sweeps only receipted staging artifacts and retains unknown contents", async () => { const preview = await previewEnvMigration(project); await mkdir(dirname(preview.journalPath), { recursive: true, mode: 0o700 }); From a87b13bf7fdf01811efe33e97dfc27e1b39e87a3 Mon Sep 17 00:00:00 2001 From: Kit <344842+roodboi@users.noreply.github.com> Date: Sat, 14 Nov 2026 04:11:47 +0000 Subject: [PATCH 15/32] fix(env): revalidate snapshots before resuming publication --- docs/reference/env-migration.md | 6 +- src/lib/env-migration.ts | 6 ++ tests/env-migration-provider.test.ts | 91 +++++++++++++++++++++++++++- 3 files changed, 101 insertions(+), 2 deletions(-) diff --git a/docs/reference/env-migration.md b/docs/reference/env-migration.md index 8b6c688fe..f04e47024 100644 --- a/docs/reference/env-migration.md +++ b/docs/reference/env-migration.md @@ -82,7 +82,11 @@ remain private. Each publication has an original checksum, an intended checksum and a completion receipt. A retry skips completed steps and adopts a matching publication whose -rename succeeded before its receipt. An unexpected input or destination change +rename succeeded before its receipt. Before publishing another file, a retry +revalidates the snapshot manifest and every original-file blob against the +journal. Missing or damaged rollback coverage refuses before publishing another +migration target; restore the intact snapshot bytes before retrying. An +unexpected input or destination change refuses instead of guessing. Re-run the same apply command after interruption. After completion it returns `already_complete` without rewriting the project or journal. `doctor --migrate-env-config` uses the same apply/journal path and keeps diff --git a/src/lib/env-migration.ts b/src/lib/env-migration.ts index 39af5be7e..c93693aa9 100644 --- a/src/lib/env-migration.ts +++ b/src/lib/env-migration.ts @@ -795,6 +795,12 @@ export async function applyEnvMigration(input: { } await input.hooks?.boundary?.({ phase: "prepared", index: -1 }); await assertSources(journal); + // A retained journal is not proof that its original rollback bytes + // survived the interruption. Admit them before any further publication. + await input.snapshots.assertCoverage({ + manifestPath: journal.manifestPath, + files: journal.sources, + }); for (const [index, entry] of journal.writes.entries()) { if (entry.done) { continue; diff --git a/tests/env-migration-provider.test.ts b/tests/env-migration-provider.test.ts index d1a391862..c8d3b3344 100644 --- a/tests/env-migration-provider.test.ts +++ b/tests/env-migration-provider.test.ts @@ -11,7 +11,10 @@ import { } from "node:fs/promises"; import { tmpdir } from "node:os"; import { dirname, isAbsolute, join, resolve } from "node:path"; -import { applyEnvMigration } from "../src/lib/env-migration.ts"; +import { + applyEnvMigration, + envMigrationJournalPath, +} from "../src/lib/env-migration.ts"; import { bindEnvMigrationSnapshots } from "../src/lib/env-migration-snapshot.ts"; import { isRecord } from "../src/lib/guards.ts"; import { resolveProjectEnvConfig } from "../src/lib/project-env-config.ts"; @@ -168,6 +171,92 @@ function pathField(data: Record, name: string): string { return path; } +for (const damage of ["manifest", "missing-blob", "changed-blob"] as const) { + test(`resumed production apply refuses ${damage} damage before another publication`, async () => { + const project = await fixture(); + await expect( + applyEnvMigration({ + project, + snapshots: bindEnvMigrationSnapshots(snapshotLibrary), + acceptTrackedChanges: true, + hooks: { + boundary: async ({ phase }) => { + if (phase === "prepared") { + throw new Error("synthetic prepared interruption"); + } + }, + }, + }) + ).rejects.toThrow("synthetic prepared interruption"); + const journalPath = envMigrationJournalPath(project.projectRoot); + const journal: unknown = JSON.parse(await readFile(journalPath, "utf8")); + if (!isRecord(journal)) { + throw new Error("Expected interrupted migration journal"); + } + const manifestPath = pathField(journal, "manifestPath"); + const loaded = await snapshotLibrary.readUpgradeSnapshot({ manifestPath }); + if (loaded.status !== "ok") { + throw new Error("Expected intact initial snapshot"); + } + const original = loaded.snapshot.manifest.entries.find( + (entry) => entry.path === join(project.projectDir, "hack.config.json") + ); + if (original?.before.kind !== "file") { + throw new Error("Expected config original in snapshot"); + } + const damaged = + damage === "manifest" + ? manifestPath + : join(loaded.snapshot.dir, "blobs", original.before.sha256); + const bytes = await readFile(damaged); + if (damage === "changed-blob") { + await writeFile(damaged, "synthetic damaged snapshot payload", { + mode: 0o600, + }); + } else { + await rm(damaged); + } + const paths = ( + await readdir(project.projectRoot, { recursive: true }) + ).sort(); + const before = await Promise.all( + paths.map(async (name) => { + const path = join(project.projectRoot, name); + return (await lstat(path)).isFile() ? await readFile(path) : null; + }) + ); + const journalBefore = await readFile(journalPath); + const result = await cli(project.projectRoot, [ + "--apply", + "--accept-tracked-env-changes", + ]); + expect(result.exitCode).not.toBe(0); + expect( + (await readdir(project.projectRoot, { recursive: true })).sort() + ).toEqual(paths); + for (const [index, name] of paths.entries()) { + const expected = before[index]; + if (expected) { + expect( + (await readFile(join(project.projectRoot, name))).equals(expected) + ).toBe(true); + } + } + expect((await readFile(journalPath)).equals(journalBefore)).toBe(true); + await writeFile(damaged, bytes, { mode: 0o600 }); + const resumed = await cli(project.projectRoot, [ + "--apply", + "--accept-tracked-env-changes", + ]); + expect(resumed.exitCode, resumed.stderr).toBe(0); + expect(resumed.envelope.data.status).toBe("applied"); + expect( + (await resolveProjectEnvConfig({ ...project, envName: null }))?.serviceEnv + .api?.SECRET + ).toBe(sentinel); + }); +} + test("production snapshot provider backs source CLI apply, no-op and guarded rollback", async () => { const project = await fixture(); const preview = await cli(project.projectRoot, []); From 632221672d5f90bc020eb02d6f55806c99fe0e7e Mon Sep 17 00:00:00 2001 From: Kit <344842+roodboi@users.noreply.github.com> Date: Sat, 14 Nov 2026 06:15:42 +0000 Subject: [PATCH 16/32] fix(env): preserve migration refusal guidance and compatibility --- docs/versions/v5/guides/upgrade-from-v4.md | 65 +++++++++++++++++----- src/lib/project-env-config.ts | 9 ++- tests/env-migration-safety.test.ts | 2 +- tests/env-set-local-contract.test.ts | 7 ++- 4 files changed, 61 insertions(+), 22 deletions(-) diff --git a/docs/versions/v5/guides/upgrade-from-v4.md b/docs/versions/v5/guides/upgrade-from-v4.md index 826d67279..c9a72fbf3 100644 --- a/docs/versions/v5/guides/upgrade-from-v4.md +++ b/docs/versions/v5/guides/upgrade-from-v4.md @@ -453,16 +453,27 @@ and separate backend keys still need the documented delivery and consent checks before removal. This does not repair an installed next.16 bundle or recover a key removed earlier. Keep the backup until recovery is verified. -After those prerequisites, the explicit migration command is: +Builds containing HACK-2191 (planned after next.17) add a read-only preview and +resumable migration. Check the chosen executable's help before using them: ```sh -HACK_RUNTIME_BACKEND=compose "$V5" doctor --path /absolute/project --migrate-env-config +HACK_RUNTIME_BACKEND=compose "$V5" env migrate --path /absolute/project --json +HACK_RUNTIME_BACKEND=compose "$V5" env migrate --path /absolute/project --apply ``` -This is a mutation, not a dry run. Review the tracked-file consent list and the -diff afterward. `--json` cannot be combined with this repair. Without a prompt, -tracked edits/deletions require explicit `--accept-tracked-env-changes`; do not -add that flag just to get past an unfamiliar refusal. +Preview lists paths, file actions and key names, never values, and writes nothing. +Apply keeps a private rollback snapshot and journal outside working trees. +Re-running the same apply command resumes interrupted steps; a completed run +reports a no-op. Snapshot storage and all targets must share a filesystem and +mount with hard links permitted. Choose a private `HACK_HOME` there if needed. +Neither command needs Docker, a daemon or the native backend. + +Review the tracked-file consent list and the diff afterward. Without a prompt, +tracked edits require explicit `--accept-tracked-env-changes`; the same flag +governs later tracked removals. Do not add it just to get past an unfamiliar +refusal. `doctor --migrate-env-config` uses the same journalled migration, but +its separate hygiene repairs can run before the migration snapshot. Its repair +mode still cannot be combined with `--json`. - The new model uses `hack.env.default.yaml`, named overlay YAML, and private `hack.env.local.yaml` / `hack.env..local.yaml` layers. With a legacy @@ -470,21 +481,45 @@ add that flag just to get past an unfamiliar refusal. - Without `hack.env.json`, legacy `.hack/.env` and overlay values go to private local layers; tracked layers are empty. Teammates must migrate their own local values after the shared change is committed. -- Doctor can remove old contract/dotenv files and Compose `env_file` references - with the documented consent and delivery checks. Undelivered values and files - still needed for interpolation are retained. It can also rewrite legacy config - keys. Do not delete all `.env*` files by hand. +- Migration can repair Compose `env_file` references and legacy config keys, + but keeps every old contract, dotenv file, store, backend key and store pointer. + Cleanup never runs as part of apply or Doctor migration. - Env-consuming commands can migrate implicitly. The implicit path keeps the - contract and tracked files and does not edit Compose, but can rewrite legacy - config keys and remove migrated untracked dotenv files. It is still a write. + legacy files and does not edit Compose, but can rewrite legacy config keys. + It is still a write. An unfinished apply or rollback blocks env readers until + the explicit migration command completes recovery. - Migration has refusal paths for collisions, symlinks, unreadable files and - changed delivery semantics. Earlier layer or ignore-rule changes can remain - after a refusal; inspect before retrying. Secret-backend/keychain discovery has - limits. Follow the full audit rather than assuming every overlay was found. + changed delivery semantics. A journal can record partial publication after an + interruption; resume it rather than editing its files. Secret-backend/keychain + discovery has limits. Follow the full audit rather than assuming every overlay + was found. - Managed values containing `$` now reach containers literally. A previously expanded `$VAR` or escaped `$$` can change the application value and cause container recreation on the next start. +After checking the application with a successful full-project `hack up`, preview +and request cleanup as a separate later step: + +```sh +HACK_RUNTIME_BACKEND=compose "$V5" env migrate --path /absolute/project --cleanup --json +HACK_RUNTIME_BACKEND=compose "$V5" env migrate --path /absolute/project --cleanup --apply +``` + +Cleanup rechecks the snapshot, delivery of every migrated value and live inputs +at that moment. A file current resolution reads, including an alias of a live +key, cannot be removed. Changed layers require another full-project run. +Rollback uses `env migrate --rollback-manifest ` with the path returned +by apply. It refuses later edits unless `--preserve-changes` is explicit; that +option retains those bytes privately and reports their recovery paths. An +incomplete rollback exits nonzero and must be resumed. See the +[preview and recovery contract](https://github.com/hack-dance/hack/blob/a87b13bf7fdf01811efe33e97dfc27e1b39e87a3/docs/reference/env-migration.md) for the +versioned JSON and partial-cleanup recovery rules. + +Older builds without `env migrate` still use `doctor --migrate-env-config` as a +mutation without this preview, journal or separate-cleanup guarantee. They can +remove legacy files during migration; retain the backups and the next.16 key +warning above. These new recovery guarantees do not apply to those binaries. + Sources: [migration and refusal audit](https://github.com/hack-dance/hack/blob/26a142921c9b9aeb8821b7e0a19c2cfb3be0189f/docs/env.md#legacy-repos-and-migration), [tracked-consent regression tests](https://github.com/hack-dance/hack/blob/26a142921c9b9aeb8821b7e0a19c2cfb3be0189f/tests/env-migration-tracked-consent.test.ts), [migration safety tests](https://github.com/hack-dance/hack/blob/26a142921c9b9aeb8821b7e0a19c2cfb3be0189f/tests/env-migration-safety.test.ts), diff --git a/src/lib/project-env-config.ts b/src/lib/project-env-config.ts index e3d6bc7bc..61a88a246 100644 --- a/src/lib/project-env-config.ts +++ b/src/lib/project-env-config.ts @@ -6785,9 +6785,12 @@ async function prepareLegacyProjectEnvMigrationInternal(opts: { await assertLegacyProjectInputFamily(opts); const contract = await readHackEnvContract(opts); if (contract.parseError !== undefined) { - throw new ClosedCliError( - "Cannot prepare migration: the env contract is unreadable or malformed; values omitted." - ); + throw new LegacyDotenvMigrationRefusalError({ + code: "E_CONFIG_INVALID", + reason: + "hack.env.json can't be read or parsed as an env contract, so the migration can't tell which values to keep. Correct it, or remove it if this project doesn't use a contract, then retry.", + closing: "No migration files were changed; values omitted.", + }); } const texts = new Map(); const extraSources = new Set([ diff --git a/tests/env-migration-safety.test.ts b/tests/env-migration-safety.test.ts index 7eaee7e6e..ea4abdb9f 100644 --- a/tests/env-migration-safety.test.ts +++ b/tests/env-migration-safety.test.ts @@ -393,7 +393,7 @@ test("an implicit env add keeps the pointer of a store with an undeclared entry, }); expect(await store.get({ key: "UNDECL" })).toBe(undeclared); expect(run.output).toContain( - "Kept the encrypted secret store pointer (controlPlane.secrets) in .hack/hack.config.json for secrets.enc.json, which stays: it still holds SEC, UNDECL (values omitted)" + "Legacy env files were kept. After validating the project, preview the separate cleanup step with hack env migrate --cleanup." ); for (const value of [declared, undeclared, added]) { expect(run.output).not.toContain(value); diff --git a/tests/env-set-local-contract.test.ts b/tests/env-set-local-contract.test.ts index 7ac34e099..cff951133 100644 --- a/tests/env-set-local-contract.test.ts +++ b/tests/env-set-local-contract.test.ts @@ -390,9 +390,10 @@ for (const lateContract of [false, true]) { ) ).not.toContain(VALUE); expect(await readFile(local, "utf8")).toContain(VALUE); - expect(run.output).toContain( - "Moved LOCAL_KEY to .hack/hack.env.local.yaml" - ); + expect(run.output).toContain("Env migration completed."); + expect(run.output).toContain("Legacy files were kept"); + expect(run.output).toContain("hack env migrate"); + expect(run.output).toContain("--cleanup"); expect(await git(project, ["check-ignore", "--", local])).toContain( "hack.env.local.yaml" ); From 66621eb463c2aab1f8676372d4db6e7c9f93fe7d Mon Sep 17 00:00:00 2001 From: Kit <344842+roodboi@users.noreply.github.com> Date: Sat, 14 Nov 2026 06:52:09 +0000 Subject: [PATCH 17/32] fix(env): journal explicit Compose repair after implicit migration --- docs/env.md | 5 + docs/reference/env-migration.md | 13 + docs/reference/env-migration.mmd | 5 + docs/reference/env-migration.svg | 4 +- docs/reference/v5-model-coverage.md | 2 +- docs/versions/v5/guides/upgrade-from-v4.md | 4 + scripts/lib/tla-runtime-models.ts | 18 +- src/commands/doctor.ts | 17 + src/lib/env-migration.ts | 351 ++++++++++++++---- src/lib/project-env-config.ts | 27 +- tests/env-migration-journal.test.ts | 232 +++++++++++- tests/env-migration-provider.test.ts | 66 +++- tests/env-migration-tracked-consent.test.ts | 48 +++ tests/models/tla/env-migration/COVERAGE.md | 15 +- tests/models/tla/env-migration/Migration.tla | 43 ++- tests/models/tla/env-migration/negative.cfg | 3 +- tests/models/tla/env-migration/positive.cfg | 3 +- .../tla/env-migration/stale-repair-proof.cfg | 6 + 18 files changed, 749 insertions(+), 113 deletions(-) create mode 100644 tests/models/tla/env-migration/stale-repair-proof.cfg diff --git a/docs/env.md b/docs/env.md index 4c4649b59..ee4f08125 100644 --- a/docs/env.md +++ b/docs/env.md @@ -1382,6 +1382,11 @@ project that has `hack.env.json` and no env layers. These implicit migrations report moved key names, retain every legacy copy and keep Compose `env_file` references. They never delete a tracked file or silently edit tracked config. Use the explicit preview/apply command when a shared config repair needs consent. +After an implicit migration, explicit preview and Doctor can still offer the +deferred Compose repair. That later phase keeps existing layers and legacy +files, rechecks delivery and snapshot coverage, and requires consent for tracked +edits. It invalidates an earlier project run, so start the complete project +again before cleanup. A repo that has no `.hack/hack.env.json` is migrated as well. Its values live only in `.hack/.env` and `.hack/.env.`, which are gitignored and may hold passwords or diff --git a/docs/reference/env-migration.md b/docs/reference/env-migration.md index f04e47024..0ac8e35a5 100644 --- a/docs/reference/env-migration.md +++ b/docs/reference/env-migration.md @@ -100,6 +100,19 @@ Compose `env_file` references. Implicit commands refuse an unfinished apply or rollback journal before using partially published layers. Resume the explicit apply or rollback command first. +When implicit migration retained legacy Compose `env_file` references, a later +explicit preview or `doctor --migrate-env-config` can offer their repair. This is +a new journalled phase: completed env layers are not published again, tracked +Compose edits require fresh consent, and migrated values must still resolve. +The Compose before image must be unchanged and covered by the original intact +snapshot; the same manifest rolls back the whole migration, including this +later repair. A fresh read revision permits unrelated additions in existing +layers and permission tightening by Doctor, while refusing changed legacy bytes +or lost migrated values. Its pending journal freezes that revision for resume. +The phase retains all legacy files and clears an earlier startup proof, so +cleanup still needs a later successful full-project run. Once the repair is +complete, another explicit apply is a no-op. + Migration refuses a Git-tracked project key path even when its file is absent from the worktree, including case-folded aliases such as `.Hack.Secret.Key`. Tracked-change consent does not allow publishing key bytes at that path. diff --git a/docs/reference/env-migration.mmd b/docs/reference/env-migration.mmd index 6ac017855..3d1d26873 100644 --- a/docs/reference/env-migration.mmd +++ b/docs/reference/env-migration.mmd @@ -8,6 +8,7 @@ stateDiagram-v2 apply --> apply : Publish / Record apply --> idle : Crash / Finish idle --> apply : Resume [unfinished] + idle --> apply : RequestRepair [complete, deferred, snapshot, legacy] idle --> idle : SuccessfulRun / DamageBackup idle --> cleanup : RequestCleanup [proof] cleanup --> idle : RemoveLegacy [safe] @@ -23,6 +24,10 @@ stateDiagram-v2 INV no live targets Fresh delivery required end note + note right of idle + RequestRepair clears run proof + Completed layers stay published + end note classDef live stroke-width:1.6px classDef wait stroke-width:1.6px classDef idle stroke-dasharray:4 3 diff --git a/docs/reference/env-migration.svg b/docs/reference/env-migration.svg index c6331fa07..7cddd0abf 100644 --- a/docs/reference/env-migration.svg +++ b/docs/reference/env-migration.svg @@ -1,4 +1,4 @@ -Migration · journal and separate cleanupPrepare [snapshot]Publish / RecordCrash / FinishResume [unfinished]SuccessfulRun /DamageBackupRequestCleanup [proof]RemoveLegacy [safe]RefuseCleanup [before unlink]Rollback [snapshot]Preparepreviewidlerolled_backapplycleanupINV legacy retainedReceipts follow publicationINV intact snapshotINV no live targetsFresh delivery required \ No newline at end of file +]]>Migration · journal and separate cleanupPrepare [snapshot]Publish / RecordCrash / FinishResume [unfinished]RequestRepair [complete,deferred, snapshot, legacy]SuccessfulRun /DamageBackupRequestCleanup [proof]RemoveLegacy [safe]RefuseCleanup [before unlink]Rollback [snapshot]Preparepreviewrolled_backapplyidlecleanupINV legacy retainedReceipts follow publicationRequestRepair clears run proofCompleted layers stay publishedINV intact snapshotINV no live targetsFresh delivery required \ No newline at end of file diff --git a/docs/reference/v5-model-coverage.md b/docs/reference/v5-model-coverage.md index ef07d1b9d..8dcc4c1d5 100644 --- a/docs/reference/v5-model-coverage.md +++ b/docs/reference/v5-model-coverage.md @@ -36,7 +36,7 @@ must be attached to the eventual exact integration revision. | Property and critical states | Model coverage | Implementation/regression boundary | Remaining qualification | | --- | --- | --- | --- | -| Env migration publication, crash/resume and later cleanup | [env-migration](../../tests/models/tla/env-migration/COVERAGE.md): 37 states, two publication steps, separate publication and receipt, safe pre-removal cleanup refusal; unsafe same-run deletion fails `NoSameRunCleanup` | `env-migration.ts` binds immutable payloads and source checksums to a private snapshot, adopts interrupted renames, records completion, requires a successful project run and fresh snapshot/delivery/live-input proofs for explicit cleanup. Real-file tests check preview parity, sentinels, backup damage, refused cleanup retry and rollback; owned child processes receive SIGKILL at every journal/publication boundary | The model assumes atomic per-file publication/journal writes and cooperative locks; it does not qualify fsync durability, real snapshot provider behavior, partial cleanup/rollback, unsynchronized writers, actual runtime environment or macOS. Implementation and dependency integration gates remain separate. | +| Env migration publication, crash/resume and later cleanup | [env-migration](../../tests/models/tla/env-migration/COVERAGE.md): 74 states, two steps per phase, deferred Compose repair, separate publication and receipt, safe pre-removal cleanup refusal; unsafe controls fail `NoSameRunCleanup` and `FreshRunProof` | `env-migration.ts` binds immutable payloads and source checksums to a private snapshot, adopts interrupted renames, records completion, admits later explicit Compose repair without republishing env layers, and requires a new successful project run plus fresh snapshot/delivery/live-input proofs for cleanup. Real-file tests check preview parity, sentinels, backup damage, consent, tighter modes, refused cleanup retry and rollback; owned child processes receive SIGKILL at initial and later journal/publication boundaries | The model assumes atomic per-file publication/journal writes and cooperative locks; it does not qualify fsync durability, real snapshot provider behavior, partial cleanup/rollback, unsynchronized writers, actual runtime environment or macOS. Implementation and dependency integration gates remain separate. | | Interactive exec write-ahead admission, crash, EOF, replacement boot and retirement | interactive-session (459 states); no-replay, EOF-retirement, failed-inspect and wrong-boot negative controls | `interactive_session.rs` atomic intent/exec records, exact stopped-boot proof, known-undispatched withdrawal, restore-before-busy-retirement and run-scoped cleanup-report tests, and lock-free observation/revalidation tests; Engine effect fences; authored owner start and SCM_RIGHTS handoff | One session, two boot identities, two possible starts. Atomic durable publication and exact identity/absence proofs are assumptions; real fsync/power loss, real owner/Engine handoff and VM loss need Mac acceptance; receipt lock timing and terminal restoration are regression-covered, not modeled. Validated exact exec exit permits retirement; it makes no descendant cleanup claim. | | Private interactive envelope, echo state and bytes still in flight on failure | interactive-envelope (15 states); restore-on-error negative control | HKEE2 launcher and host gate, Linux PTY seven-case matrix, bounded marker/ACK and coalesced-output tests | Two abstract private bytes, one handshake. PTY fixtures prove local termios ordering; Docker detach scanning, real guest wiring, same-user access and VM effects remain separate. Source/compiled Bun PTY and native-process fixtures cover lifetime/terminal boundaries, not model liveness. | | Native status across startup lease stages; failed start with running services (HACK-2110) | [native-status-start](../../tests/models/tla/native-status-start/COVERAGE.md) | Observer-only presentation, stable journal fence, distinct authored Observe/Status replies; real-flock held-reader start test and expiry diagnostic; plain-down exact-proof recovery tests. Failure capture occurs at startup/effect verification and owned cleanup, never status; first failure wins, incomplete-rebind capture and owner-crash reconstruction are excluded | Seven acquisition stages, one reader, one start; 110 states. Negative control fails after services start. Second-mutator refusal and complete recovery proof are assumptions checked separately in code, not independently proved by this model. No new control-loop tick or lease; cleanup settle is bounded at ten seconds per exited listener. Lead Mac gate: ps every 0.5 s across ten starts, cross-project polling and failed-start plain down. | diff --git a/docs/versions/v5/guides/upgrade-from-v4.md b/docs/versions/v5/guides/upgrade-from-v4.md index c9a72fbf3..2832c2267 100644 --- a/docs/versions/v5/guides/upgrade-from-v4.md +++ b/docs/versions/v5/guides/upgrade-from-v4.md @@ -488,6 +488,10 @@ mode still cannot be combined with `--json`. legacy files and does not edit Compose, but can rewrite legacy config keys. It is still a write. An unfinished apply or rollback blocks env readers until the explicit migration command completes recovery. + After implicit completion, explicit preview and Doctor can still offer the + deferred Compose repair, with fresh delivery/snapshot checks and tracked-file + consent. They keep existing layers and legacy files. Start the full project + again after that repair before requesting cleanup. - Migration has refusal paths for collisions, symlinks, unreadable files and changed delivery semantics. A journal can record partial publication after an interruption; resume it rather than editing its files. Secret-backend/keychain diff --git a/scripts/lib/tla-runtime-models.ts b/scripts/lib/tla-runtime-models.ts index 56f971066..a5e161cf0 100644 --- a/scripts/lib/tla-runtime-models.ts +++ b/scripts/lib/tla-runtime-models.ts @@ -144,7 +144,7 @@ const contracts: readonly ModelContract[] = [ { name: "env-migration", module: "Migration", - states: 37, + states: 74, invariant: "NoSameRunCleanup", action: "UnsafeRemoveDuringApply", fields: [ @@ -157,6 +157,22 @@ const contracts: readonly ModelContract[] = [ "complete = FALSE", "runProof = FALSE", ], + additionalControls: [ + { + name: "stale-repair-proof", + negative: true, + invariant: "FreshRunProof", + action: "RequestRepair", + fields: [ + 'phase = "apply"', + "complete = FALSE", + "runProof = TRUE", + "snapshot = TRUE", + "legacy = TRUE", + "deferred = FALSE", + ], + }, + ], }, { name: "interactive-session", diff --git a/src/commands/doctor.ts b/src/commands/doctor.ts index 332f7b9df..185d519d1 100644 --- a/src/commands/doctor.ts +++ b/src/commands/doctor.ts @@ -4315,6 +4315,23 @@ async function buildEnvMigrationPlanStep(opts: { throw error; } if (modernExists) { + const config = await readProjectConfig(opts.project); + const composeFile = opts.project.composeFile; + const preview = await previewEnvMigration({ + projectRoot: opts.project.projectRoot, + projectDir: opts.project.projectDir, + projectName: config.name ?? "project", + serviceNames: await discoverComposeServiceNames({ + composeFile, + }), + }); + if ( + preview.files.some( + (file) => file.action === "change" && file.path === composeFile + ) + ) { + return "Prompt to repair retained Compose env_file references; keep completed env layers and legacy files."; + } return "Skip env migration because this project already uses hack.env.*.yaml."; } diff --git a/src/lib/env-migration.ts b/src/lib/env-migration.ts index c93693aa9..cc1a3270d 100644 --- a/src/lib/env-migration.ts +++ b/src/lib/env-migration.ts @@ -34,6 +34,7 @@ import { } from "./private-staging.ts"; import { acquireLegacyEnvCleanupContext, + assertLegacyEnvMigrationDelivery, assertMachineLocalFileGitPrivate, assertProjectEnvKeyUntracked, assertProjectEnvLayerPaths, @@ -102,6 +103,10 @@ type Journal = { readonly overlayNames: readonly string[]; readonly store?: { readonly path: string; readonly keyPath: string }; readonly removed: readonly string[]; + /** Implicit publication leaves Compose repair for a later, consented phase. */ + readonly composeRepair?: "deferred" | "applying" | "complete"; + /** Read revision for the later phase; original sources remain the snapshot's before images. */ + readonly composeRepairSources?: readonly EnvMigrationFileState[]; /** Layer revision observed after a complete project startup succeeded. */ readonly validated?: readonly EnvMigrationFileState[]; readonly cleanupConfig?: Publication; @@ -285,6 +290,15 @@ async function readJournal( ].includes(String(raw.state)) && Array.isArray(raw.sources) && raw.sources.every(validState) && + (raw.composeRepair === undefined || + ["deferred", "applying", "complete"].includes( + String(raw.composeRepair) + )) && + (raw.composeRepairSources === undefined || + (Array.isArray(raw.composeRepairSources) && + raw.composeRepairSources.every(validState))) && + (raw.composeRepair !== "applying" || + Array.isArray(raw.composeRepairSources)) && Array.isArray(raw.writes) && raw.writes.every((entry) => validPublication(entry, path)) && Array.isArray(raw.removable) && @@ -381,11 +395,18 @@ async function prepareConsistent( } export async function previewEnvMigration( - input: EnvMigrationProject + input: EnvMigrationProject & { readonly repairCompose?: boolean } ): Promise { const project = await canonicalProject(input); const path = envMigrationJournalPath(project.projectRoot); const journal = await readJournal(path, project); + if (needsComposeRepair(journal, input.repairCompose)) { + const prepared = await prepareDeferredComposeRepair(project, journal); + return { + ...(await previewPrepared(project, prepared.prepared, path)), + manifestPath: journal.manifestPath, + }; + } if (journal && journal.state !== "rolled_back") { const files: EnvMigrationPreview["files"][number][] = []; for (const file of journal.writes) { @@ -431,7 +452,12 @@ export async function previewEnvMigration( } return await previewPrepared( project, - (await prepareConsistent(project)).prepared, + ( + await prepareConsistent({ + ...project, + repairCompose: input.repairCompose, + }) + ).prepared, path ); } @@ -448,7 +474,10 @@ export async function previewEnvCleanup( input: EnvMigrationProject ): Promise { const project = await canonicalProject(input); - const preview = await previewEnvMigration(project); + const preview = await previewEnvMigration({ + ...project, + repairCompose: false, + }); const journal = await readJournal(preview.journalPath, project); const files = preview.files.filter((file) => file.action === "removable"); if ( @@ -491,24 +520,33 @@ async function assertConsent( async function assertSources( journal: Journal, - allowLayerChanges = false + { + allowLayerChanges = false, + allowTighterModes = false, + }: { + readonly allowLayerChanges?: boolean; + readonly allowTighterModes?: boolean; + } = {} ): Promise { - for (const source of journal.sources) { + const revised = journal.composeRepairSources !== undefined; + for (const source of journal.composeRepairSources ?? journal.sources) { if (journal.removed.includes(source.path)) { continue; } const current = (await readState(source.path)).state; - const write = journal.writes.find((entry) => entry.path === source.path); + const publication = journal.writes.find( + (entry) => entry.path === source.path + ); + const write = + revised && publication?.kind !== "compose" ? undefined : publication; if ( allowLayerChanges && - (write?.kind === "layer" || write?.kind === "local_layer") + (publication?.kind === "layer" || publication?.kind === "local_layer") ) { continue; } if ( - write?.done - ? !same(current, write) - : !(same(current, source) || (write && same(current, write))) + !matchesMigrationSource({ current, source, write, allowTighterModes }) ) { throw new ClosedCliError( `Env migration input changed: ${JSON.stringify(source.path)}. Keep the legacy files and snapshot. To restore originals while retaining later edits privately, use hack env migrate --rollback-manifest ${JSON.stringify(journal.manifestPath)} --preserve-changes (values omitted).` @@ -517,6 +555,33 @@ async function assertSources( } } +function matchesMigrationSource({ + current, + source, + write, + allowTighterModes, +}: { + readonly current: EnvMigrationFileState; + readonly source: EnvMigrationFileState; + readonly write: Publication | undefined; + readonly allowTighterModes: boolean; +}): boolean { + const expected = write?.done ? write : source; + if ( + allowTighterModes && + expected.mode !== null && + current.mode === (expected.mode & 0o600) && + current.sha256 === expected.sha256 && + current.size === expected.size + ) { + return true; + } + return ( + same(current, expected) || + (!write?.done && write !== undefined && same(current, write)) + ); +} + async function prepareJournal( project: EnvMigrationProject, path: string, @@ -530,6 +595,32 @@ async function prepareJournal( await assertOutsideWorkingTrees(snapshot.manifestPath, project); await snapshots.assertCoverage({ ...snapshot, files: sources }); await snapshots.recordAfter({ ...snapshot, files: sources }); + const writes = await stagePublications(project, path, prepared, sources); + const journal: Journal = { + version: 1, + kind: "hack-env-migration", + project, + manifestPath: snapshot.manifestPath, + state: "applying", + sources, + writes, + removable: prepared.removable, + overlayNames: prepared.overlayNames, + ...(prepared.store ? { store: prepared.store } : {}), + removed: [], + composeRepair: prepared.composeRepairDeferred ? "deferred" : "complete", + }; + await assertSources(journal); + await writeMetadata(path, journal); + return journal; +} + +async function stagePublications( + project: EnvMigrationProject, + path: string, + prepared: PreparedLegacyEnvMigration, + sources: readonly EnvMigrationFileState[] +): Promise { const payloads = prepared.files.map((file) => { const text = prepared.payload(file.path); return { text, sha256: digest(Buffer.from(text)) }; @@ -539,7 +630,7 @@ async function prepareJournal( projectRoot: project.projectRoot, files: payloads, }); - const writes: Publication[] = prepared.files.map((file, index) => { + return prepared.files.map((file, index) => { const before = sources.find((entry) => entry.path === file.path); const payload = staged[index]; const bytes = payloads[index]; @@ -555,22 +646,88 @@ async function prepareJournal( done: false, }; }); - const journal: Journal = { - version: 1, - kind: "hack-env-migration", +} + +function needsComposeRepair( + journal: Journal | null, + repairCompose: boolean | undefined +): journal is Journal & { + readonly state: "complete"; + readonly composeRepair: "deferred"; +} { + return ( + journal?.state === "complete" && + journal.composeRepair === "deferred" && + repairCompose !== false + ); +} + +/** Only an unchanged, already snapshotted Compose file may extend a completed implicit migration. */ +async function prepareDeferredComposeRepair( + project: EnvMigrationProject, + journal: Journal +): Promise { + await assertSources(journal, { + allowLayerChanges: true, + allowTighterModes: true, + }); + const preparation = await prepareConsistent(project); + if ( + preparation.prepared.files.some( + (file) => + file.kind !== "compose" || + journal.writes.some((write) => write.path === file.path) + ) || + preparation.sources.some( + (source) => + !journal.sources.some((original) => original.path === source.path) + ) + ) { + throw refuse(); + } + await assertLegacyEnvMigrationDelivery({ + ...project, + overlayNames: preparation.prepared.overlayNames, + }); + return preparation; +} + +async function extendComposeRepair( + project: EnvMigrationProject, + path: string, + journal: Journal, + snapshots: EnvMigrationSnapshots, + accept: boolean +): Promise { + const { prepared, sources } = await prepareDeferredComposeRepair( project, - manifestPath: snapshot.manifestPath, + journal + ); + await assertConsent( + project, + prepared.files.map((file) => file.path), + accept + ); + if (prepared.files.length === 0) { + return journal; + } + await snapshots.assertCoverage({ + manifestPath: journal.manifestPath, + files: journal.sources, + }); + const writes = await stagePublications(project, path, prepared, sources); + const extended: Journal = { + ...journal, state: "applying", - sources, - writes, + writes: [...journal.writes, ...writes], removable: prepared.removable, - overlayNames: prepared.overlayNames, - ...(prepared.store ? { store: prepared.store } : {}), - removed: [], + composeRepair: "applying", + composeRepairSources: sources, + validated: undefined, }; - await assertSources(journal); - await writeMetadata(path, journal); - return journal; + await assertSources(extended); + await writeMetadata(path, extended); + return extended; } async function publishStagingGuard( @@ -704,14 +861,21 @@ export async function applyEnvMigration(input: { const existing = await readJournal(path, project); assertApplyJournalState(existing); await input.hooks?.beforeLock?.(); - if (existing && ["complete", "cleaned"].includes(existing.state)) { + if ( + existing && + ["complete", "cleaned"].includes(existing.state) && + !needsComposeRepair(existing, input.repairCompose) + ) { return await withProjectsRegistryLock({ lockPath: `${path}.lock`, label: "env migration", run: async () => { const current = await readJournal(path, project); assertApplyJournalState(current); - if (!(current && ["complete", "cleaned"].includes(current.state))) { + if ( + !(current && ["complete", "cleaned"].includes(current.state)) || + needsComposeRepair(current, input.repairCompose) + ) { throw new ClosedCliError( "Env migration state changed while waiting. Repeat the preview and apply command; no project files were changed." ); @@ -731,17 +895,20 @@ export async function applyEnvMigration(input: { }, }); } - const preparation = - existing && existing.state !== "rolled_back" - ? null - : await prepareConsistent({ - ...project, - repairCompose: input.repairCompose, - }); - const changed = - existing && existing.state !== "rolled_back" - ? existing.writes.map((entry) => entry.path) - : (preparation?.prepared.files.map((entry) => entry.path) ?? []); + let preparation: Preparation | null = null; + if (needsComposeRepair(existing, input.repairCompose)) { + preparation = await prepareDeferredComposeRepair(project, existing); + } else if (!existing || existing.state === "rolled_back") { + preparation = await prepareConsistent({ + ...project, + repairCompose: input.repairCompose, + }); + } + const changed = preparation + ? preparation.prepared.files.map((entry) => entry.path) + : (existing?.writes + .filter((entry) => !entry.done) + .map((entry) => entry.path) ?? []); await assertConsent(project, changed, input.acceptTrackedChanges); if ( preparation && @@ -781,9 +948,20 @@ export async function applyEnvMigration(input: { repairCompose: input.repairCompose, })) ); + if (needsComposeRepair(journal, input.repairCompose)) { + journal = await extendComposeRepair( + project, + path, + journal, + input.snapshots, + input.acceptTrackedChanges + ); + } await assertConsent( project, - journal.writes.map((entry) => entry.path), + journal.writes + .filter((entry) => !entry.done) + .map((entry) => entry.path), input.acceptTrackedChanges ); if (journal.state !== "applying") { @@ -801,11 +979,24 @@ export async function applyEnvMigration(input: { manifestPath: journal.manifestPath, files: journal.sources, }); + if (journal.composeRepair === "applying") { + await withEnvMigrationVerification({ + projectRoot: project.projectRoot, + run: () => + assertLegacyEnvMigrationDelivery({ + ...project, + overlayNames: journal.overlayNames, + }), + }); + } for (const [index, entry] of journal.writes.entries()) { if (entry.done) { continue; } await input.hooks?.boundary?.({ phase: "before_publish", index }); + if (journal.composeRepairSources) { + await assertSources(journal); + } await publishEntry(project, entry); await input.hooks?.boundary?.({ phase: "published", index }); await input.snapshots.recordAfter({ @@ -821,36 +1012,60 @@ export async function applyEnvMigration(input: { await writeMetadata(path, journal); await input.hooks?.boundary?.({ phase: "recorded", index }); } - await assertSources(journal); - await withEnvMigrationVerification({ - projectRoot: project.projectRoot, - run: async () => { - for (const envName of [null, ...journal.overlayNames]) { - if (!(await resolveProjectEnvConfig({ ...project, envName }))) { - throw refuse(); - } - } - }, - }); - await input.hooks?.boundary?.({ phase: "verified", index: -1 }); - journal = { ...journal, state: "complete" }; - await writeMetadata(path, journal); - await input.hooks?.boundary?.({ phase: "complete", index: -1 }); - const payloadCleanup = await purgeMigrationPayloads({ - journalPath: path, - projectRoot: project.projectRoot, - retain: [], + return await finishApplyPhase({ + project, + path, + journal, + hooks: input.hooks, }); - return { - payloadCleanup: payloadCleanup ? "complete" : "kept", - status: "applied" as const, - journalPath: path, - manifestPath: journal.manifestPath, - }; }, }); } +/** Both initial migration and later repair verify resolution before recording completion. */ +async function finishApplyPhase({ + project, + path, + journal, + hooks, +}: { + readonly project: EnvMigrationProject; + readonly path: string; + readonly journal: Journal; + readonly hooks?: EnvMigrationHooks; +}) { + await assertSources(journal); + await withEnvMigrationVerification({ + projectRoot: project.projectRoot, + run: async () => { + for (const envName of [null, ...journal.overlayNames]) { + if (!(await resolveProjectEnvConfig({ ...project, envName }))) { + throw refuse(); + } + } + }, + }); + await hooks?.boundary?.({ phase: "verified", index: -1 }); + await writeMetadata(path, { + ...journal, + state: "complete", + composeRepair: + journal.composeRepair === "applying" ? "complete" : journal.composeRepair, + }); + await hooks?.boundary?.({ phase: "complete", index: -1 }); + const payloadCleanup = await purgeMigrationPayloads({ + journalPath: path, + projectRoot: project.projectRoot, + retain: [], + }); + return { + payloadCleanup: payloadCleanup ? "complete" : "kept", + status: "applied" as const, + journalPath: path, + manifestPath: journal.manifestPath, + }; +} + /** Explicit later action: a snapshot and fresh legacy-to-current delivery proof are both required. */ export async function cleanupEnvMigration(input: { readonly project: EnvMigrationProject; @@ -1046,7 +1261,7 @@ async function cleanupLocked( manifestPath: journal.manifestPath, files: journal.sources, }); - await assertSources(journal, true); + await assertSources(journal, { allowLayerChanges: true }); const context = await acquireLegacyEnvCleanupContext({ ...project, overlayNames: journal.overlayNames, @@ -1085,7 +1300,7 @@ async function cleanupLocked( }, }; } - await assertSources(journal, true); + await assertSources(journal, { allowLayerChanges: true }); journal = { ...journal, state: "cleaning" }; await writeMetadata(path, journal); let removed: readonly string[] = []; @@ -1250,7 +1465,11 @@ async function rollbackLocked( journal.removable.includes(source.path) && current.sha256 === null); let expected = - journal.writes.find((write) => write.path === source.path) ?? source; + journal.writes.find((write) => write.path === source.path) ?? + journal.composeRepairSources?.find( + (entry) => entry.path === source.path + ) ?? + source; if (removed) { expected = { ...source, sha256: null, mode: null, size: 0 }; } else if ( @@ -1307,7 +1526,7 @@ export async function migrateLegacyEnvForCommand( : await prepareConsistent({ ...canonical, repairCompose: false }); const preview = retained && retained.state !== "rolled_back" - ? await previewEnvMigration(canonical) + ? await previewEnvMigration({ ...canonical, repairCompose: false }) : await previewPrepared( canonical, preparation?.prepared ?? diff --git a/src/lib/project-env-config.ts b/src/lib/project-env-config.ts index 61a88a246..eaef8e5c2 100644 --- a/src/lib/project-env-config.ts +++ b/src/lib/project-env-config.ts @@ -6339,6 +6339,7 @@ export type PreparedLegacyEnvMigration = { readonly removable: readonly string[]; readonly overlayNames: readonly string[]; readonly hasContract: boolean; + readonly composeRepairDeferred: boolean; readonly retainedEnvFiles: readonly RetainedLegacyEnvFile[]; readonly blockedCleanupCandidates: readonly string[]; readonly store?: { readonly path: string; readonly keyPath: string }; @@ -6875,16 +6876,17 @@ async function prepareLegacyProjectEnvMigrationInternal(opts: { if (opts.repairCompose !== false) { await prepareMigrationCompose(opts, add); } + const composeEnvFileReferences = + opts.repairCompose === false + ? await inspectLegacyComposeEnvFileReferences({ ...opts, composeFile }) + : []; const candidates = await collectLegacyProjectEnvCleanupCandidates({ ...opts, overlayNames, materialize: false, deliveredValues, configCleanupCandidates: configCandidates, - composeEnvFileReferences: - opts.repairCompose === false - ? await inspectLegacyComposeEnvFileReferences({ ...opts, composeFile }) - : [], + composeEnvFileReferences, composeInterpolatedKeys: new Map([ [ resolve(opts.projectDir, PROJECT_ENV_FILENAME), @@ -6957,6 +6959,7 @@ async function prepareLegacyProjectEnvMigrationInternal(opts: { removable, overlayNames, hasContract: contract.exists, + composeRepairDeferred: composeEnvFileReferences.length > 0, retainedEnvFiles: candidates.retainedEnvFiles, blockedCleanupCandidates: candidates.blocked, ...(store ? { store } : {}), @@ -6996,6 +6999,22 @@ export async function acquireLegacyEnvCleanupContext(opts: { return { ...opts, contract: contract.contract, receipts }; } +/** A later Compose repair must not rely on layers merely existing after implicit migration. */ +export async function assertLegacyEnvMigrationDelivery(opts: { + readonly projectRoot: string; + readonly projectDir: string; + readonly projectName: string; + readonly serviceNames: readonly string[]; + readonly overlayNames: readonly string[]; +}): Promise { + const context = await acquireLegacyEnvCleanupContext({ ...opts, paths: [] }); + await assertLegacyCleanupDelivery({ + projectRoot: opts.projectRoot, + context, + paths: [resolve(opts.projectDir, PROJECT_ENV_CONTRACT_FILENAME)], + }); +} + /** * Migrates a legacy v2 `.env`-based project env layout to the v3 * `hack.env.*.yaml` config system. diff --git a/tests/env-migration-journal.test.ts b/tests/env-migration-journal.test.ts index e85932d14..53f9d83f7 100644 --- a/tests/env-migration-journal.test.ts +++ b/tests/env-migration-journal.test.ts @@ -1,5 +1,6 @@ import { afterEach, beforeEach, expect, test } from "bun:test"; import { + chmod, lstat, mkdir, mkdtemp, @@ -24,6 +25,7 @@ import { } from "../src/lib/env-migration.ts"; import { migrationDigest } from "../src/lib/env-migration-files.ts"; import { stageMigrationPayloads } from "../src/lib/env-migration-payloads.ts"; +import { envMigrationSnapshots } from "../src/lib/env-migration-snapshot.ts"; import { LegacyEnvCleanupError, resolveProjectEnvConfig, @@ -263,6 +265,183 @@ test("journalled implicit migration keeps undeclared Compose overlay inputs; exp await readFile(envMigrationJournalPath(project.projectRoot), "utf8") ).state ).toBe("complete"); + await expect(previewEnvMigration(project)).rejects.toThrow(".env.staging"); + await expect( + applyEnvMigration({ + project, + snapshots: envMigrationSnapshots(), + acceptTrackedChanges: true, + }) + ).rejects.toThrow(".env.staging"); + expect(await readFile(composePath, "utf8")).toBe(compose); +}); + +test("explicit migration after implicit completion previews only Compose repair and keeps one reversible journal", async () => { + const composePath = join(project.projectDir, "docker-compose.yml"); + const compose = + "services:\n api:\n image: alpine:3.22\n env_file: .env\n"; + await writeFile(composePath, compose); + const before = await projectFiles(); + await migrateLegacyEnvForCommand({ ...project, materialize: false }); + const journalPath = envMigrationJournalPath(project.projectRoot); + const implicit = JSON.parse(await readFile(journalPath, "utf8")); + const migrated = await projectFiles(); + const preview = await previewEnvMigration(project); + expect(await projectFiles()).toEqual(migrated); + expect(preview.status).toBe("ready"); + expect(preview.manifestPath).toBe(implicit.manifestPath); + expect(preview.files.filter((file) => file.action !== "removable")).toEqual([ + { path: composePath, action: "change", keys: [], tracked: false }, + ]); + expect(preview.files).toContainEqual({ + path: join(project.projectDir, ".env"), + action: "removable", + keys: [], + tracked: false, + }); + const snapshots = envMigrationSnapshots(); + const repaired = await apply({ + project, + snapshots, + acceptTrackedChanges: true, + }); + expect(repaired.status).toBe("applied"); + expect(repaired.manifestPath).toBe(implicit.manifestPath); + const after = await projectFiles(); + expect( + [...after] + .filter(([path, text]) => migrated.get(path) !== text) + .map(([path]) => path) + ).toEqual([composePath]); + expect(await readFile(composePath, "utf8")).not.toContain("env_file"); + expect(await readFile(join(project.projectDir, ".env"), "utf8")).toContain( + SENTINEL + ); + expect(JSON.stringify(preview)).not.toContain(SENTINEL); + expect(await readFile(journalPath, "utf8")).not.toContain(SENTINEL); + expect( + (await apply({ project, snapshots, acceptTrackedChanges: true })).status + ).toBe("already_complete"); + await recordEnvMigrationProjectRun(project); + await cleanupEnvMigration({ project, snapshots, acceptTrackedChanges: true }); + expect(await Bun.file(join(project.projectDir, ".env")).exists()).toBe(false); + await rollbackEnvMigration({ + project, + snapshots, + manifestPath: repaired.manifestPath, + }); + expectRestored(await projectFiles(), before); +}); + +test("later Compose repair accepts layer additions and tighter modes, invalidates old run proof and refuses lost values", async () => { + await writeFile( + join(project.projectDir, "docker-compose.yml"), + "services:\n api:\n image: alpine:3.22\n env_file: .env\n" + ); + await migrateLegacyEnvForCommand({ ...project, materialize: false }); + await recordEnvMigrationProjectRun(project); + const layer = join(project.projectDir, "hack.env.local.yaml"); + const original = await readFile(layer, "utf8"); + await writeFile(layer, original.replace(SENTINEL, "changed-synthetic-value")); + await expect(previewEnvMigration(project)).rejects.toThrow(); + const changedLayer = original.replace( + " global:\n", + " global:\n EXTRA: additional-synthetic-value\n" + ); + expect(changedLayer).not.toBe(original); + await writeFile(layer, changedLayer); + await chmod(join(project.projectDir, ".env"), 0o600); + await apply({ + project, + snapshots: envMigrationSnapshots(), + acceptTrackedChanges: true, + }); + expect(await readFile(layer, "utf8")).toBe(changedLayer); + await expect( + cleanupEnvMigration({ + project, + snapshots: envMigrationSnapshots(), + acceptTrackedChanges: true, + }) + ).rejects.toThrow("hack up"); + await recordEnvMigrationProjectRun(project); + await cleanupEnvMigration({ + project, + snapshots: envMigrationSnapshots(), + acceptTrackedChanges: true, + }); + expect(await Bun.file(join(project.projectDir, ".env")).exists()).toBe(false); +}); + +test("later Compose repair refuses damaged originals before changing the completed journal", async () => { + const composePath = join(project.projectDir, "docker-compose.yml"); + await writeFile( + composePath, + "services:\n api:\n image: alpine:3.22\n env_file: .env\n" + ); + const snapshots = fixtureEnvSnapshots(root); + const implicit = await apply({ + project, + snapshots, + acceptTrackedChanges: true, + repairCompose: false, + }); + const manifest = JSON.parse(await readFile(implicit.manifestPath, "utf8")); + const entry = manifest.files.find( + (file: { path: string }) => file.path === composePath + ); + const original = await readFile(entry.payload); + await writeFile(entry.payload, "damaged synthetic original"); + const before = await projectFiles(); + const journal = await readFile(implicit.journalPath, "utf8"); + await expect( + apply({ project, snapshots, acceptTrackedChanges: true }) + ).rejects.toThrow("coverage"); + expect(await projectFiles()).toEqual(before); + expect(await readFile(implicit.journalPath, "utf8")).toBe(journal); + await writeFile(entry.payload, original); + expect( + (await apply({ project, snapshots, acceptTrackedChanges: true })).status + ).toBe("applied"); +}); + +test("later Compose repair rechecks its read revision before publication and on resume", async () => { + const composePath = join(project.projectDir, "docker-compose.yml"); + const compose = + "services:\n api:\n image: alpine:3.22\n env_file: .env\n"; + await writeFile(composePath, compose); + const snapshots = fixtureEnvSnapshots(root); + await apply({ + project, + snapshots, + acceptTrackedChanges: true, + repairCompose: false, + }); + const envPath = join(project.projectDir, ".env"); + const original = await readFile(envPath, "utf8"); + await expect( + apply({ + project, + snapshots, + acceptTrackedChanges: true, + hooks: { + boundary: async ({ phase }) => { + if (phase === "before_publish") { + await writeFile(envPath, "TOKEN=changed-synthetic-value\n"); + } + }, + }, + }) + ).rejects.toThrow("input changed"); + expect(await readFile(composePath, "utf8")).toBe(compose); + await expect( + apply({ project, snapshots, acceptTrackedChanges: true }) + ).rejects.toThrow("input changed"); + expect(await readFile(composePath, "utf8")).toBe(compose); + await writeFile(envPath, original); + expect( + (await apply({ project, snapshots, acceptTrackedChanges: true })).status + ).toBe("applied"); }); test("retry sweeps only receipted staging artifacts and retains unknown contents", async () => { @@ -1074,22 +1253,39 @@ test("rollback retains a new shared key used by another checkout", async () => { }); test("SIGKILL at every journal/publication boundary resumes without losing an overlay value", async () => { - for (const scenario of ["plain", "encrypted"] as const) { + const composePath = join(project.projectDir, "docker-compose.yml"); + const originalCompose = await readFile(composePath, "utf8"); + for (const scenario of ["plain", "repair", "encrypted"] as const) { + await writeFile( + composePath, + scenario === "repair" + ? `${originalCompose} env_file: .env\n` + : originalCompose + ); if (scenario === "encrypted") { await addLegacySecret(); } const sources = await projectFiles(); - const preview = await previewEnvMigration(project); + const preview = await previewEnvMigration({ + ...project, + repairCompose: scenario !== "repair", + }); const count = preview.files.filter( (file) => file.action !== "removable" ).length; const boundaries = [ "prepared:-1", - ...Array.from({ length: count }, (_, index) => [ - `before_publish:${index}`, - `published:${index}`, - `recorded:${index}`, - ]).flat(), + ...Array.from( + { length: scenario === "repair" ? 1 : count }, + (_, localIndex) => { + const index = scenario === "repair" ? count : localIndex; + return [ + `before_publish:${index}`, + `published:${index}`, + `recorded:${index}`, + ]; + } + ).flat(), "verified:-1", "complete:-1", ]; @@ -1116,6 +1312,20 @@ test("SIGKILL at every journal/publication boundary resumes without losing an ov }); } const dataHome = join(selected, "hack-home"); + if (scenario === "repair") { + const previousHome = process.env.HACK_HOME; + process.env.HACK_HOME = dataHome; + try { + await apply({ + project: selectedProject, + snapshots: fixtureEnvSnapshots(selected), + acceptTrackedChanges: true, + repairCompose: false, + }); + } finally { + restoreEnv("HACK_HOME", previousHome); + } + } const child = Bun.spawn( [ process.execPath, @@ -1188,6 +1398,14 @@ test("SIGKILL at every journal/publication boundary resumes without losing an ov expect( await readFile(join(selectedProject.projectDir, ".env"), "utf8") ).toContain(SENTINEL); + if (scenario === "repair") { + expect( + await readFile( + join(selectedProject.projectDir, "docker-compose.yml"), + "utf8" + ) + ).not.toContain("env_file"); + } } finally { restoreEnv("HACK_HOME", previousHome); } diff --git a/tests/env-migration-provider.test.ts b/tests/env-migration-provider.test.ts index c8d3b3344..8d568fba9 100644 --- a/tests/env-migration-provider.test.ts +++ b/tests/env-migration-provider.test.ts @@ -108,6 +108,17 @@ async function runMigrationCli( args: readonly string[], { json = true }: { readonly json?: boolean } = {} ) { + return await runCliCommand(projectRoot, [ + "env", + "migrate", + "--path", + projectRoot, + ...(json ? ["--json"] : []), + ...args, + ]); +} + +async function runCliCommand(projectRoot: string, args: readonly string[]) { if (compiled && !isAbsolute(compiled)) { throw new Error("Compiled test executable must be absolute"); } @@ -119,15 +130,7 @@ async function runMigrationCli( resolve(import.meta.dir, "..", "index.ts"), ]; const result = await children.run({ - cmd: [ - ...command, - "env", - "migrate", - "--path", - projectRoot, - ...(json ? ["--json"] : []), - ...args, - ], + cmd: [...command, ...args], cwd: projectRoot, env: privateCliEnv({ root, @@ -171,6 +174,51 @@ function pathField(data: Record, name: string): string { return path; } +test("production CLI explicitly repairs Compose after implicit env exec, then rolls the whole migration back", async () => { + const project = await fixture(); + const composePath = join(project.projectDir, "docker-compose.yml"); + const compose = + "services:\n api:\n image: alpine:3.22\n env_file: .env\n"; + await writeFile(composePath, compose); + const implicit = await runCliCommand(project.projectRoot, [ + "env", + "exec", + "--path", + project.projectRoot, + "--", + "/usr/bin/true", + ]); + expect(implicit.exitCode, implicit.stderr).toBe(0); + expect(await readFile(composePath, "utf8")).toBe(compose); + const preview = await cli(project.projectRoot, []); + expect(preview.exitCode, preview.stderr).toBe(0); + expect(preview.envelope.data.status).toBe("ready"); + expect(preview.envelope.data.files).toContainEqual({ + path: composePath, + action: "change", + keys: [], + tracked: false, + }); + const applied = await cli(project.projectRoot, ["--apply"]); + expect(applied.exitCode, applied.stderr).toBe(0); + expect(applied.envelope.data.status).toBe("applied"); + expect(await readFile(composePath, "utf8")).not.toContain("env_file"); + const repeated = await cli(project.projectRoot, ["--apply"]); + expect(repeated.envelope.data.status).toBe("already_complete"); + const rollback = await cli(project.projectRoot, [ + "--rollback-manifest", + pathField(applied.envelope.data, "manifestPath"), + ]); + expect(rollback.exitCode, rollback.stderr).toBe(0); + expect(await readFile(composePath, "utf8")).toBe(compose); + expect(await readFile(join(project.projectDir, ".env"), "utf8")).toContain( + sentinel + ); + expect(await (await resolveSecretStore(project)).get({ key: "SECRET" })).toBe( + sentinel + ); +}); + for (const damage of ["manifest", "missing-blob", "changed-blob"] as const) { test(`resumed production apply refuses ${damage} damage before another publication`, async () => { const project = await fixture(); diff --git a/tests/env-migration-tracked-consent.test.ts b/tests/env-migration-tracked-consent.test.ts index 57dc7ead7..798557c81 100644 --- a/tests/env-migration-tracked-consent.test.ts +++ b/tests/env-migration-tracked-consent.test.ts @@ -170,6 +170,54 @@ function runDoctor(opts: { } for (const backend of ["compose", "native"] as const) { + test( + `${backend} doctor repairs tracked Compose after implicit migration only with fresh consent`, + async () => { + const project = await createProject({ compose: COMPOSE_WITH_ENV_FILE }); + const implicit = await runImplicit(project); + expect(implicit.exitCode, implicit.output).toBe(0); + expect(implicit.output).not.toContain(VALUE); + const before = await status(project); + const localPath = join(project.projectDir, "hack.env.local.yaml"); + const local = await readFile(localPath, "utf8"); + const refused = await runDoctor({ project, backend, accept: false }); + expect(refused.exitCode).toBe(1); + expect(refused.output).toContain("--accept-tracked-env-changes"); + expect(refused.output).not.toContain(VALUE); + expect(await status(project)).toEqual(before); + expect(await readFile(localPath, "utf8")).toBe(local); + const repaired = await runDoctor({ project, backend, accept: true }); + // Compose's intentionally unreachable Docker remains an independent doctor error. + expect(repaired.exitCode, repaired.output).toBe( + backend === "compose" ? 1 : 0 + ); + expect(repaired.output).toContain("Env migration completed."); + if (backend === "compose") { + expect(repaired.output).toContain("repair retained Compose"); + } + expect(repaired.output).not.toContain(VALUE); + expect( + await readFile(join(project.projectDir, "docker-compose.yml"), "utf8") + ).not.toContain("env_file"); + expect(await readFile(localPath, "utf8")).toBe(local); + expect(await readFile(join(project.projectDir, ".env"), "utf8")).toBe( + `${KEY}=${VALUE}\n` + ); + expect(await status(project)).toEqual([ + " M .hack/docker-compose.yml", + "?? .hack/.gitignore", + "?? .hack/hack.env.default.yaml", + ]); + const repeated = await runDoctor({ project, backend, accept: false }); + expect(repeated.exitCode, repeated.output).toBe( + backend === "compose" ? 1 : 0 + ); + expect(repeated.output).toContain("already completed"); + expect(repeated.output).not.toContain(VALUE); + }, + CLI_TEST_TIMEOUT_MS + ); + test( `${backend} doctor without a prompt refuses to change tracked files and changes nothing`, async () => { diff --git a/tests/models/tla/env-migration/COVERAGE.md b/tests/models/tla/env-migration/COVERAGE.md index 8620263f8..e03bb2da3 100644 --- a/tests/models/tla/env-migration/COVERAGE.md +++ b/tests/models/tla/env-migration/COVERAGE.md @@ -1,21 +1,28 @@ # Env migration journal -`Migration.tla` bounds publication to two files. Publish and receipt are separate +`Migration.tla` bounds each publication phase to two files. Publish and receipt are separate actions: an invocation can crash between them, then resume from retained bytes. Completion, successful project startup and a later explicit cleanup request are -distinct. Snapshot damage while legacy files remain blocks cleanup. Live targets +distinct. An implicit migration can defer a later explicit Compose repair; +`RequestRepair` starts that phase under intact snapshot coverage and invalidates +the earlier successful-run proof. The counters reset for the abstract later +phase; already completed layer publications remain outside that phase in the +implementation. Snapshot damage while legacy files remain blocks repair and cleanup. Live targets cannot be removed. The unsafe control restores same-run deletion and must fail `NoSameRunCleanup` at `UnsafeRemoveDuringApply`. -The positive TLC exploration exhausts 37 distinct states (two initial states -cover ordinary and live targets). The negative witness has zero publications, +The positive TLC exploration exhausts 74 distinct states (four initial states +cover ordinary/live targets and deferred/completed Compose repair). The negative witness has zero publications, an intact snapshot, a retained applying journal, no later cleanup request and a removed legacy target. Both live-target initial states admit that witness. +The `stale-repair-proof` negative control retains an earlier successful-run proof +when repair starts and fails `FreshRunProof` at `RequestRepair`. | Action / invariant | Source and regression | | --- | --- | | Prepare / NoLostOriginal | `prepareConsistent` and `prepareJournal` bind immutable payloads and source checksums to a checked private snapshot before project writes. Preview/apply parity and damaged-blob refusal use real files. | | Publish, Record, Crash, Resume | `applyEnvMigration` publishes before recording provider/journal receipts; matching intended bytes are adopted on retry. `env-migration-journal.test.ts` SIGKILLs an owned child at every journal/publication boundary. | | Finish / CompleteHasAllSteps | Resolution succeeds before the complete journal is published; completed retry is a no-op. | +| RequestRepair / FreshRunProof | `prepareDeferredComposeRepair` and `extendComposeRepair` admit only an unchanged, already snapshotted Compose target after implicit migration. Existing layer publications are kept, new input checksums are recorded, and prior startup proof is cleared. Real-file tests prove preview/apply parity, tracked consent, missing-value/backup refusal, tighter modes and rollback. Six additional SIGKILL boundaries cover the later repair. | | SuccessfulRun, RequestCleanup, RemoveLegacy / NoSameRunCleanup | `recordEnvMigrationProjectRun` and `cleanupEnvMigration` require a later request, unchanged successful-run layer revision, intact snapshot and fresh delivery. Migration, including implicit routes, retains originals. | | RefuseCleanup | A known delivery or live-input refusal before unlink returns to completed migration; unknown or partial effects require rollback. A real-file regression changes delivery immediately before removal, checks that every legacy file stays and the journal remains complete, then corrects delivery and successfully retries. The model abstracts the pre-removal refusal only; it does not model receipt failures or partial cleanup. | | LiveInputsRetained | Existing #769 identity guard remains at candidate selection and immediately before deletion. The live-key regression uses real files. | diff --git a/tests/models/tla/env-migration/Migration.tla b/tests/models/tla/env-migration/Migration.tla index d8b701349..479963cbd 100644 --- a/tests/models/tla/env-migration/Migration.tla +++ b/tests/models/tla/env-migration/Migration.tla @@ -1,55 +1,62 @@ ----------------------------- MODULE Migration ----------------------------- EXTENDS Naturals, TLC -CONSTANTS Steps, UnsafeCleanup +CONSTANTS Steps, UnsafeCleanup, UnsafeRepairProof VARIABLES phase, published, recorded, snapshot, legacy, complete, runProof, - separate, live + separate, live, deferred vars == <> + separate, live, deferred>> Init == /\ phase = "preview" /\ published = 0 /\ recorded = 0 /\ snapshot = FALSE /\ legacy = TRUE /\ complete = FALSE /\ runProof = FALSE /\ separate = FALSE /\ live \in BOOLEAN + /\ deferred \in BOOLEAN Prepare == /\ phase \in {"preview", "rolled_back"} /\ phase' = "apply" /\ snapshot' = TRUE - /\ UNCHANGED <> + /\ UNCHANGED <> Publish == /\ phase = "apply" /\ published = recorded /\ published < Steps /\ published' = published + 1 - /\ UNCHANGED <> + /\ UNCHANGED <> Record == /\ phase = "apply" /\ recorded < published /\ recorded' = published - /\ UNCHANGED <> + /\ UNCHANGED <> Crash == /\ phase = "apply" /\ phase' = "idle" - /\ UNCHANGED <> + /\ UNCHANGED <> Resume == /\ phase = "idle" /\ ~complete /\ snapshot /\ phase' = "apply" - /\ UNCHANGED <> + /\ UNCHANGED <> Finish == /\ phase = "apply" /\ recorded = Steps /\ complete' = TRUE /\ phase' = "idle" - /\ UNCHANGED <> + /\ UNCHANGED <> +RequestRepair == /\ phase = "idle" /\ complete /\ deferred /\ snapshot /\ legacy + /\ phase' = "apply" /\ complete' = FALSE /\ deferred' = FALSE + /\ published' = 0 /\ recorded' = 0 /\ separate' = FALSE + /\ runProof' = IF UnsafeRepairProof THEN runProof ELSE FALSE + /\ UNCHANGED <> SuccessfulRun == /\ phase = "idle" /\ complete /\ ~runProof /\ runProof' = TRUE - /\ UNCHANGED <> + /\ UNCHANGED <> DamageBackup == /\ phase = "idle" /\ complete /\ legacy /\ snapshot /\ snapshot' = FALSE - /\ UNCHANGED <> + /\ UNCHANGED <> RequestCleanup == /\ phase = "idle" /\ complete /\ runProof /\ snapshot /\ ~live /\ phase' = "cleanup" /\ separate' = TRUE - /\ UNCHANGED <> + /\ UNCHANGED <> RemoveLegacy == /\ phase = "cleanup" /\ snapshot /\ runProof /\ ~live /\ legacy' = FALSE /\ phase' = "idle" - /\ UNCHANGED <> + /\ UNCHANGED <> RefuseCleanup == /\ phase = "cleanup" /\ legacy /\ phase' = "idle" - /\ UNCHANGED <> + /\ UNCHANGED <> UnsafeRemoveDuringApply == /\ UnsafeCleanup /\ phase = "apply" /\ legacy /\ published = 0 /\ legacy' = FALSE - /\ UNCHANGED <> + /\ UNCHANGED <> Rollback == /\ phase = "idle" /\ snapshot /\ phase' = "rolled_back" /\ published' = 0 /\ recorded' = 0 /\ legacy' = TRUE /\ complete' = FALSE /\ runProof' = FALSE - /\ separate' = FALSE /\ UNCHANGED <> + /\ separate' = FALSE /\ deferred' \in BOOLEAN + /\ UNCHANGED <> -Next == Prepare \/ Publish \/ Record \/ Crash \/ Resume \/ Finish \/ SuccessfulRun +Next == Prepare \/ Publish \/ Record \/ Crash \/ Resume \/ Finish \/ RequestRepair \/ SuccessfulRun \/ DamageBackup \/ RequestCleanup \/ RemoveLegacy \/ RefuseCleanup \/ UnsafeRemoveDuringApply \/ Rollback Spec == Init /\ [][Next]_vars @@ -57,9 +64,11 @@ TypeOK == /\ phase \in {"preview", "apply", "idle", "cleanup", "rolled_back"} /\ published \in 0..Steps /\ recorded \in 0..Steps /\ snapshot \in BOOLEAN /\ legacy \in BOOLEAN /\ complete \in BOOLEAN /\ runProof \in BOOLEAN /\ separate \in BOOLEAN /\ live \in BOOLEAN + /\ deferred \in BOOLEAN NoSameRunCleanup == ~legacy => separate NoLostOriginal == legacy \/ snapshot LiveInputsRetained == live => legacy ReceiptsFollowPublication == recorded <= published CompleteHasAllSteps == complete => recorded = Steps +FreshRunProof == runProof => complete ============================================================================= diff --git a/tests/models/tla/env-migration/negative.cfg b/tests/models/tla/env-migration/negative.cfg index 2869ccc5c..e39595427 100644 --- a/tests/models/tla/env-migration/negative.cfg +++ b/tests/models/tla/env-migration/negative.cfg @@ -1,5 +1,6 @@ CONSTANTS Steps = 2 UnsafeCleanup = TRUE + UnsafeRepairProof = FALSE SPECIFICATION Spec -INVARIANTS TypeOK NoSameRunCleanup NoLostOriginal LiveInputsRetained ReceiptsFollowPublication CompleteHasAllSteps +INVARIANTS TypeOK NoSameRunCleanup NoLostOriginal LiveInputsRetained ReceiptsFollowPublication CompleteHasAllSteps FreshRunProof CHECK_DEADLOCK FALSE diff --git a/tests/models/tla/env-migration/positive.cfg b/tests/models/tla/env-migration/positive.cfg index ee5550271..460eec9d7 100644 --- a/tests/models/tla/env-migration/positive.cfg +++ b/tests/models/tla/env-migration/positive.cfg @@ -1,5 +1,6 @@ CONSTANTS Steps = 2 UnsafeCleanup = FALSE + UnsafeRepairProof = FALSE SPECIFICATION Spec -INVARIANTS TypeOK NoSameRunCleanup NoLostOriginal LiveInputsRetained ReceiptsFollowPublication CompleteHasAllSteps +INVARIANTS TypeOK NoSameRunCleanup NoLostOriginal LiveInputsRetained ReceiptsFollowPublication CompleteHasAllSteps FreshRunProof CHECK_DEADLOCK FALSE diff --git a/tests/models/tla/env-migration/stale-repair-proof.cfg b/tests/models/tla/env-migration/stale-repair-proof.cfg new file mode 100644 index 000000000..ba051da2d --- /dev/null +++ b/tests/models/tla/env-migration/stale-repair-proof.cfg @@ -0,0 +1,6 @@ +CONSTANTS Steps = 2 + UnsafeCleanup = FALSE + UnsafeRepairProof = TRUE +SPECIFICATION Spec +INVARIANTS TypeOK NoSameRunCleanup NoLostOriginal LiveInputsRetained ReceiptsFollowPublication CompleteHasAllSteps FreshRunProof +CHECK_DEADLOCK FALSE From 149cc6aca1183635e66145a308cc35eaffe08f74 Mon Sep 17 00:00:00 2001 From: Kit <344842+roodboi@users.noreply.github.com> Date: Sat, 14 Nov 2026 07:05:46 +0000 Subject: [PATCH 18/32] fix(env): allow cleanup after Doctor tightens file permissions --- docs/reference/env-migration.md | 6 ++++ src/lib/env-migration.ts | 10 ++++-- tests/env-migration-journal.test.ts | 53 ++++++++++++++++++++++++++++ tests/env-migration-provider.test.ts | 31 ++++++++++++++++ 4 files changed, 98 insertions(+), 2 deletions(-) diff --git a/docs/reference/env-migration.md b/docs/reference/env-migration.md index 0ac8e35a5..59a99c809 100644 --- a/docs/reference/env-migration.md +++ b/docs/reference/env-migration.md @@ -160,6 +160,12 @@ Migration never includes cleanup. A later explicit cleanup requires: Current project/shared keys, configuration and YAML layers, including aliases and hard links, cannot be removed. +Doctor's permission tightening (for example, a retained dotenv file from 0644 +to 0600) does not invalidate cleanup when its bytes and size are unchanged. +Both source checks still refuse changed content or wider permissions; original +snapshot coverage, fresh writer-locked receipts and the live-input guard remain +required. Rollback retains the original snapshot's bytes and modes. + Legacy stores leave before their keys. Confirmed store removal also repairs its config pointer. A partial cleanup preserves confirmed removals in the journal and reports every confirmed removal even if pointer repair or snapshot receipt diff --git a/src/lib/env-migration.ts b/src/lib/env-migration.ts index cc1a3270d..c02f17f7f 100644 --- a/src/lib/env-migration.ts +++ b/src/lib/env-migration.ts @@ -1261,7 +1261,10 @@ async function cleanupLocked( manifestPath: journal.manifestPath, files: journal.sources, }); - await assertSources(journal, { allowLayerChanges: true }); + await assertSources(journal, { + allowLayerChanges: true, + allowTighterModes: true, + }); const context = await acquireLegacyEnvCleanupContext({ ...project, overlayNames: journal.overlayNames, @@ -1300,7 +1303,10 @@ async function cleanupLocked( }, }; } - await assertSources(journal, { allowLayerChanges: true }); + await assertSources(journal, { + allowLayerChanges: true, + allowTighterModes: true, + }); journal = { ...journal, state: "cleaning" }; await writeMetadata(path, journal); let removed: readonly string[] = []; diff --git a/tests/env-migration-journal.test.ts b/tests/env-migration-journal.test.ts index 53f9d83f7..67b526847 100644 --- a/tests/env-migration-journal.test.ts +++ b/tests/env-migration-journal.test.ts @@ -492,6 +492,59 @@ test("cleanup is explicit, rechecks delivery, and requires intact snapshot paylo expect(await Bun.file(result.manifestPath).exists()).toBe(true); }); +for (const change of [ + "permissions only", + "changed bytes", + "wider permissions", +] as const) { + test(`cleanup after migration handles ${change} without adopting byte or access drift`, async () => { + const envPath = join(project.projectDir, ".env"); + await chmod(envPath, 0o644); + const snapshots = fixtureEnvSnapshots(root); + const migrated = await apply({ + project, + snapshots, + acceptTrackedChanges: true, + }); + await chmod(envPath, change === "wider permissions" ? 0o666 : 0o600); + if (change === "changed bytes") { + await writeFile(envPath, "TOKEN=changed-synthetic-value\n"); + } + await recordEnvMigrationProjectRun(project); + if (change !== "permissions only") { + await expect( + cleanupEnvMigration({ project, snapshots, acceptTrackedChanges: true }) + ).rejects.toThrow("input changed"); + expect(await Bun.file(envPath).exists()).toBe(true); + expect( + JSON.parse(await readFile(migrated.journalPath, "utf8")).state + ).toBe("complete"); + return; + } + const result = await cleanupEnvMigration({ + project, + snapshots, + acceptTrackedChanges: true, + }); + expect(result.removed).toContain(envPath); + expect(await Bun.file(envPath).exists()).toBe(false); + expect( + (await resolveProjectEnvConfig({ ...project, envName: null }))?.globalEnv + .TOKEN + ).toBe(SENTINEL); + await rollbackEnvMigration({ + project, + snapshots, + manifestPath: migrated.manifestPath, + }); + expect(await readFile(envPath, "utf8")).toBe(`TOKEN=${SENTINEL}\n`); + expect((await lstat(envPath)).mode & 0o777).toBe(0o644); + expect(await readFile(migrated.journalPath, "utf8")).not.toContain( + SENTINEL + ); + }); +} + test("rollback restores original bytes and refuses a later user edit", async () => { const before = await projectFiles(); const snapshots = fixtureEnvSnapshots(root); diff --git a/tests/env-migration-provider.test.ts b/tests/env-migration-provider.test.ts index 8d568fba9..b327357ad 100644 --- a/tests/env-migration-provider.test.ts +++ b/tests/env-migration-provider.test.ts @@ -1,5 +1,6 @@ import { afterEach, beforeEach, expect, test } from "bun:test"; import { + chmod, lstat, mkdir, mkdtemp, @@ -14,6 +15,7 @@ import { dirname, isAbsolute, join, resolve } from "node:path"; import { applyEnvMigration, envMigrationJournalPath, + recordEnvMigrationProjectRun, } from "../src/lib/env-migration.ts"; import { bindEnvMigrationSnapshots } from "../src/lib/env-migration-snapshot.ts"; import { isRecord } from "../src/lib/guards.ts"; @@ -219,6 +221,35 @@ test("production CLI explicitly repairs Compose after implicit env exec, then ro ); }); +test("production cleanup accepts Doctor permission tightening while preserving original rollback bytes", async () => { + const project = await fixture(); + const envPath = join(project.projectDir, ".env"); + await chmod(envPath, 0o644); + expect((await lstat(envPath)).mode & 0o777).toBe(0o644); + const migrated = await cli(project.projectRoot, ["--apply"]); + expect(migrated.exitCode, migrated.stderr).toBe(0); + const doctor = await runCliCommand(project.projectRoot, [ + "doctor", + "--fix", + "--path", + project.projectRoot, + ]); + // The fixture intentionally has no Docker or other host tooling; hygiene still runs. + expect(doctor.exitCode).toBe(1); + expect((await lstat(envPath)).mode & 0o777).toBe(0o600); + expect(await readFile(envPath, "utf8")).toBe(`TOKEN=${sentinel}\n`); + await recordEnvMigrationProjectRun(project); + const cleaned = await cli(project.projectRoot, ["--cleanup", "--apply"]); + expect(cleaned.exitCode, cleaned.stderr).toBe(0); + expect(await Bun.file(envPath).exists()).toBe(false); + const rollback = await cli(project.projectRoot, [ + "--rollback-manifest", + pathField(migrated.envelope.data, "manifestPath"), + ]); + expect(rollback.exitCode, rollback.stderr).toBe(0); + expect(await readFile(envPath, "utf8")).toBe(`TOKEN=${sentinel}\n`); +}); + for (const damage of ["manifest", "missing-blob", "changed-blob"] as const) { test(`resumed production apply refuses ${damage} damage before another publication`, async () => { const project = await fixture(); From ffb5db8b74179a1f1e0ecf25a52659b285d964b2 Mon Sep 17 00:00:00 2001 From: Kit <344842+roodboi@users.noreply.github.com> Date: Sat, 14 Nov 2026 07:21:54 +0000 Subject: [PATCH 19/32] fix(env): report unavailable repairs and permission recovery --- docs/env.md | 3 + docs/reference/env-migration.md | 20 +++- docs/versions/v5/guides/upgrade-from-v4.md | 4 +- src/commands/doctor.ts | 3 + src/lib/env-migration.ts | 83 +++++++++++++--- src/lib/project-env-config.ts | 6 ++ tests/env-migration-journal.test.ts | 106 ++++++++++++++++++++- tests/env-migration-provider.test.ts | 66 +++++++++++++ 8 files changed, 276 insertions(+), 15 deletions(-) diff --git a/docs/env.md b/docs/env.md index ee4f08125..ca2621ee6 100644 --- a/docs/env.md +++ b/docs/env.md @@ -1376,6 +1376,9 @@ filesystem aliases. Use `--rollback-manifest ` to restore the snapshot; later edits are refused. See the [versioned preview and recovery contract](reference/env-migration.md) for the JSON shape, journal boundaries and partial-cleanup recovery. +Rollback also restores original file modes, including modes Doctor may have +tightened since migration. Its result warns to run `hack doctor --fix --path +` after recovery to review those permissions. Commands that need env values, such as `hack env exec`, can also migrate a project that has `hack.env.json` and no env layers. These implicit migrations diff --git a/docs/reference/env-migration.md b/docs/reference/env-migration.md index 59a99c809..b731d1d01 100644 --- a/docs/reference/env-migration.md +++ b/docs/reference/env-migration.md @@ -36,7 +36,8 @@ The preview's `data` has these stable fields: | --- | --- | | `version` | Integer `1`; consumers must reject unsupported versions. | | `project` | `projectRoot`, `projectDir`, `projectName` and `serviceNames`; paths are absolute and canonical. | -| `status` | `ready` for unfinished migration work; `complete` when migration has completed; `recovery_required` for interrupted cleanup or rollback. | +| `status` | `ready` for unfinished migration work; `complete` when migration has completed; `recovery_required` for interrupted cleanup/rollback or an unavailable deferred Compose repair. | +| `composeRepair` | Optional `"unavailable"` when completed env migration still has Compose references but their repair cannot be verified. The preview exits 0 with no file actions; apply still refuses. Existing env layers remain usable. | | `action` | `preview`, or `cleanup` with `--cleanup`. | | `dryRun` | Always `true` for a preview. | | `files` | Ordered array of `{ path, action, keys, tracked }`. | @@ -113,6 +114,16 @@ The phase retains all legacy files and clears an earlier startup proof, so cleanup still needs a later successful full-project run. Once the repair is complete, another explicit apply is a no-op. +If retained legacy inputs were edited or cannot be checked, default preview and +Doctor's plan report the deferred repair as unavailable instead of failing the +read-only report. They do not infer new write or cleanup authority. Keep the +legacy copies and review them privately against current layers and Compose +references. Editing those references yourself is an alternative to restoring +pre-migration inputs; a later explicit apply records that no repair remains, +without rewriting the project. Such manual edits stay outside the journal and +normal cleanup checks still apply. Snapshot rollback with `--preserve-changes` +remains an explicit recovery option, not a prerequisite to inspect the plan. + Migration refuses a Git-tracked project key path even when its file is absent from the worktree, including case-folded aliases such as `.Hack.Secret.Key`. Tracked-change consent does not allow publishing key bytes at that path. @@ -188,6 +199,13 @@ select or merge values. Rollback can resume an incomplete restoration. An incomplete rollback exits 1 in both JSON and plain modes, preserving its per-file restoration details and leaving the journal in `rolling_back`. Project commands remain blocked until the same rollback completes; a restored result exits 0. +Rollback accepts byte-identical permission tightening without requiring +`--preserve-changes`. It restores the snapshot's original modes as well as its +bytes, so it can undo Doctor's earlier permission repair. Every rollback result +includes a fixed `warnings` entry to run `hack doctor --fix --path ` +after recovery to review the restored permissions; both JSON and plain output +carry it. The snapshot's permission metadata is never rewritten to hide that +original state. Keep the manifest, blobs and journal together until the upgrade is accepted. Neither file-level atomic writes nor cooperative locks protect against an unrelated process replacing files outside Hack's locks. diff --git a/docs/versions/v5/guides/upgrade-from-v4.md b/docs/versions/v5/guides/upgrade-from-v4.md index 2832c2267..0d078b3f7 100644 --- a/docs/versions/v5/guides/upgrade-from-v4.md +++ b/docs/versions/v5/guides/upgrade-from-v4.md @@ -515,7 +515,9 @@ key, cannot be removed. Changed layers require another full-project run. Rollback uses `env migrate --rollback-manifest ` with the path returned by apply. It refuses later edits unless `--preserve-changes` is explicit; that option retains those bytes privately and reports their recovery paths. An -incomplete rollback exits nonzero and must be resumed. See the +incomplete rollback exits nonzero and must be resumed. +Rollback also restores original file permissions and reports a warning to run +`hack doctor --fix --path /absolute/project` after recovery to review them. See the [preview and recovery contract](https://github.com/hack-dance/hack/blob/a87b13bf7fdf01811efe33e97dfc27e1b39e87a3/docs/reference/env-migration.md) for the versioned JSON and partial-cleanup recovery rules. diff --git a/src/commands/doctor.ts b/src/commands/doctor.ts index 185d519d1..59bd95834 100644 --- a/src/commands/doctor.ts +++ b/src/commands/doctor.ts @@ -4325,6 +4325,9 @@ async function buildEnvMigrationPlanStep(opts: { composeFile, }), }); + if (preview.composeRepair === "unavailable") { + return "Compose repair unavailable: keep legacy copies and review their values privately against current layers and Compose references, or explicitly restore the snapshot with --preserve-changes."; + } if ( preview.files.some( (file) => file.action === "change" && file.path === composeFile diff --git a/src/lib/env-migration.ts b/src/lib/env-migration.ts index c02f17f7f..00fa1d545 100644 --- a/src/lib/env-migration.ts +++ b/src/lib/env-migration.ts @@ -38,6 +38,7 @@ import { assertMachineLocalFileGitPrivate, assertProjectEnvKeyUntracked, assertProjectEnvLayerPaths, + inspectLegacyComposeEnvFileReferences, isGitTrackedOrUnknownProjectPath, LegacyEnvCleanupError, type LegacyProjectEnvMigration, @@ -75,6 +76,8 @@ export type EnvMigrationPreview = { }[]; readonly journalPath: string; readonly manifestPath: string | null; + /** Completed env layers remain usable while a deferred repair needs manual review. */ + readonly composeRepair?: "unavailable"; }; type Publication = EnvMigrationFileState & { @@ -401,11 +404,10 @@ export async function previewEnvMigration( const path = envMigrationJournalPath(project.projectRoot); const journal = await readJournal(path, project); if (needsComposeRepair(journal, input.repairCompose)) { - const prepared = await prepareDeferredComposeRepair(project, journal); - return { - ...(await previewPrepared(project, prepared.prepared, path)), - manifestPath: journal.manifestPath, - }; + const preview = await previewDeferredComposeRepair(project, journal, path); + if (preview) { + return preview; + } } if (journal && journal.state !== "rolled_back") { const files: EnvMigrationPreview["files"][number][] = []; @@ -462,6 +464,32 @@ export async function previewEnvMigration( ); } +async function previewDeferredComposeRepair( + project: EnvMigrationProject, + journal: Journal, + path: string +): Promise { + try { + const prepared = await prepareDeferredComposeRepair(project, journal); + return prepared + ? { + ...(await previewPrepared(project, prepared.prepared, path)), + manifestPath: journal.manifestPath, + } + : null; + } catch { + return { + version: 1, + project, + status: "recovery_required", + composeRepair: "unavailable", + files: [], + journalPath: path, + manifestPath: journal.manifestPath, + }; + } +} + function previewStatus(journal: Journal): EnvMigrationPreview["status"] { if (["cleaning", "rolling_back"].includes(journal.state)) { return "recovery_required"; @@ -666,7 +694,20 @@ function needsComposeRepair( async function prepareDeferredComposeRepair( project: EnvMigrationProject, journal: Journal -): Promise { +): Promise { + const composeFile = join(project.projectDir, "docker-compose.yml"); + const before = (await readState(composeFile)).state; + const references = await inspectLegacyComposeEnvFileReferences({ + ...project, + composeFile, + strict: true, + }); + if (!same(before, (await readState(composeFile)).state)) { + throw refuse(); + } + if (references.length === 0) { + return null; + } await assertSources(journal, { allowLayerChanges: true, allowTighterModes: true, @@ -699,10 +740,13 @@ async function extendComposeRepair( snapshots: EnvMigrationSnapshots, accept: boolean ): Promise { - const { prepared, sources } = await prepareDeferredComposeRepair( - project, - journal - ); + const preparation = await prepareDeferredComposeRepair(project, journal); + if (!preparation) { + const completed: Journal = { ...journal, composeRepair: "complete" }; + await writeMetadata(path, completed); + return completed; + } + const { prepared, sources } = preparation; await assertConsent( project, prepared.files.map((file) => file.path), @@ -912,6 +956,7 @@ export async function applyEnvMigration(input: { await assertConsent(project, changed, input.acceptTrackedChanges); if ( preparation && + (!existing || existing.state === "rolled_back") && preparation.prepared.files.length === 0 && preparation.prepared.removable.length === 0 ) { @@ -1484,6 +1529,16 @@ async function rollbackLocked( ) { expected = journal.cleanupConfig; } + if ( + matchesMigrationSource({ + current, + source: expected, + write: undefined, + allowTighterModes: true, + }) + ) { + expected = current; + } expectedCurrent.push(expected); } journal = { @@ -1502,11 +1557,17 @@ async function rollbackLocked( ], }; await writeMetadata(path, journal); - const result = await input.snapshots.rollback({ + const restored = await input.snapshots.rollback({ manifestPath: journal.manifestPath, expectedCurrent, preserveChanges: input.preserveChanges, }); + const result = { + ...restored, + warnings: [ + "Rollback restores original file permissions too. After recovery, run hack doctor --fix --path to review them.", + ], + }; if (result.status !== "restored") { return result; } diff --git a/src/lib/project-env-config.ts b/src/lib/project-env-config.ts index eaef8e5c2..86cffbd45 100644 --- a/src/lib/project-env-config.ts +++ b/src/lib/project-env-config.ts @@ -5877,9 +5877,15 @@ export async function discoverComposeServiceNames(opts: { export async function inspectLegacyComposeEnvFileReferences(opts: { readonly composeFile: string; readonly projectDir: string; + readonly strict?: boolean; }): Promise { const parsed = await readComposeFile({ composeFile: opts.composeFile }); if (!parsed) { + if (opts.strict) { + throw new ClosedCliError( + "Cannot inspect Compose repair: the configuration is unavailable or malformed; values omitted." + ); + } return []; } diff --git a/tests/env-migration-journal.test.ts b/tests/env-migration-journal.test.ts index 67b526847..8711e17d8 100644 --- a/tests/env-migration-journal.test.ts +++ b/tests/env-migration-journal.test.ts @@ -12,6 +12,7 @@ import { } from "node:fs/promises"; import { tmpdir } from "node:os"; import { dirname, join, relative } from "node:path"; +import { buildDoctorRemediationPlanLines } from "../src/commands/doctor.ts"; import { applyEnvMigration, cleanupEnvMigration, @@ -265,7 +266,11 @@ test("journalled implicit migration keeps undeclared Compose overlay inputs; exp await readFile(envMigrationJournalPath(project.projectRoot), "utf8") ).state ).toBe("complete"); - await expect(previewEnvMigration(project)).rejects.toThrow(".env.staging"); + expect(await previewEnvMigration(project)).toMatchObject({ + status: "recovery_required", + composeRepair: "unavailable", + files: [], + }); await expect( applyEnvMigration({ project, @@ -333,6 +338,88 @@ test("explicit migration after implicit completion previews only Compose repair expectRestored(await projectFiles(), before); }); +test("deferred repair preview and Doctor plan report unavailable legacy drift without changing or blocking current layers", async () => { + await writeFile( + join(project.projectDir, "docker-compose.yml"), + "services:\n api:\n image: alpine:3.22\n env_file: .env\n" + ); + await migrateLegacyEnvForCommand({ ...project, materialize: false }); + await writeFile(join(project.projectDir, ".env"), `TOKEN=${QA}\n`); + const before = await projectFiles(); + const path = envMigrationJournalPath(project.projectRoot); + const journal = await readFile(path, "utf8"); + const preview = await previewEnvMigration(project); + expect(preview).toMatchObject({ + status: "recovery_required", + composeRepair: "unavailable", + files: [], + }); + expect(JSON.stringify(preview)).not.toContain(QA); + const plan = await buildDoctorRemediationPlanLines({ + startDir: project.projectRoot, + migrateEnvConfig: true, + }); + expect(plan.some((line) => line.includes("Compose repair unavailable"))).toBe( + true + ); + expect( + (await resolveProjectEnvConfig({ ...project, envName: null }))?.globalEnv + .TOKEN + ).toBe(SENTINEL); + await expect( + apply({ + project, + snapshots: envMigrationSnapshots(), + acceptTrackedChanges: true, + }) + ).rejects.toThrow("input changed"); + expect(await projectFiles()).toEqual(before); + expect(await readFile(path, "utf8")).toBe(journal); +}); + +test("explicit no-op clears deferred Compose repair after manual removal of references", async () => { + const composePath = join(project.projectDir, "docker-compose.yml"); + await writeFile( + composePath, + "services:\n api:\n image: alpine:3.22\n env_file: .env\n" + ); + await migrateLegacyEnvForCommand({ ...project, materialize: false }); + await writeFile(composePath, "services: [unclosed\n"); + expect(await previewEnvMigration(project)).toMatchObject({ + status: "recovery_required", + composeRepair: "unavailable", + }); + await expect( + apply({ + project, + snapshots: envMigrationSnapshots(), + acceptTrackedChanges: false, + }) + ).rejects.toThrow("configuration"); + await writeFile(composePath, "services:\n api:\n image: alpine:3.22\n"); + const before = await projectFiles(); + expect((await previewEnvMigration(project)).status).toBe("complete"); + const result = await apply({ + project, + snapshots: envMigrationSnapshots(), + acceptTrackedChanges: false, + }); + expect(result.status).toBe("already_complete"); + const journal = await readFile(result.journalPath, "utf8"); + expect(JSON.parse(journal).composeRepair).toBe("complete"); + expect(await projectFiles()).toEqual(before); + expect( + ( + await apply({ + project, + snapshots: envMigrationSnapshots(), + acceptTrackedChanges: false, + }) + ).status + ).toBe("already_complete"); + expect(await readFile(result.journalPath, "utf8")).toBe(journal); +}); + test("later Compose repair accepts layer additions and tighter modes, invalidates old run proof and refuses lost values", async () => { await writeFile( join(project.projectDir, "docker-compose.yml"), @@ -343,7 +430,18 @@ test("later Compose repair accepts layer additions and tighter modes, invalidate const layer = join(project.projectDir, "hack.env.local.yaml"); const original = await readFile(layer, "utf8"); await writeFile(layer, original.replace(SENTINEL, "changed-synthetic-value")); - await expect(previewEnvMigration(project)).rejects.toThrow(); + expect(await previewEnvMigration(project)).toMatchObject({ + status: "recovery_required", + composeRepair: "unavailable", + files: [], + }); + await expect( + apply({ + project, + snapshots: envMigrationSnapshots(), + acceptTrackedChanges: true, + }) + ).rejects.toThrow(); const changedLayer = original.replace( " global:\n", " global:\n EXTRA: additional-synthetic-value\n" @@ -351,12 +449,16 @@ test("later Compose repair accepts layer additions and tighter modes, invalidate expect(changedLayer).not.toBe(original); await writeFile(layer, changedLayer); await chmod(join(project.projectDir, ".env"), 0o600); + await chmod(join(project.projectDir, "docker-compose.yml"), 0o600); await apply({ project, snapshots: envMigrationSnapshots(), acceptTrackedChanges: true, }); expect(await readFile(layer, "utf8")).toBe(changedLayer); + expect( + (await lstat(join(project.projectDir, "docker-compose.yml"))).mode & 0o777 + ).toBe(0o600); await expect( cleanupEnvMigration({ project, diff --git a/tests/env-migration-provider.test.ts b/tests/env-migration-provider.test.ts index b327357ad..4fb153718 100644 --- a/tests/env-migration-provider.test.ts +++ b/tests/env-migration-provider.test.ts @@ -247,9 +247,75 @@ test("production cleanup accepts Doctor permission tightening while preserving o pathField(migrated.envelope.data, "manifestPath"), ]); expect(rollback.exitCode, rollback.stderr).toBe(0); + expect(rollback.envelope.data.warnings).toContainEqual( + expect.stringContaining("original file permissions") + ); + expect(await readFile(envPath, "utf8")).toBe(`TOKEN=${sentinel}\n`); +}); + +test("production rollback accepts a byte-identical tighter mode before cleanup and reports original-mode review", async () => { + const project = await fixture(); + const envPath = join(project.projectDir, ".env"); + await chmod(envPath, 0o644); + const migrated = await cli(project.projectRoot, ["--apply"]); + expect(migrated.exitCode, migrated.stderr).toBe(0); + await runCliCommand(project.projectRoot, [ + "doctor", + "--fix", + "--path", + project.projectRoot, + ]); + expect((await lstat(envPath)).mode & 0o777).toBe(0o600); + const rollback = await cli(project.projectRoot, [ + "--rollback-manifest", + pathField(migrated.envelope.data, "manifestPath"), + ]); + expect(rollback.exitCode, rollback.stderr).toBe(0); + expect(rollback.envelope.data.warnings).toContainEqual( + expect.stringContaining("original file permissions") + ); expect(await readFile(envPath, "utf8")).toBe(`TOKEN=${sentinel}\n`); }); +test("production previews report deferred repair unavailable without failing, printing drift values or changing files", async () => { + const project = await fixture(); + const composePath = join(project.projectDir, "docker-compose.yml"); + const compose = + "services:\n api:\n image: alpine:3.22\n env_file: .env\n"; + await writeFile(composePath, compose); + const implicit = await runCliCommand(project.projectRoot, [ + "env", + "exec", + "--path", + project.projectRoot, + "--", + "/usr/bin/true", + ]); + expect(implicit.exitCode, implicit.stderr).toBe(0); + const drift = "synthetic-drift-preview-value"; + const envPath = join(project.projectDir, ".env"); + await writeFile(envPath, `TOKEN=${drift}\n`); + const journalPath = envMigrationJournalPath(project.projectRoot); + const journal = await readFile(journalPath, "utf8"); + for (const json of [true, false]) { + const result = await runMigrationCli(project.projectRoot, [], { json }); + expect(result.exitCode, result.stderr).toBe(0); + expect(result.stdout + result.stderr).not.toContain(drift); + const parsed = JSON.parse(result.stdout); + expect(json ? parsed.data : parsed).toMatchObject({ + status: "recovery_required", + composeRepair: "unavailable", + files: [], + }); + } + const refused = await cli(project.projectRoot, ["--apply"]); + expect(refused.exitCode).not.toBe(0); + expect(refused.stdout + refused.stderr).not.toContain(drift); + expect(await readFile(journalPath, "utf8")).toBe(journal); + expect(await readFile(envPath, "utf8")).toBe(`TOKEN=${drift}\n`); + expect(await readFile(composePath, "utf8")).toBe(compose); +}); + for (const damage of ["manifest", "missing-blob", "changed-blob"] as const) { test(`resumed production apply refuses ${damage} damage before another publication`, async () => { const project = await fixture(); From 332c4d1c8abe2e2f9a0b883d5d2c12550bc0794f Mon Sep 17 00:00:00 2001 From: Kit <344842+roodboi@users.noreply.github.com> Date: Sat, 14 Nov 2026 07:36:59 +0000 Subject: [PATCH 20/32] fix(env): skip unavailable deferred repairs in Doctor --- docs/env.md | 3 +++ docs/reference/env-migration.md | 7 ++++-- src/commands/doctor.ts | 9 ++++++- tests/env-migration-provider.test.ts | 11 ++++++++ tests/env-migration-tracked-consent.test.ts | 28 +++++++++++++++++++++ 5 files changed, 55 insertions(+), 3 deletions(-) diff --git a/docs/env.md b/docs/env.md index ca2621ee6..ff273b5b0 100644 --- a/docs/env.md +++ b/docs/env.md @@ -1390,6 +1390,9 @@ deferred Compose repair. That later phase keeps existing layers and legacy files, rechecks delivery and snapshot coverage, and requires consent for tracked edits. It invalidates an earlier project run, so start the complete project again before cleanup. +If retained legacy inputs have drifted, Doctor reports the deferred repair as +unavailable and skips it, keeping current layers and legacy files. Explicit +`env migrate --apply` still refuses that unverified repair. A repo that has no `.hack/hack.env.json` is migrated as well. Its values live only in `.hack/.env` and `.hack/.env.`, which are gitignored and may hold passwords or diff --git a/docs/reference/env-migration.md b/docs/reference/env-migration.md index b731d1d01..2455e2872 100644 --- a/docs/reference/env-migration.md +++ b/docs/reference/env-migration.md @@ -116,8 +116,11 @@ complete, another explicit apply is a no-op. If retained legacy inputs were edited or cannot be checked, default preview and Doctor's plan report the deferred repair as unavailable instead of failing the -read-only report. They do not infer new write or cleanup authority. Keep the -legacy copies and review them privately against current layers and Compose +read-only report. They do not infer new write or cleanup authority. +Doctor prints the same guidance and skips the deferred apply; explicit +`hack env migrate --apply` keeps its refusal. Doctor's other hygiene repairs +remain separate. Keep the legacy copies and review them privately against +current layers and Compose references. Editing those references yourself is an alternative to restoring pre-migration inputs; a later explicit apply records that no repair remains, without rewriting the project. Such manual edits stay outside the journal and diff --git a/src/commands/doctor.ts b/src/commands/doctor.ts index 9de978685..2a3be231c 100644 --- a/src/commands/doctor.ts +++ b/src/commands/doctor.ts @@ -4015,6 +4015,9 @@ async function confirmTrackedEnvChange(opts: { }); } +const COMPOSE_REPAIR_UNAVAILABLE_HINT = + "Compose repair unavailable: keep legacy copies and review their values privately against current layers and Compose references, or explicitly restore the snapshot with --preserve-changes."; + /** Confirms the Compose env_file repair, as a tracked change when git tracks the file. */ async function maybeMigrateProjectEnvConfig(opts: { readonly startDir: string; @@ -4034,6 +4037,10 @@ async function maybeMigrateProjectEnvConfig(opts: { }), }; const preview = await previewEnvMigration(migrationProject); + if (preview.composeRepair === "unavailable") { + note(COMPOSE_REPAIR_UNAVAILABLE_HINT, "env migration"); + return; + } const tracked = preview.files.filter( (file) => file.tracked && file.action !== "removable" ); @@ -4290,7 +4297,7 @@ async function buildEnvMigrationPlanStep(opts: { }), }); if (preview.composeRepair === "unavailable") { - return "Compose repair unavailable: keep legacy copies and review their values privately against current layers and Compose references, or explicitly restore the snapshot with --preserve-changes."; + return COMPOSE_REPAIR_UNAVAILABLE_HINT; } if ( preview.files.some( diff --git a/tests/env-migration-provider.test.ts b/tests/env-migration-provider.test.ts index 4fb153718..64a0af5c7 100644 --- a/tests/env-migration-provider.test.ts +++ b/tests/env-migration-provider.test.ts @@ -308,6 +308,17 @@ test("production previews report deferred repair unavailable without failing, pr files: [], }); } + const doctor = await runCliCommand(project.projectRoot, [ + "doctor", + "--migrate-env-config", + "--path", + project.projectRoot, + ]); + expect(doctor.stdout + doctor.stderr).toContain("Compose repair unavailable"); + expect(doctor.stdout + doctor.stderr).not.toContain( + "Env migration input changed" + ); + expect(doctor.stdout + doctor.stderr).not.toContain(drift); const refused = await cli(project.projectRoot, ["--apply"]); expect(refused.exitCode).not.toBe(0); expect(refused.stdout + refused.stderr).not.toContain(drift); diff --git a/tests/env-migration-tracked-consent.test.ts b/tests/env-migration-tracked-consent.test.ts index 798557c81..464177f93 100644 --- a/tests/env-migration-tracked-consent.test.ts +++ b/tests/env-migration-tracked-consent.test.ts @@ -170,6 +170,34 @@ function runDoctor(opts: { } for (const backend of ["compose", "native"] as const) { + test( + `${backend} doctor skips unavailable deferred repair and retains both legacy and current values`, + async () => { + const project = await createProject({ compose: COMPOSE_WITH_ENV_FILE }); + expect((await runImplicit(project)).exitCode).toBe(0); + const drift = "synthetic-doctor-retained-drift"; + const envPath = join(project.projectDir, ".env"); + await writeFile(envPath, `${KEY}=${drift}\n`); + const before = await status(project); + const run = await runDoctor({ project, backend, accept: false }); + expect(run.exitCode, run.output).toBe(backend === "compose" ? 1 : 0); + expect(run.output).toContain("Compose repair unavailable"); + expect(run.output).not.toContain("Env migration input changed"); + expect(run.output).not.toContain("Env migration completed."); + expect(run.output).not.toContain(drift); + expect(run.output).not.toContain(VALUE); + expect(await status(project)).toEqual(before); + expect( + await readFile(join(project.projectDir, "docker-compose.yml"), "utf8") + ).toBe(COMPOSE_WITH_ENV_FILE); + expect(await readFile(envPath, "utf8")).toBe(`${KEY}=${drift}\n`); + expect( + await readFile(join(project.projectDir, "hack.env.local.yaml"), "utf8") + ).toContain(VALUE); + }, + CLI_TEST_TIMEOUT_MS + ); + test( `${backend} doctor repairs tracked Compose after implicit migration only with fresh consent`, async () => { From 4a162b3e99fdde59937e6a1b0c985adb803df488 Mon Sep 17 00:00:00 2001 From: Kit <344842+roodboi@users.noreply.github.com> Date: Sat, 14 Nov 2026 07:48:24 +0000 Subject: [PATCH 21/32] fix(env): keep deferred repair delivery checks read-only --- docs/reference/env-migration.md | 2 + src/lib/project-env-config.ts | 8 +- tests/env-migration-journal.test.ts | 66 +++++++++++ .../fixtures/env-migration-keychain-worker.ts | 106 ++++++++++++++++++ 4 files changed, 180 insertions(+), 2 deletions(-) create mode 100644 tests/fixtures/env-migration-keychain-worker.ts diff --git a/docs/reference/env-migration.md b/docs/reference/env-migration.md index 2455e2872..37bdcacc4 100644 --- a/docs/reference/env-migration.md +++ b/docs/reference/env-migration.md @@ -71,6 +71,8 @@ prepares and checks its own immutable plan against the inputs before publication The exact plan requires reading legacy values and checking delivery in memory. A configured native credential backend can request its ordinary authorization; Hack does not bypass that prompt or include the values in the preview. +Delivery checks, including deferred Compose repair, read existing backend keys +without generating them or copying a keychain key to an absent key file. ## Apply and recovery diff --git a/src/lib/project-env-config.ts b/src/lib/project-env-config.ts index 86cffbd45..c4e3c1fa7 100644 --- a/src/lib/project-env-config.ts +++ b/src/lib/project-env-config.ts @@ -10439,7 +10439,7 @@ async function assertLegacyCleanupReceipts(opts: { } } -/** Re-evaluate delivery and Compose references after prompts, while writer locks are held. */ +/** Re-evaluate delivery without provisioning keys, including during read-only previews. */ async function assertLegacyCleanupDelivery(opts: { readonly projectRoot: string; readonly paths: readonly string[]; @@ -10510,7 +10510,11 @@ async function assertLegacyCleanupDelivery(opts: { } } if (contractSelected || (envName === null && opts.paths.includes(path))) { - const legacy = await resolveHackEnv({ ...context, envName }); + const legacy = await resolveHackEnv({ + ...context, + envName, + allowKeyProvisioning: false, + }); if (contractSelected) { for (const entry of legacy.values) { if (entry.value !== null) { diff --git a/tests/env-migration-journal.test.ts b/tests/env-migration-journal.test.ts index 8711e17d8..ed8b3fee2 100644 --- a/tests/env-migration-journal.test.ts +++ b/tests/env-migration-journal.test.ts @@ -377,6 +377,72 @@ test("deferred repair preview and Doctor plan report unavailable legacy drift wi expect(await readFile(path, "utf8")).toBe(journal); }); +test("deferred repair preview reads a keychain-backed store without copying its absent backend key or invalidating apply", async () => { + await writeFile( + join(project.projectDir, "docker-compose.yml"), + "services:\n api:\n image: alpine:3.22\n env_file: .env\n" + ); + await writeFile( + join(project.projectDir, "hack.env.json"), + JSON.stringify({ + version: 1, + vars: [ + { key: "TOKEN", source: "plain_env" }, + { key: "SEC", source: "keychain", services: ["api"] }, + ], + }) + ); + await writeFile( + join(project.projectDir, "hack.config.json"), + JSON.stringify({ + name: "journal", + controlPlane: { + secrets: { + backend: "encrypted_file", + encryptedFile: { + path: join(project.projectRoot, "legacy-store.json"), + keyPath: join(project.projectRoot, "legacy-backend.key"), + }, + }, + }, + }) + ); + const result = await commands.run({ + cmd: [ + process.execPath, + "--no-env-file", + join(import.meta.dir, "fixtures", "env-migration-keychain-worker.ts"), + ], + cwd: project.projectRoot, + env: privateCliEnv({ + root, + env: { + HACK_HOME: join(root, "state"), + HACK_NO_INTERACTIVE: "1", + HACK_TEST_PROJECT_ROOT: project.projectRoot, + HACK_SECRETS_DISABLE_KEYCHAIN_FALLBACK: undefined, + PATH: "/nonexistent", + }, + }), + }); + expect(result.stdout + result.stderr).not.toContain(SENTINEL); + expect(result.stdout + result.stderr).not.toContain( + "synthetic-deferred-repair-keychain" + ); + expect(result.stderr).toBe(""); + expect(result.exitCode).toBe(0); + expect(JSON.parse(result.stdout)).toEqual({ + previewStatus: "ready", + unchangedAfterPreview: true, + absentAfterPreview: true, + absentAfterApply: true, + applied: true, + preserved: true, + calls: { get: expect.any(Number), set: 0, delete: 0 }, + }); + expect(JSON.parse(result.stdout).calls.get).toBeGreaterThan(0); +}); + test("explicit no-op clears deferred Compose repair after manual removal of references", async () => { const composePath = join(project.projectDir, "docker-compose.yml"); await writeFile( diff --git a/tests/fixtures/env-migration-keychain-worker.ts b/tests/fixtures/env-migration-keychain-worker.ts new file mode 100644 index 000000000..0f71de20e --- /dev/null +++ b/tests/fixtures/env-migration-keychain-worker.ts @@ -0,0 +1,106 @@ +import { spyOn } from "bun:test"; +import { lstat, readdir, readFile } from "node:fs/promises"; +import { join, relative } from "node:path"; +import { secrets } from "bun"; + +// One process owns the stubs and secret-store cache; no real keychain is reached. +const root = process.env.HACK_TEST_PROJECT_ROOT; +if (!root) { + throw new Error("Missing private fixture root"); +} +const key = "synthetic-deferred-repair-keychain-key"; +const value = "synthetic-deferred-repair-keychain-value"; +const calls = { get: 0, set: 0, delete: 0 }; +spyOn(secrets, "get").mockImplementation(async () => { + calls.get += 1; + return key; +}); +spyOn(secrets, "set").mockImplementation(async () => { + calls.set += 1; +}); +spyOn(secrets, "delete").mockImplementation(async () => { + calls.delete += 1; + return false; +}); +if ((await secrets.get({ service: "fixture", name: "preflight" })) !== key) { + throw new Error("Keychain stub is not active"); +} +calls.get = 0; + +const { resolveSecretStore } = await import("../../src/lib/secret-store.ts"); +const { resolveProjectEnvConfig } = await import( + "../../src/lib/project-env-config.ts" +); +const { envMigrationSnapshots } = await import( + "../../src/lib/env-migration-snapshot.ts" +); +const { applyEnvMigration, migrateLegacyEnvForCommand, previewEnvMigration } = + await import("../../src/lib/env-migration.ts"); +const project = { + projectRoot: root, + projectDir: join(root, ".hack"), + projectName: "journal", + serviceNames: ["api"], +}; +// Seed the synthetic store without provisioning its configured key file. Later +// reads can obtain the same key only from the stubbed native-keychain fallback. +process.env.HACK_SECRETS_FILE_KEY = key; +await (await resolveSecretStore(project)).set({ key: "SEC", value }); +Reflect.deleteProperty(process.env, "HACK_SECRETS_FILE_KEY"); +await migrateLegacyEnvForCommand({ ...project, materialize: false }); + +async function snapshot(fixtureRoot: string): Promise { + const paths = await readdir(fixtureRoot, { + recursive: true, + withFileTypes: true, + }); + const entries = await Promise.all( + paths.map(async (entry) => { + const path = join(entry.parentPath, entry.name); + const state = await lstat(path); + return { + path: relative(fixtureRoot, path), + mode: state.mode, + digest: entry.isFile() + ? new Bun.CryptoHasher("sha256") + .update(await readFile(path)) + .digest("hex") + : null, + }; + }) + ); + return JSON.stringify(entries.sort((a, b) => a.path.localeCompare(b.path))); +} +const before = await snapshot(root); +const preview = await previewEnvMigration(project); +const unchangedAfterPreview = before === (await snapshot(root)); +const absentAfterPreview = !(await Bun.file( + join(root, "legacy-backend.key") +).exists()); +let applied = false; +try { + applied = + ( + await applyEnvMigration({ + project, + snapshots: envMigrationSnapshots(), + acceptTrackedChanges: true, + }) + ).status === "applied"; +} catch { + // Report only the outcome, never backend diagnostics or key material. +} +const resolved = await resolveProjectEnvConfig({ ...project, envName: null }); +console.log( + JSON.stringify({ + previewStatus: preview.status, + unchangedAfterPreview, + absentAfterPreview, + absentAfterApply: !(await Bun.file( + join(root, "legacy-backend.key") + ).exists()), + applied, + preserved: resolved?.serviceEnv.api?.SEC === value, + calls, + }) +); From af78804f519401147a85ba6ed848ab0e027f81aa Mon Sep 17 00:00:00 2001 From: Kit <344842+roodboi@users.noreply.github.com> Date: Sat, 14 Nov 2026 08:09:45 +0000 Subject: [PATCH 22/32] test(native): account for lifecycle activity after startup refusals --- scripts/check-native-acquisition-plan-cli.ts | 20 ++- scripts/check-native-endpoint-plan-cli.ts | 18 ++- scripts/check-native-env-plan-cli.ts | 25 +++- scripts/check-native-host-plan-cli.ts | 25 +++- scripts/check-native-process-plan-cli.ts | 20 ++- scripts/check-native-routing-plan-cli.ts | 19 ++- scripts/lib/native-plan-home-state.ts | 88 +++++++++++++ tests/native-plan-home-state.test.ts | 127 +++++++++++++++++++ 8 files changed, 323 insertions(+), 19 deletions(-) create mode 100644 scripts/lib/native-plan-home-state.ts create mode 100644 tests/native-plan-home-state.test.ts diff --git a/scripts/check-native-acquisition-plan-cli.ts b/scripts/check-native-acquisition-plan-cli.ts index d38b95270..364147979 100644 --- a/scripts/check-native-acquisition-plan-cli.ts +++ b/scripts/check-native-acquisition-plan-cli.ts @@ -10,6 +10,10 @@ import { import { tmpdir } from "node:os"; import { join, resolve } from "node:path"; import { isRecord } from "../src/lib/guards.ts"; +import { + assertNativePlanRefusalActivity, + snapshotNativePlanHome, +} from "./lib/native-plan-home-state.ts"; /** Qualify pure acquisition intent in a relocated bundle and a real linked worktree. */ const root = resolve(import.meta.dir, ".."); @@ -264,6 +268,11 @@ try { ], checkout ); + assert( + (await readdir(home)).length === 0, + "metadata planning creates no HOME state" + ); + const beforeStartup = await snapshotNativePlanHome(home); const runtime = await invoke(["up", "web", "--detach"], checkout, false, { HACK_RUNTIME_BACKEND: "native", HACK_NATIVE_BINARY: join(bundle, "absent-runtime"), @@ -282,10 +291,13 @@ try { "planning creates no runtime state" ); } - assert( - (await readdir(join(home, ".hack")).catch(() => [])).length === 0, - "planning creates no registry or runtime home state" - ); + await assertNativePlanRefusalActivity({ + home, + before: beforeStartup, + attempts: 1, + sentinel: canary, + fixtureRoot: directory, + }); process.stdout.write( `Relocated native acquisition planning: ${checks} checks passed; no hooks/pulls/builds/decryption/runtime effects\n` ); diff --git a/scripts/check-native-endpoint-plan-cli.ts b/scripts/check-native-endpoint-plan-cli.ts index 8b98031cd..04ac9e1a8 100644 --- a/scripts/check-native-endpoint-plan-cli.ts +++ b/scripts/check-native-endpoint-plan-cli.ts @@ -10,6 +10,10 @@ import { import { tmpdir } from "node:os"; import { join, resolve } from "node:path"; import { isRecord } from "../src/lib/guards.ts"; +import { + assertNativePlanRefusalActivity, + snapshotNativePlanHome, +} from "./lib/native-plan-home-state.ts"; /** Exercise pure endpoint planning through the relocated CLI and real Git linkage. */ const root = resolve(import.meta.dir, ".."); @@ -367,6 +371,11 @@ try { "actual resolution rejects unbound logical reference" ); await authored(checkout, project); + assert( + (await readdir(home)).length === 0, + "metadata planning creates no HOME state" + ); + const beforeStartup = await snapshotNativePlanHome(home); const runtime = await invoke( ["up", "web", "--detach"], checkout, @@ -393,8 +402,13 @@ try { "no generated runtime state" ); } - const homeEntries = await readdir(join(home, ".hack")).catch(() => []); - assert(homeEntries.length === 0, "no registry or runtime home writes"); + await assertNativePlanRefusalActivity({ + home, + before: beforeStartup, + attempts: 1, + sentinel: canary, + fixtureRoot: directory, + }); process.stdout.write( `Relocated native endpoint planning: ${checks} checks passed; no hooks/DNS/trust/runtime effects\n` ); diff --git a/scripts/check-native-env-plan-cli.ts b/scripts/check-native-env-plan-cli.ts index 04e189bc0..f82581703 100644 --- a/scripts/check-native-env-plan-cli.ts +++ b/scripts/check-native-env-plan-cli.ts @@ -10,6 +10,10 @@ import { import { tmpdir } from "node:os"; import { join, resolve } from "node:path"; import { isRecord } from "../src/lib/guards.ts"; +import { + assertNativePlanRefusalActivity, + snapshotNativePlanHome, +} from "./lib/native-plan-home-state.ts"; /** Real compiled CLI/sidecar, isolated HOME and Git worktrees, no source or toolchain on PATH. */ const root = resolve(import.meta.dir, ".."); @@ -249,6 +253,11 @@ try { await Bun.write(join(checkout, ".hack/hack.config.json"), "{}"); await refused([], "E_NATIVE_PROJECT_CONFLICT", checkout); await rm(join(checkout, ".hack/hack.config.json")); + require((await readdir(home)).length === 0 && + !(await readdir(join(checkout, ".hack"))).some( + (name) => name === ".internal" || name === ".branch" + ), "metadata planning creates no HOME, registry or runtime state"); + const beforeStartup = await snapshotNativePlanHome(home); const unsupported = await invoke( ["up", "web", "--detach", "--json"], checkout, @@ -276,10 +285,18 @@ try { detachedResult.ok === false && record(detachedResult.error).code === "E_STARTUP_INCOMPLETE", "admitted detached startup returns one JSON failure envelope without readiness"); - require((await readdir(home)).length === 0 && - !(await readdir(join(checkout, ".hack"))).some( - (name) => name === ".internal" || name === ".branch" - ), "no registry or generated runtime state"); + // Lifecycle attempts record names-only activity even when input admission + // refuses. Planning above remains read-only; neither refusal starts runtime. + await assertNativePlanRefusalActivity({ + home, + before: beforeStartup, + attempts: 2, + sentinel, + fixtureRoot: directory, + }); + require(!(await readdir(join(checkout, ".hack"))).some( + (name) => name === ".internal" || name === ".branch" + ), "refused startup creates no generated runtime state"); process.stdout.write( "Relocated native env plan acceptance: offline validation, metadata-only output, missing overlays/refs, immutable directives, separate hashes, real linked-worktree layers, CI/slim/opt-out, unsafe-input and runtime refusal passed\n" ); diff --git a/scripts/check-native-host-plan-cli.ts b/scripts/check-native-host-plan-cli.ts index de6dd3033..ebe67fe58 100644 --- a/scripts/check-native-host-plan-cli.ts +++ b/scripts/check-native-host-plan-cli.ts @@ -3,6 +3,10 @@ import { copyFile, mkdir, mkdtemp, readdir, rm } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join, resolve } from "node:path"; import { isRecord } from "../src/lib/guards.ts"; +import { + assertNativePlanRefusalActivity, + snapshotNativePlanHome, +} from "./lib/native-plan-home-state.ts"; /** Actual relocated CLI and sidecar; no Bun/Rust on PATH and no host effects. */ const root = resolve(import.meta.dir, ".."); @@ -277,6 +281,11 @@ try { "local settings cannot inject host commands" ); await rm(join(checkout, ".hack/hack.local.json")); + assert( + (await readdir(home)).length === 0, + "metadata planning creates no HOME state" + ); + const beforeStartup = await snapshotNativePlanHome(home); const runtime = await invoke(["up", "web", "--detach", "--json"], checkout, { HACK_RUNTIME_BACKEND: "native", HACK_NATIVE_BINARY: join(bundle, "absent-runtime"), @@ -287,13 +296,21 @@ try { "native detached service selection refuses before input or runtime work" ); assert( - !(await Bun.file(marker).exists()) && - (await readdir(home)).length === 0 && - !(await readdir(join(checkout, ".hack"))).some( + !( + (await Bun.file(marker).exists()) || + (await readdir(join(checkout, ".hack"))).some( (name) => name === ".internal" || name === ".branch" - ), + ) + ), "no hooks, process, registry or runtime effects" ); + await assertNativePlanRefusalActivity({ + home, + before: beforeStartup, + attempts: 1, + sentinel: canary, + fixtureRoot: directory, + }); process.stdout.write( `Relocated native host planning: ${checks} checks passed; no host execution or secret delivery\n` ); diff --git a/scripts/check-native-process-plan-cli.ts b/scripts/check-native-process-plan-cli.ts index a9e2d0c8b..580ab0dc4 100644 --- a/scripts/check-native-process-plan-cli.ts +++ b/scripts/check-native-process-plan-cli.ts @@ -10,6 +10,10 @@ import { import { tmpdir } from "node:os"; import { join, resolve } from "node:path"; import { isRecord } from "../src/lib/guards.ts"; +import { + assertNativePlanRefusalActivity, + snapshotNativePlanHome, +} from "./lib/native-plan-home-state.ts"; /** Qualify pure process planning through a relocated bundle and real Git worktree. */ const root = resolve(import.meta.dir, ".."); @@ -262,6 +266,11 @@ try { ], checkout ); + assert( + (await readdir(home)).length === 0, + "metadata planning creates no HOME state" + ); + const beforeStartup = await snapshotNativePlanHome(home); const runtime = await invoke( ["up", "web", "--detach"], checkout, @@ -288,10 +297,13 @@ try { "planning creates no runtime state" ); } - assert( - (await readdir(join(home, ".hack")).catch(() => [])).length === 0, - "planning creates no registry or runtime home state" - ); + await assertNativePlanRefusalActivity({ + home, + before: beforeStartup, + attempts: 1, + sentinel: canary, + fixtureRoot: directory, + }); process.stdout.write( `Relocated native process planning: ${checks} checks passed; no hooks/DNS/trust/runtime effects\n` ); diff --git a/scripts/check-native-routing-plan-cli.ts b/scripts/check-native-routing-plan-cli.ts index f85076287..f2d2fed9a 100644 --- a/scripts/check-native-routing-plan-cli.ts +++ b/scripts/check-native-routing-plan-cli.ts @@ -10,6 +10,10 @@ import { import { tmpdir } from "node:os"; import { join, resolve } from "node:path"; import { isRecord } from "../src/lib/guards.ts"; +import { + assertNativePlanRefusalActivity, + snapshotNativePlanHome, +} from "./lib/native-plan-home-state.ts"; /** Exercise the relocated executable/sidecar with isolated policy and real linked Git worktrees. */ const root = resolve(import.meta.dir, ".."); @@ -378,6 +382,12 @@ try { "local route injection refused" ); await rm(join(checkout, ".hack/hack.local.json")); + assert( + (await readdir(home)).join() === ".hack" && + (await readdir(join(home, ".hack"))).join() === "hack.config.json", + "metadata planning preserves only the authored global config" + ); + const beforeStartup = await snapshotNativePlanHome(home); const runtime = await invoke(["up", "web", "--detach", "--json"], checkout, { HACK_RUNTIME_BACKEND: "native", HACK_NATIVE_BINARY: join(bundle, "absent-runtime"), @@ -393,9 +403,16 @@ try { (await readdir(join(checkout, ".hack"))).some( (name) => name === ".internal" || name === ".branch" ) - ) && (await readdir(join(home, ".hack"))).join() === "hack.config.json", + ), "no hook, registry, route publication or generated-state effects" ); + await assertNativePlanRefusalActivity({ + home, + before: beforeStartup, + attempts: 1, + sentinel: canary, + fixtureRoot: directory, + }); process.stdout.write( `Relocated native routing planning: ${checks} checks passed; no DNS/trust/runtime effects\n` ); diff --git a/scripts/lib/native-plan-home-state.ts b/scripts/lib/native-plan-home-state.ts new file mode 100644 index 000000000..11cad9394 --- /dev/null +++ b/scripts/lib/native-plan-home-state.ts @@ -0,0 +1,88 @@ +import { lstat, readdir, readFile } from "node:fs/promises"; +import { join } from "node:path"; +import { isRecord } from "../../src/lib/guards.ts"; + +const ACTIVITY_FIELDS = new Set([ + "v", + "ts", + "project", + "branch", + "action", + "origin", + "outcome", + "failed_services", + "duration_ms", +]); + +/** Fingerprint a private acceptance HOME, including modes, without retaining values. */ +export async function snapshotNativePlanHome( + home: string +): Promise> { + const result = new Map(); + for (const name of (await readdir(home, { recursive: true })).sort()) { + const path = join(home, name); + const info = await lstat(path); + if (!(info.isFile() || info.isDirectory()) || info.isSymbolicLink()) { + throw new Error("Native plan fixture HOME contains an unexpected entry"); + } + const digest = info.isFile() + ? new Bun.CryptoHasher("sha256") + .update(await readFile(path)) + .digest("hex") + : "directory"; + result.set(name, `${info.mode}:${digest}`); + } + return result; +} + +/** Admit exactly the failed lifecycle records added after read-only planning. */ +export async function assertNativePlanRefusalActivity(opts: { + readonly home: string; + readonly before: ReadonlyMap; + readonly attempts: number; + readonly sentinel: string; + readonly fixtureRoot: string; +}): Promise { + const name = ".hack/activity.jsonl"; + const path = join(opts.home, name); + const after = new Map(await snapshotNativePlanHome(opts.home)); + const activity = await readFile(path, "utf8"); + let records: unknown[]; + try { + records = activity + .trim() + .split("\n") + .map((line): unknown => JSON.parse(line)); + } catch { + throw new Error( + "Native plan fixture activity is malformed; values omitted" + ); + } + after.delete(name); + if (!opts.before.has(".hack")) { + after.delete(".hack"); + } + if ( + opts.before.has(name) || + after.size !== opts.before.size || + [...opts.before].some(([entry, state]) => after.get(entry) !== state) || + ((await lstat(path)).mode & 0o077) !== 0 || + (!opts.before.has(".hack") && + ((await lstat(join(opts.home, ".hack"))).mode & 0o077) !== 0) || + records.length !== opts.attempts || + records.some( + (entry) => + !isRecord(entry) || + Object.keys(entry).some((field) => !ACTIVITY_FIELDS.has(field)) || + entry.v !== 1 || + entry.action !== "up" || + entry.outcome !== "failed" + ) || + activity.includes(opts.sentinel) || + activity.includes(opts.fixtureRoot) + ) { + throw new Error( + "Native plan startup refusal changed HOME beyond private value-free failure activity" + ); + } +} diff --git a/tests/native-plan-home-state.test.ts b/tests/native-plan-home-state.test.ts new file mode 100644 index 000000000..71d76ede1 --- /dev/null +++ b/tests/native-plan-home-state.test.ts @@ -0,0 +1,127 @@ +import { afterEach, expect, test } from "bun:test"; +import { chmod, mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { + assertNativePlanRefusalActivity, + snapshotNativePlanHome, +} from "../scripts/lib/native-plan-home-state.ts"; + +const roots: string[] = []; +const sentinel = "synthetic-native-plan-private-value"; +const record = { v: 1, action: "up", outcome: "failed" }; + +afterEach(async () => { + for (const root of roots.splice(0)) { + await rm(root, { recursive: true, force: true }); + } +}); + +async function fixture(authoredConfig = true, directoryMode = 0o700) { + const root = await mkdtemp(join(tmpdir(), "plan-home-")); + roots.push(root); + const home = join(root, "home"); + await mkdir(home, { mode: 0o700 }); + const hack = join(home, ".hack"); + const config = join(hack, "hack.config.json"); + if (authoredConfig) { + await mkdir(hack, { mode: directoryMode }); + await writeFile(config, '{"fixture":true}\n', { mode: 0o600 }); + } + const before = await snapshotNativePlanHome(home); + await mkdir(hack, { mode: 0o700, recursive: true }); + const activity = join(hack, "activity.jsonl"); + await writeFile(activity, `${JSON.stringify(record)}\n`, { mode: 0o600 }); + return { + root, + home, + hack, + config, + activity, + options: { home, before, attempts: 1, sentinel, fixtureRoot: root }, + }; +} + +for (const authoredConfig of [true, false]) { + test(`native plan HOME admits exactly failure activity (authored config ${authoredConfig})`, async () => { + const { options } = await fixture(authoredConfig); + await assertNativePlanRefusalActivity(options); + expect(options.before.has(".hack/activity.jsonl")).toBe(false); + }); +} + +test("native plan HOME preserves an existing metadata directory mode inside its private HOME", async () => { + const { options } = await fixture(true, 0o755); + await assertNativePlanRefusalActivity(options); + expect(options.before.has(".hack/hack.config.json")).toBe(true); +}); + +test("native plan HOME refuses a newly created activity directory with wider permissions", async () => { + const { options, hack } = await fixture(false); + await chmod(hack, 0o755); + await expect(assertNativePlanRefusalActivity(options)).rejects.toThrow( + "Native plan" + ); +}); + +for (const change of [ + "registry", + "config", + "mode", + "extra-record", + "outcome", + "value", + "path", + "malformed", +] as const) { + test(`native plan HOME refuses ${change} instead of widening its activity exception`, async () => { + const state = await fixture(); + switch (change) { + case "registry": + await writeFile(join(state.hack, "projects.json"), "{}\n"); + break; + case "config": + await writeFile(state.config, '{"fixture":false}\n'); + break; + case "mode": + await chmod(state.activity, 0o644); + break; + case "extra-record": + await writeFile( + state.activity, + `${JSON.stringify(record)}\n`.repeat(2) + ); + break; + case "outcome": + await writeFile( + state.activity, + JSON.stringify({ ...record, outcome: "ok" }) + ); + break; + case "value": + case "path": + await writeFile( + state.activity, + JSON.stringify({ + ...record, + project: change === "value" ? sentinel : state.root, + }) + ); + break; + case "malformed": + await writeFile(state.activity, `{${sentinel}`); + break; + default: + throw new Error("Unknown native plan HOME negative control"); + } + const error = await assertNativePlanRefusalActivity(state.options).then( + () => null, + (caught: unknown) => caught + ); + expect(error).toBeInstanceOf(Error); + if (error instanceof Error) { + expect(error.message).not.toContain(sentinel); + expect(error.message).not.toContain(state.root); + } + }); +} From f192008728ec3f524145e9f61986033e36cc1137 Mon Sep 17 00:00:00 2001 From: Kit <344842+roodboi@users.noreply.github.com> Date: Sat, 14 Nov 2026 09:58:15 +0000 Subject: [PATCH 23/32] test(env): use canonical migration fixtures and bounded workflow budgets --- tests/doctor-fix-noninteractive.test.ts | 3 +- tests/env-contract-shell-values.test.ts | 5 +- tests/env-legacy-service-scope.test.ts | 9 +- tests/env-migration-provider.test.ts | 223 ++++++++++-------- tests/env-migration-safety.test.ts | 6 +- .../fixtures/upgrade-check-failure-worker.ts | 22 ++ ...e-compose-retained-storage-command.test.ts | 194 ++++++++------- tests/upgrade-check-failure.test.ts | 72 +++--- 8 files changed, 309 insertions(+), 225 deletions(-) create mode 100644 tests/fixtures/upgrade-check-failure-worker.ts diff --git a/tests/doctor-fix-noninteractive.test.ts b/tests/doctor-fix-noninteractive.test.ts index b5541cc03..3f8e6a8cd 100644 --- a/tests/doctor-fix-noninteractive.test.ts +++ b/tests/doctor-fix-noninteractive.test.ts @@ -15,6 +15,7 @@ import { readdir, readFile, readlink, + realpath, rm, symlink, writeFile, @@ -177,7 +178,7 @@ beforeEach(async () => { // This fixture exercises daemon repair through a mocked shell. Its own // policy must be independent of the outer suite's daemon safety setting. Reflect.deleteProperty(process.env, "HACK_DAEMON_AUTOSTART_DISABLE"); - tempDir = await mkdtemp(join(tmpdir(), "hack-doctor-fix-")); + tempDir = await realpath(await mkdtemp(join(tmpdir(), "hack-doctor-fix-"))); process.env.HOME = tempDir; process.env.HACK_NO_INTERACTIVE = "1"; // Short-circuit mutagen path resolution so the fix flow never attempts a diff --git a/tests/env-contract-shell-values.test.ts b/tests/env-contract-shell-values.test.ts index 980fdd180..97abb50f9 100644 --- a/tests/env-contract-shell-values.test.ts +++ b/tests/env-contract-shell-values.test.ts @@ -6,6 +6,7 @@ import { mkdtemp, readdir, readFile, + realpath, rename, rm, stat, @@ -66,7 +67,9 @@ async function createContractRepo(opts: { readonly otherVars?: readonly Record[]; readonly secrets?: (projectRoot: string) => Record; }): Promise { - const root = await mkdtemp(join(tmpdir(), "hack-env-contract-shell-")); + const root = await realpath( + await mkdtemp(join(tmpdir(), "hack-env-contract-shell-")) + ); tempDirs.add(root); const projectRoot = resolve(root, "repo"); const projectDir = resolve(projectRoot, ".hack"); diff --git a/tests/env-legacy-service-scope.test.ts b/tests/env-legacy-service-scope.test.ts index a0f3f8351..9529c64c6 100644 --- a/tests/env-legacy-service-scope.test.ts +++ b/tests/env-legacy-service-scope.test.ts @@ -9,6 +9,7 @@ import { readdir, readFile, readlink, + realpath, rm, symlink, writeFile, @@ -280,7 +281,9 @@ async function useStagingAsDefaultOverlay(repo: Repo): Promise { } async function createLegacyRepo(): Promise { - const root = await mkdtemp(join(tmpdir(), "hack-env-legacy-scope-")); + const root = await realpath( + await mkdtemp(join(tmpdir(), "hack-env-legacy-scope-")) + ); tempDirs.add(root); const projectRoot = resolve(root, "repo"); const projectDir = resolve(projectRoot, ".hack"); @@ -3936,7 +3939,9 @@ async function runStubbedDoctor(opts: { } test("doctor doesn't rewrite a Compose file outside the project through a linked .hack", async () => { - const root = await mkdtemp(join(tmpdir(), "hack-env-legacy-scope-")); + const root = await realpath( + await mkdtemp(join(tmpdir(), "hack-env-legacy-scope-")) + ); tempDirs.add(root); const projectRoot = resolve(root, "repo"); const outside = resolve(root, "outside"); diff --git a/tests/env-migration-provider.test.ts b/tests/env-migration-provider.test.ts index 64a0af5c7..1a28e2f0d 100644 --- a/tests/env-migration-provider.test.ts +++ b/tests/env-migration-provider.test.ts @@ -37,6 +37,9 @@ const keys = [ const sentinel = "synthetic-provider-cli-sentinel"; const children = fixtureChildren("env-migration-provider"); const compiled = process.env.HACK_TEST_ENV_MIGRATION_BINARY; +// These cases run several real CLIs plus Doctor; hosted macOS exceeds Bun's +// five-second default. The test envelope also bounds child cleanup on refusal. +const DOCTOR_WORKFLOW_TIMEOUT_MS = 45_000; let root = ""; let saved: (string | undefined)[] = []; @@ -221,111 +224,125 @@ test("production CLI explicitly repairs Compose after implicit env exec, then ro ); }); -test("production cleanup accepts Doctor permission tightening while preserving original rollback bytes", async () => { - const project = await fixture(); - const envPath = join(project.projectDir, ".env"); - await chmod(envPath, 0o644); - expect((await lstat(envPath)).mode & 0o777).toBe(0o644); - const migrated = await cli(project.projectRoot, ["--apply"]); - expect(migrated.exitCode, migrated.stderr).toBe(0); - const doctor = await runCliCommand(project.projectRoot, [ - "doctor", - "--fix", - "--path", - project.projectRoot, - ]); - // The fixture intentionally has no Docker or other host tooling; hygiene still runs. - expect(doctor.exitCode).toBe(1); - expect((await lstat(envPath)).mode & 0o777).toBe(0o600); - expect(await readFile(envPath, "utf8")).toBe(`TOKEN=${sentinel}\n`); - await recordEnvMigrationProjectRun(project); - const cleaned = await cli(project.projectRoot, ["--cleanup", "--apply"]); - expect(cleaned.exitCode, cleaned.stderr).toBe(0); - expect(await Bun.file(envPath).exists()).toBe(false); - const rollback = await cli(project.projectRoot, [ - "--rollback-manifest", - pathField(migrated.envelope.data, "manifestPath"), - ]); - expect(rollback.exitCode, rollback.stderr).toBe(0); - expect(rollback.envelope.data.warnings).toContainEqual( - expect.stringContaining("original file permissions") - ); - expect(await readFile(envPath, "utf8")).toBe(`TOKEN=${sentinel}\n`); -}); +test( + "production cleanup accepts Doctor permission tightening while preserving original rollback bytes", + async () => { + const project = await fixture(); + const envPath = join(project.projectDir, ".env"); + await chmod(envPath, 0o644); + expect((await lstat(envPath)).mode & 0o777).toBe(0o644); + const migrated = await cli(project.projectRoot, ["--apply"]); + expect(migrated.exitCode, migrated.stderr).toBe(0); + const doctor = await runCliCommand(project.projectRoot, [ + "doctor", + "--fix", + "--path", + project.projectRoot, + ]); + // The fixture intentionally has no Docker or other host tooling; hygiene still runs. + expect(doctor.exitCode).toBe(1); + expect((await lstat(envPath)).mode & 0o777).toBe(0o600); + expect(await readFile(envPath, "utf8")).toBe(`TOKEN=${sentinel}\n`); + await recordEnvMigrationProjectRun(project); + const cleaned = await cli(project.projectRoot, ["--cleanup", "--apply"]); + expect(cleaned.exitCode, cleaned.stderr).toBe(0); + expect(await Bun.file(envPath).exists()).toBe(false); + const rollback = await cli(project.projectRoot, [ + "--rollback-manifest", + pathField(migrated.envelope.data, "manifestPath"), + ]); + expect(rollback.exitCode, rollback.stderr).toBe(0); + expect(rollback.envelope.data.warnings).toContainEqual( + expect.stringContaining("original file permissions") + ); + expect(await readFile(envPath, "utf8")).toBe(`TOKEN=${sentinel}\n`); + }, + DOCTOR_WORKFLOW_TIMEOUT_MS +); -test("production rollback accepts a byte-identical tighter mode before cleanup and reports original-mode review", async () => { - const project = await fixture(); - const envPath = join(project.projectDir, ".env"); - await chmod(envPath, 0o644); - const migrated = await cli(project.projectRoot, ["--apply"]); - expect(migrated.exitCode, migrated.stderr).toBe(0); - await runCliCommand(project.projectRoot, [ - "doctor", - "--fix", - "--path", - project.projectRoot, - ]); - expect((await lstat(envPath)).mode & 0o777).toBe(0o600); - const rollback = await cli(project.projectRoot, [ - "--rollback-manifest", - pathField(migrated.envelope.data, "manifestPath"), - ]); - expect(rollback.exitCode, rollback.stderr).toBe(0); - expect(rollback.envelope.data.warnings).toContainEqual( - expect.stringContaining("original file permissions") - ); - expect(await readFile(envPath, "utf8")).toBe(`TOKEN=${sentinel}\n`); -}); +test( + "production rollback accepts a byte-identical tighter mode before cleanup and reports original-mode review", + async () => { + const project = await fixture(); + const envPath = join(project.projectDir, ".env"); + await chmod(envPath, 0o644); + const migrated = await cli(project.projectRoot, ["--apply"]); + expect(migrated.exitCode, migrated.stderr).toBe(0); + await runCliCommand(project.projectRoot, [ + "doctor", + "--fix", + "--path", + project.projectRoot, + ]); + expect((await lstat(envPath)).mode & 0o777).toBe(0o600); + const rollback = await cli(project.projectRoot, [ + "--rollback-manifest", + pathField(migrated.envelope.data, "manifestPath"), + ]); + expect(rollback.exitCode, rollback.stderr).toBe(0); + expect(rollback.envelope.data.warnings).toContainEqual( + expect.stringContaining("original file permissions") + ); + expect(await readFile(envPath, "utf8")).toBe(`TOKEN=${sentinel}\n`); + }, + DOCTOR_WORKFLOW_TIMEOUT_MS +); -test("production previews report deferred repair unavailable without failing, printing drift values or changing files", async () => { - const project = await fixture(); - const composePath = join(project.projectDir, "docker-compose.yml"); - const compose = - "services:\n api:\n image: alpine:3.22\n env_file: .env\n"; - await writeFile(composePath, compose); - const implicit = await runCliCommand(project.projectRoot, [ - "env", - "exec", - "--path", - project.projectRoot, - "--", - "/usr/bin/true", - ]); - expect(implicit.exitCode, implicit.stderr).toBe(0); - const drift = "synthetic-drift-preview-value"; - const envPath = join(project.projectDir, ".env"); - await writeFile(envPath, `TOKEN=${drift}\n`); - const journalPath = envMigrationJournalPath(project.projectRoot); - const journal = await readFile(journalPath, "utf8"); - for (const json of [true, false]) { - const result = await runMigrationCli(project.projectRoot, [], { json }); - expect(result.exitCode, result.stderr).toBe(0); - expect(result.stdout + result.stderr).not.toContain(drift); - const parsed = JSON.parse(result.stdout); - expect(json ? parsed.data : parsed).toMatchObject({ - status: "recovery_required", - composeRepair: "unavailable", - files: [], - }); - } - const doctor = await runCliCommand(project.projectRoot, [ - "doctor", - "--migrate-env-config", - "--path", - project.projectRoot, - ]); - expect(doctor.stdout + doctor.stderr).toContain("Compose repair unavailable"); - expect(doctor.stdout + doctor.stderr).not.toContain( - "Env migration input changed" - ); - expect(doctor.stdout + doctor.stderr).not.toContain(drift); - const refused = await cli(project.projectRoot, ["--apply"]); - expect(refused.exitCode).not.toBe(0); - expect(refused.stdout + refused.stderr).not.toContain(drift); - expect(await readFile(journalPath, "utf8")).toBe(journal); - expect(await readFile(envPath, "utf8")).toBe(`TOKEN=${drift}\n`); - expect(await readFile(composePath, "utf8")).toBe(compose); -}); +test( + "production previews report deferred repair unavailable without failing, printing drift values or changing files", + async () => { + const project = await fixture(); + const composePath = join(project.projectDir, "docker-compose.yml"); + const compose = + "services:\n api:\n image: alpine:3.22\n env_file: .env\n"; + await writeFile(composePath, compose); + const implicit = await runCliCommand(project.projectRoot, [ + "env", + "exec", + "--path", + project.projectRoot, + "--", + "/usr/bin/true", + ]); + expect(implicit.exitCode, implicit.stderr).toBe(0); + const drift = "synthetic-drift-preview-value"; + const envPath = join(project.projectDir, ".env"); + await writeFile(envPath, `TOKEN=${drift}\n`); + const journalPath = envMigrationJournalPath(project.projectRoot); + const journal = await readFile(journalPath, "utf8"); + for (const json of [true, false]) { + const result = await runMigrationCli(project.projectRoot, [], { json }); + expect(result.exitCode, result.stderr).toBe(0); + expect(result.stdout + result.stderr).not.toContain(drift); + const parsed = JSON.parse(result.stdout); + expect(json ? parsed.data : parsed).toMatchObject({ + status: "recovery_required", + composeRepair: "unavailable", + files: [], + }); + } + const doctor = await runCliCommand(project.projectRoot, [ + "doctor", + "--migrate-env-config", + "--path", + project.projectRoot, + ]); + expect(doctor.stdout + doctor.stderr).toContain( + "Compose repair unavailable" + ); + expect(doctor.stdout + doctor.stderr).not.toContain( + "Env migration input changed" + ); + expect(doctor.stdout + doctor.stderr).not.toContain(drift); + const refused = await cli(project.projectRoot, ["--apply"]); + expect(refused.exitCode).not.toBe(0); + expect(refused.stdout + refused.stderr).not.toContain(drift); + expect(await readFile(journalPath, "utf8")).toBe(journal); + expect(await readFile(envPath, "utf8")).toBe(`TOKEN=${drift}\n`); + expect(await readFile(composePath, "utf8")).toBe(compose); + }, + DOCTOR_WORKFLOW_TIMEOUT_MS +); for (const damage of ["manifest", "missing-blob", "changed-blob"] as const) { test(`resumed production apply refuses ${damage} damage before another publication`, async () => { diff --git a/tests/env-migration-safety.test.ts b/tests/env-migration-safety.test.ts index ea4abdb9f..a2ad12ebc 100644 --- a/tests/env-migration-safety.test.ts +++ b/tests/env-migration-safety.test.ts @@ -5,6 +5,7 @@ import { mkdir, mkdtemp, readFile, + realpath, rm, symlink, writeFile, @@ -49,7 +50,10 @@ beforeEach(async () => { for (const key of ENV_KEYS) { savedEnv.set(key, process.env[key]); } - root = await mkdtemp(join(tmpdir(), "hack-env-migration-safety-")); + // macOS TMPDIR can use /var aliases; migration metadata requires a canonical home. + root = await realpath( + await mkdtemp(join(tmpdir(), "hack-env-migration-safety-")) + ); process.env.HOME = resolve(root, "home"); Reflect.deleteProperty(process.env, "HACK_HOME"); Reflect.deleteProperty(process.env, "HACK_ENV_SECRET_KEY"); diff --git a/tests/fixtures/upgrade-check-failure-worker.ts b/tests/fixtures/upgrade-check-failure-worker.ts new file mode 100644 index 000000000..e33f44911 --- /dev/null +++ b/tests/fixtures/upgrade-check-failure-worker.ts @@ -0,0 +1,22 @@ +import { CommandError } from "../../src/lib/shell.ts"; +import { upgradeCheckFailureClass } from "../../src/lib/upgrade-check-failure.ts"; + +// Other test files replace shell.ts's CommandError constructor. Classify real +// module instances in one fresh child, without printing their private fields. +const sentinel = "fixture-value-must-stay-private"; +const cases: unknown[] = [ + new TypeError(sentinel), + new SyntaxError(sentinel), + new RangeError(sentinel), + Object.assign(new Error(sentinel), { code: "EACCES" }), + Object.assign(new Error(sentinel), { code: "ENOENT" }), + Object.assign(new Error(sentinel), { code: sentinel, name: sentinel }), + new CommandError({ + cmd: [sentinel], + exitCode: 1, + stdout: sentinel, + stderr: sentinel, + }), + sentinel, +]; +console.log(JSON.stringify(cases.map(upgradeCheckFailureClass))); diff --git a/tests/native-compose-retained-storage-command.test.ts b/tests/native-compose-retained-storage-command.test.ts index e45a7494e..577a45b85 100644 --- a/tests/native-compose-retained-storage-command.test.ts +++ b/tests/native-compose-retained-storage-command.test.ts @@ -6,8 +6,13 @@ import { fixture, invoke, state } from "./helpers/native-compose-command.ts"; // The source CLI now performs finite witness proofs as well as the workload // transaction. Keep the same proof/wrapper budgets as its activation controls. +const COMMAND_BUDGET_MS = 120_000; +const FIXTURE_SETTLE_MS = 30_000; +// A restore case starts, stops, then starts again. Its test envelope must +// cover all three bounded commands and their fixture teardown. +const RETAINED_WORKFLOW_BUDGET_MS = 3 * COMMAND_BUDGET_MS + FIXTURE_SETTLE_MS; function command(root: string, args = ["up", "--detach", "--json"]) { - return invoke(root, args, 30_000, 120_000); + return invoke(root, args, 30_000, COMMAND_BUDGET_MS); } type Volume = { @@ -108,91 +113,108 @@ async function startAndStop() { return { root, volume }; } -test("genuinely cold source CLI startup may create its first persistent volume", async () => { - const root = await storageFixture(); - expect(await Bun.file(join(root, "volumes")).exists()).toBe(false); - const result = await command(root); - expect(result.code).toBe(0); - expect(JSON.parse(result.stdout)).toMatchObject({ - ok: true, - data: { status: "ready" }, - }); - await retainedVolume(root); - expect(await composeStarts(root)).toHaveLength(1); -}, 120_000); +test( + "genuinely cold source CLI startup may create its first persistent volume", + async () => { + const root = await storageFixture(); + expect(await Bun.file(join(root, "volumes")).exists()).toBe(false); + const result = await command(root); + expect(result.code).toBe(0); + expect(JSON.parse(result.stdout)).toMatchObject({ + ok: true, + data: { status: "ready" }, + }); + await retainedVolume(root); + expect(await composeStarts(root)).toHaveLength(1); + }, + COMMAND_BUDGET_MS + FIXTURE_SETTLE_MS +); -test("source CLI down/up restores the unchanged physical volume birth", async () => { - const { root, volume } = await startAndStop(); - const result = await command(root); - expect(result.code).toBe(0); - expect(JSON.parse(result.stdout)).toMatchObject({ - ok: true, - data: { status: "ready" }, - }); - expect(await retainedVolume(root)).toEqual(volume); - expect(await composeStarts(root)).toHaveLength(2); - expect(await state(root)).toMatchObject({ stopped: false, pending: false }); -}, 120_000); +test( + "source CLI down/up restores the unchanged physical volume birth", + async () => { + const { root, volume } = await startAndStop(); + const result = await command(root); + expect(result.code).toBe(0); + expect(JSON.parse(result.stdout)).toMatchObject({ + ok: true, + data: { status: "ready" }, + }); + expect(await retainedVolume(root)).toEqual(volume); + expect(await composeStarts(root)).toHaveLength(2); + expect(await state(root)).toMatchObject({ stopped: false, pending: false }); + }, + RETAINED_WORKFLOW_BUDGET_MS +); -test.each([ - "missing", - "replaced", -] as const)("source CLI refuses a %s retained volume before Compose can create or start", async (change) => { - const { root, volume } = await startAndStop(); - const before = await savedState(root); - const original: unknown = await Bun.file(join(root, "volumes")).json(); - if ( - !Array.isArray(original) || - original.length !== 1 || - !isRecord(original[0]) - ) { - throw new Error("Expected the retained synthetic volume row"); - } - const changed = - change === "missing" - ? [] - : [ - { - // Preserve provision/root/witness backing facts so only birth drifts. - ...original[0], - createdAt: new Date( - Date.parse(volume.createdAt) - 1000 - ).toISOString(), - }, - ]; - await Bun.write(join(root, "volumes"), JSON.stringify(changed)); - const result = await command(root); - // The effect count is the safety oracle even if a later readiness check fails. - expect(await composeStarts(root)).toHaveLength(1); - expect(result.code).toBe(1); - expect(JSON.parse(result.stdout)).toMatchObject({ ok: false }); - expect(await Bun.file(join(root, "engine")).exists()).toBe(false); - expect(await Bun.file(join(root, "volumes")).json()).toEqual(changed); - expect(await savedState(root)).toEqual(before); -}, 120_000); +test.each(["missing", "replaced"] as const)( + "source CLI refuses a %s retained volume before Compose can create or start", + async (change) => { + const { root, volume } = await startAndStop(); + const before = await savedState(root); + const original: unknown = await Bun.file(join(root, "volumes")).json(); + if ( + !Array.isArray(original) || + original.length !== 1 || + !isRecord(original[0]) + ) { + throw new Error("Expected the retained synthetic volume row"); + } + const changed = + change === "missing" + ? [] + : [ + { + // Preserve provision/root/witness backing facts so only birth drifts. + ...original[0], + createdAt: new Date( + Date.parse(volume.createdAt) - 1000 + ).toISOString(), + }, + ]; + await Bun.write(join(root, "volumes"), JSON.stringify(changed)); + const result = await command(root); + // The effect count is the safety oracle even if a later readiness check fails. + expect(await composeStarts(root)).toHaveLength(1); + expect(result.code).toBe(1); + expect(JSON.parse(result.stdout)).toMatchObject({ ok: false }); + expect(await Bun.file(join(root, "engine")).exists()).toBe(false); + expect(await Bun.file(join(root, "volumes")).json()).toEqual(changed); + expect(await savedState(root)).toEqual(before); + }, + RETAINED_WORKFLOW_BUDGET_MS +); -test("incomplete cold readiness retains observed storage and refuses data-loss recovery", async () => { - const root = await storageFixture(); - await Bun.write(join(root, "unready"), "true"); - const started = await command(root); - expect(started.code).toBe(1); - const volume = await retainedVolume(root); - const before = await savedState(root); - expect(before.generationId).toBeUndefined(); - expect(before.pending).not.toBeNull(); - expect(before.retainedStorage).toEqual([ - { name: volume.name, storage: volume.storage, createdAt: volume.createdAt }, - ]); - await Bun.write(join(root, "volumes"), "[]"); - const recovered = await command(root, ["down", "--recover", "--json"]); - expect(recovered.code).toBe(1); - const requests: string[][] = (await Bun.file(join(root, "requests")).text()) - .trim() - .split("\n") - .map((line) => JSON.parse(line)); - expect( - requests.filter((args) => args[0] === "compose" && args.includes("down")) - ).toHaveLength(0); - expect(await savedState(root)).toEqual(before); - expect(await Bun.file(join(root, "volumes")).json()).toEqual([]); -}, 120_000); +test( + "incomplete cold readiness retains observed storage and refuses data-loss recovery", + async () => { + const root = await storageFixture(); + await Bun.write(join(root, "unready"), "true"); + const started = await command(root); + expect(started.code).toBe(1); + const volume = await retainedVolume(root); + const before = await savedState(root); + expect(before.generationId).toBeUndefined(); + expect(before.pending).not.toBeNull(); + expect(before.retainedStorage).toEqual([ + { + name: volume.name, + storage: volume.storage, + createdAt: volume.createdAt, + }, + ]); + await Bun.write(join(root, "volumes"), "[]"); + const recovered = await command(root, ["down", "--recover", "--json"]); + expect(recovered.code).toBe(1); + const requests: string[][] = (await Bun.file(join(root, "requests")).text()) + .trim() + .split("\n") + .map((line) => JSON.parse(line)); + expect( + requests.filter((args) => args[0] === "compose" && args.includes("down")) + ).toHaveLength(0); + expect(await savedState(root)).toEqual(before); + expect(await Bun.file(join(root, "volumes")).json()).toEqual([]); + }, + RETAINED_WORKFLOW_BUDGET_MS +); diff --git a/tests/upgrade-check-failure.test.ts b/tests/upgrade-check-failure.test.ts index 6948c39eb..780a68a9a 100644 --- a/tests/upgrade-check-failure.test.ts +++ b/tests/upgrade-check-failure.test.ts @@ -1,35 +1,45 @@ -import { expect, test } from "bun:test"; -import { CommandError } from "../src/lib/shell.ts"; -import { upgradeCheckFailureClass } from "../src/lib/upgrade-check-failure.ts"; +import { afterEach, expect, test } from "bun:test"; +import { mkdtemp, realpath, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { fixtureChildren } from "./helpers/fixture-children.ts"; +import { privateCliEnv } from "./helpers/private-cli-env.ts"; -test("upgrade diagnostics classify failures without exposing messages, output or arbitrary error names", () => { +const children = fixtureChildren("upgrade-check-failure"); +const roots: string[] = []; +afterEach(async () => { + await children.stopAll(); + for (const root of roots.splice(0)) { + await rm(root, { recursive: true, force: true }); + } +}); + +test("upgrade diagnostics classify failures without exposing messages, output or arbitrary error names", async () => { const sentinel = "fixture-value-must-stay-private"; - const cases: [unknown, string][] = [ - [new TypeError(sentinel), "type-error"], - [new SyntaxError(sentinel), "parse-error"], - [new RangeError(sentinel), "range-error"], - [ - Object.assign(new Error(sentinel), { code: "EACCES" }), - "permission-denied", + const root = await realpath( + await mkdtemp(join(tmpdir(), "upgrade-failure-")) + ); + roots.push(root); + const result = await children.run({ + cmd: [ + process.execPath, + "--no-env-file", + join(import.meta.dir, "fixtures", "upgrade-check-failure-worker.ts"), ], - [Object.assign(new Error(sentinel), { code: "ENOENT" }), "missing-path"], - [ - Object.assign(new Error(sentinel), { code: sentinel, name: sentinel }), - "unexpected-error", - ], - [ - new CommandError({ - cmd: [sentinel], - exitCode: 1, - stdout: sentinel, - stderr: sentinel, - }), - "command-error", - ], - [sentinel, "unexpected-error"], - ]; - for (const [error, expected] of cases) { - expect(upgradeCheckFailureClass(error)).toBe(expected); - expect(upgradeCheckFailureClass(error)).not.toContain(sentinel); - } + cwd: root, + env: privateCliEnv({ root, env: { PATH: "/nonexistent" } }), + }); + expect(result.exitCode).toBe(0); + expect(result.stderr).toBe(""); + expect(result.stdout).not.toContain(sentinel); + expect(JSON.parse(result.stdout)).toEqual([ + "type-error", + "parse-error", + "range-error", + "permission-denied", + "missing-path", + "unexpected-error", + "command-error", + "unexpected-error", + ]); }); From 35374a94537e13f6cec74fbd52eadd82197f1609 Mon Sep 17 00:00:00 2001 From: Kit <344842+roodboi@users.noreply.github.com> Date: Sat, 14 Nov 2026 10:06:16 +0000 Subject: [PATCH 24/32] fix(env): resolve data-home aliases before migration metadata paths --- docs/reference/env-migration.md | 10 +++- src/lib/env-migration-home.ts | 42 ++++++++++++++ src/lib/env-migration-snapshot.ts | 9 ++- src/lib/env-migration-state.ts | 4 +- tests/env-migration-journal.test.ts | 87 ++++++++++++++++++++++++++++ tests/env-migration-provider.test.ts | 65 ++++++++++++++++++++- 6 files changed, 207 insertions(+), 10 deletions(-) create mode 100644 src/lib/env-migration-home.ts diff --git a/docs/reference/env-migration.md b/docs/reference/env-migration.md index 37bdcacc4..9d678152b 100644 --- a/docs/reference/env-migration.md +++ b/docs/reference/env-migration.md @@ -79,9 +79,13 @@ without generating them or copying a keychain key to an absent key file. Apply takes a private snapshot of originals and absent destinations before publishing any migration target. The journal and staged payloads live outside working trees under Hack's private data directory. The snapshot manifest and journal hold only names, -paths, statuses, modes, sizes and SHA-256 checksums. Original bytes and staged -payloads are separate private files; they can contain values and keys and must -remain private. +paths, statuses, modes, sizes and SHA-256 checksums. Existing aliases in `HOME` +or `HACK_HOME` are resolved to the data home's real path before locating +journals, payloads and snapshots; +missing child directories are appended without writes during preview. Symlinks +introduced below that resolved root still refuse migration. +Original bytes and staged payloads are separate private files; they can contain +values and keys and must remain private. Each publication has an original checksum, an intended checksum and a completion receipt. A retry skips completed steps and adopts a matching publication whose diff --git a/src/lib/env-migration-home.ts b/src/lib/env-migration-home.ts new file mode 100644 index 000000000..2a8ca9986 --- /dev/null +++ b/src/lib/env-migration-home.ts @@ -0,0 +1,42 @@ +import { lstatSync, realpathSync, statSync } from "node:fs"; +import { basename, dirname, join } from "node:path"; +import { ClosedCliError } from "./cli-result.ts"; +import { resolveGlobalHackDir } from "./config-paths.ts"; +import { isRecord } from "./guards.ts"; + +function dataHomeRefusal(): ClosedCliError { + return new ClosedCliError( + "Env migration cannot resolve its data home. Use an accessible HOME or HACK_HOME directory; values omitted." + ); +} + +/** Resolve user-selected HOME aliases once, before adding owned metadata subdirectories. */ +export function resolveEnvMigrationDataHome(): string { + let existing = resolveGlobalHackDir(); + const missing: string[] = []; + for (;;) { + try { + // lstat distinguishes an absent component from a dangling symlink. + lstatSync(existing); + } catch (error) { + if (isRecord(error) && error.code === "ENOENT") { + const parent = dirname(existing); + if (parent !== existing) { + missing.unshift(basename(existing)); + existing = parent; + continue; + } + } + throw dataHomeRefusal(); + } + try { + const canonical = realpathSync(existing); + if (!statSync(canonical).isDirectory()) { + throw dataHomeRefusal(); + } + return join(canonical, ...missing); + } catch { + throw dataHomeRefusal(); + } + } +} diff --git a/src/lib/env-migration-snapshot.ts b/src/lib/env-migration-snapshot.ts index 8148f022c..72efb2636 100644 --- a/src/lib/env-migration-snapshot.ts +++ b/src/lib/env-migration-snapshot.ts @@ -9,10 +9,11 @@ import { readMigrationFile, sameMigrationFile, } from "./env-migration-files.ts"; +import { resolveEnvMigrationDataHome } from "./env-migration-home.ts"; import { isRecord } from "./guards.ts"; import { createUpgradeSnapshot, - defaultUpgradeSnapshotRoot, + type defaultUpgradeSnapshotRoot, readUpgradeSnapshot, recordUpgradeSnapshotAfter, restoreUpgradeSnapshot, @@ -69,7 +70,8 @@ type SnapshotLibrary = { export function envMigrationSnapshots(): EnvMigrationSnapshots { return bindEnvMigrationSnapshots({ createUpgradeSnapshot, - defaultUpgradeSnapshotRoot, + defaultUpgradeSnapshotRoot: () => + join(resolveEnvMigrationDataHome(), "upgrade", "snapshots"), readUpgradeSnapshot, recordUpgradeSnapshotAfter, restoreUpgradeSnapshot, @@ -256,6 +258,9 @@ export function bindEnvMigrationSnapshots( await realpath(existing), relative(existing, root) ); + if (canonical !== root) { + throw migrationRefusal(); + } await assertMigrationStorageOutsideWorkingTrees( join(canonical, "manifest.json"), { projectRoot } diff --git a/src/lib/env-migration-state.ts b/src/lib/env-migration-state.ts index 85a9ee332..16e9851d7 100644 --- a/src/lib/env-migration-state.ts +++ b/src/lib/env-migration-state.ts @@ -3,8 +3,8 @@ import { createHash } from "node:crypto"; import { lstat, realpath } from "node:fs/promises"; import { join, resolve } from "node:path"; import { ClosedCliError } from "./cli-result.ts"; -import { resolveGlobalHackDir } from "./config-paths.ts"; import { migrationRefusal, readMigrationFile } from "./env-migration-files.ts"; +import { resolveEnvMigrationDataHome } from "./env-migration-home.ts"; import { isRecord } from "./guards.ts"; const verification = new AsyncLocalStorage(); @@ -12,7 +12,7 @@ const verification = new AsyncLocalStorage(); /** Read-only path calculation shared by the journal owner and every env reader. */ export function migrationJournalPath(projectRoot: string): string { return join( - resolveGlobalHackDir(), + resolveEnvMigrationDataHome(), "env-migrations", createHash("sha256").update(projectRoot).digest("hex"), "journal.json" diff --git a/tests/env-migration-journal.test.ts b/tests/env-migration-journal.test.ts index ed8b3fee2..2014355e7 100644 --- a/tests/env-migration-journal.test.ts +++ b/tests/env-migration-journal.test.ts @@ -8,6 +8,7 @@ import { readFile, realpath, rm, + symlink, writeFile, } from "node:fs/promises"; import { tmpdir } from "node:os"; @@ -140,6 +141,92 @@ function expectRestored( expect(actual).toEqual(before); } +test("implicit migration resumes and readers find its journal through an aliased data home", async () => { + const actualHome = join(root, "canonical-state"); + const alias = join(root, "state-link"); + await mkdir(actualHome, { mode: 0o700 }); + await symlink(actualHome, alias); + process.env.HACK_HOME = join(alias, "not-created-yet"); + const preview = await previewEnvMigration(project); + expect(await readdir(actualHome)).toEqual([]); + await expect( + applyEnvMigration({ + project, + snapshots: envMigrationSnapshots(), + acceptTrackedChanges: true, + hooks: { + boundary: async ({ phase }) => { + if (phase === "prepared") { + throw new Error("synthetic journal interruption"); + } + }, + }, + }) + ).rejects.toThrow("synthetic journal interruption"); + await expect( + resolveProjectEnvConfig({ ...project, envName: null }) + ).rejects.toThrow("unfinished"); + await migrateLegacyEnvForCommand({ ...project, materialize: false }); + expect( + (await resolveProjectEnvConfig({ ...project, envName: null }))?.globalEnv + .TOKEN + ).toBe(SENTINEL); + expect((await previewEnvMigration(project)).journalPath).toBe( + preview.journalPath + ); + const canonical = join(actualHome, "not-created-yet"); + process.env.HACK_HOME = canonical; + expect((await previewEnvMigration(project)).journalPath).toBe( + preview.journalPath + ); +}); + +for (const name of ["env-migrations", "upgrade"] as const) { + test(`data-home alias support still refuses a planted ${name} symlink below that root`, async () => { + const actualHome = join(root, "canonical-state"); + const alias = join(root, "state-link"); + const outside = join(root, "other-storage"); + await mkdir(actualHome, { mode: 0o700 }); + await mkdir(outside, { mode: 0o700 }); + await symlink(actualHome, alias); + await symlink(outside, join(actualHome, name)); + process.env.HACK_HOME = alias; + const before = await projectFiles(); + await expect( + applyEnvMigration({ + project, + snapshots: envMigrationSnapshots(), + acceptTrackedChanges: true, + }) + ).rejects.toThrow(); + expect(await projectFiles()).toEqual(before); + expect(await readdir(outside)).toEqual([]); + }); +} + +for (const kind of ["dangling", "file"] as const) { + test(`an unresolved ${kind} data home refuses without writes or private diagnostics`, async () => { + const path = join(root, "refused-home"); + if (kind === "dangling") { + await symlink(join(root, "absent-target"), path); + } else { + await writeFile(path, SENTINEL); + } + process.env.HACK_HOME = path; + const before = await projectFiles(); + const error = await previewEnvMigration(project).then( + () => null, + (caught: unknown) => caught + ); + expect(error).toBeInstanceOf(Error); + if (error instanceof Error) { + expect(error.message).toContain("cannot resolve its data home"); + expect(error.message).not.toContain(SENTINEL); + } + expect(await projectFiles()).toEqual(before); + }); +} + test("an already modern project without legacy inputs previews complete and applies without creating state", async () => { for (const name of ["hack.env.json", ".env", ".env.qa"]) { await rm(join(project.projectDir, name)); diff --git a/tests/env-migration-provider.test.ts b/tests/env-migration-provider.test.ts index 1a28e2f0d..cccf90283 100644 --- a/tests/env-migration-provider.test.ts +++ b/tests/env-migration-provider.test.ts @@ -8,6 +8,7 @@ import { readFile, realpath, rm, + symlink, writeFile, } from "node:fs/promises"; import { tmpdir } from "node:os"; @@ -123,7 +124,11 @@ async function runMigrationCli( ]); } -async function runCliCommand(projectRoot: string, args: readonly string[]) { +async function runCliCommand( + projectRoot: string, + args: readonly string[], + storage?: { readonly home: string; readonly hackHome?: string } +) { if (compiled && !isAbsolute(compiled)) { throw new Error("Compiled test executable must be absolute"); } @@ -139,9 +144,9 @@ async function runCliCommand(projectRoot: string, args: readonly string[]) { cwd: projectRoot, env: privateCliEnv({ root, - home: join(root, "home"), + home: storage?.home ?? join(root, "home"), env: { - HACK_HOME: join(root, "state"), + HACK_HOME: storage ? storage.hackHome : join(root, "state"), HACK_NO_INTERACTIVE: "1", HACK_SECRETS_DISABLE_KEYCHAIN_FALLBACK: "1", PATH: "/nonexistent", @@ -179,6 +184,60 @@ function pathField(data: Record, name: string): string { return path; } +for (const location of ["HOME", "HACK_HOME", "missing-child"] as const) { + test(`production migration resolves an uncanonicalized ${location} alias for preview, apply, no-op and rollback`, async () => { + const project = await fixture(); + const alias = join(root, "storage-alias"); + const target = join(root, "actual-storage"); + await mkdir(target, { mode: 0o700 }); + await symlink(target, alias); + const storage = + location === "HOME" + ? { home: alias } + : { + home: join(root, "home"), + hackHome: + location === "missing-child" ? join(alias, "new-data") : alias, + }; + const actualHome = + location === "HOME" + ? join(target, ".hack") + : location === "missing-child" + ? join(target, "new-data") + : target; + const command = async (flags: readonly string[]) => { + const result = await runCliCommand( + project.projectRoot, + ["env", "migrate", "--path", project.projectRoot, "--json", ...flags], + storage + ); + expect(result.exitCode, result.stderr).toBe(0); + return JSON.parse(result.stdout).data; + }; + const preview = await command([]); + expect(preview.status).toBe("ready"); + expect(preview.journalPath).toStartWith(`${actualHome}/env-migrations/`); + expect(await readdir(target)).toEqual([]); + const applied = await command(["--apply"]); + expect(applied.status).toBe("applied"); + expect(applied.manifestPath).toStartWith( + `${actualHome}/upgrade/snapshots/` + ); + expect((await command(["--apply"])).status).toBe("already_complete"); + const rollback = await command([ + "--rollback-manifest", + applied.manifestPath, + ]); + expect(rollback.status).toBe("restored"); + expect( + await (await resolveSecretStore(project)).get({ key: "SECRET" }) + ).toBe(sentinel); + expect(await readFile(join(project.projectDir, ".env"), "utf8")).toContain( + sentinel + ); + }, 45_000); +} + test("production CLI explicitly repairs Compose after implicit env exec, then rolls the whole migration back", async () => { const project = await fixture(); const composePath = join(project.projectDir, "docker-compose.yml"); From 2e0b155b01efbaa9ebd73ae876138511c4258da4 Mon Sep 17 00:00:00 2001 From: Kit <344842+roodboi@users.noreply.github.com> Date: Sat, 14 Nov 2026 10:25:11 +0000 Subject: [PATCH 25/32] fix(env): verify effective key ignores under the publication lock --- docs/reference/env-migration.md | 8 +- src/lib/env-migration.ts | 12 ++- src/lib/project-env-config.ts | 29 +++++++- tests/env-migration-journal.test.ts | 110 ++++++++++++++++++++++++++++ 4 files changed, 153 insertions(+), 6 deletions(-) diff --git a/docs/reference/env-migration.md b/docs/reference/env-migration.md index 9d678152b..902183d2b 100644 --- a/docs/reference/env-migration.md +++ b/docs/reference/env-migration.md @@ -136,8 +136,12 @@ remains an explicit recovery option, not a prerequisite to inspect the plan. Migration refuses a Git-tracked project key path even when its file is absent from the worktree, including case-folded aliases such as `.Hack.Secret.Key`. Tracked-change consent does not allow publishing key bytes at that path. -Git tracking is checked again under the key lock before publication; -keep the key out of the index before resuming a refused migration. +Git tracking is checked again under the key lock before publication. Keep the +key out of the index before resuming a refused migration. Checkout-local keys +also require an effective Git ignore result at that point +and immediately before publication. Symlinked root `.gitignore` files refuse; +Git does not use them as ignore rules. An unknown Git result inside a checkout +refuses without publishing the key. Completed publication payloads are removed; retries also sweep receipted orphan staging directories. Unknown or edited entries are retained, reported as diff --git a/src/lib/env-migration.ts b/src/lib/env-migration.ts index 00fa1d545..20eea4c1b 100644 --- a/src/lib/env-migration.ts +++ b/src/lib/env-migration.ts @@ -36,7 +36,7 @@ import { acquireLegacyEnvCleanupContext, assertLegacyEnvMigrationDelivery, assertMachineLocalFileGitPrivate, - assertProjectEnvKeyUntracked, + assertMigrationProjectEnvKeyPrivate, assertProjectEnvLayerPaths, inspectLegacyComposeEnvFileReferences, isGitTrackedOrUnknownProjectPath, @@ -800,7 +800,10 @@ async function publishEntry( ): Promise { const run = async () => { if (entry.kind === "key") { - await assertProjectEnvKeyUntracked({ keyPath: entry.path }); + await assertMigrationProjectEnvKeyPrivate({ + ...project, + keyPath: entry.path, + }); } await checkPrivateDirectory(dirname(entry.payload)); const current = await readState(entry.path); @@ -842,7 +845,10 @@ async function publishEntry( refusal: refuse, beforeRename: async () => { if (entry.kind === "key") { - await assertProjectEnvKeyUntracked({ keyPath: entry.path }); + await assertMigrationProjectEnvKeyPrivate({ + ...project, + keyPath: entry.path, + }); } if (!same((await readState(entry.path)).state, entry.before)) { throw refuse(); diff --git a/src/lib/project-env-config.ts b/src/lib/project-env-config.ts index c4e3c1fa7..4f1e9c6e6 100644 --- a/src/lib/project-env-config.ts +++ b/src/lib/project-env-config.ts @@ -48,6 +48,7 @@ import { ClosedCliError, HackCliError } from "./cli-result.ts"; import { resolveGlobalConfigPath } from "./config-paths.ts"; import { parseDotEnv, serializeDotEnv } from "./env.ts"; import { withEnvLayerLock } from "./env-layer-lock.ts"; +import { readMigrationFile } from "./env-migration-files.ts"; import { assertEnvMigrationReadable, EnvMigrationPendingError, @@ -5334,6 +5335,32 @@ export async function assertProjectEnvKeyUntracked(opts: { } } +/** Migration key publishers recheck effective Git protection under their key owner lock. */ +export async function assertMigrationProjectEnvKeyPrivate(opts: { + readonly projectRoot: string; + readonly keyPath: string; +}): Promise { + await assertProjectEnvKeyUntracked(opts); + const shared = await resolveProjectEnvSharedKeyLocation(opts); + // Only the verified Git common-directory key is outside worktree ignore rules. + if (opts.keyPath === shared?.path) { + return; + } + const projectRoot = dirname(opts.keyPath); + // Git ignores symlinked .gitignore files, even if their target lists the key. + await readMigrationFile(resolve(projectRoot, ".gitignore")); + if ( + (await machineLocalLayerGitProblem({ + projectRoot, + path: opts.keyPath, + })) !== null + ) { + throw new ClosedCliError( + "Refusing env migration: the project key is not verifiably untracked and ignored by Git. Repair the root .gitignore and make git status work before retrying; no key was published (values omitted)." + ); + } +} + export type EnsureProjectEnvSecretKeyResult = { readonly keyPath: string; readonly keyText: string; @@ -6464,7 +6491,7 @@ async function prepareMigrationKeyIgnore( add: AddPreparedMigrationFile ): Promise { const path = resolve(opts.projectRoot, ".gitignore"); - const existing = await readTextFile(path); + const existing = (await readMigrationFile(path)).text; // Appending after possible negations ensures a freshly written private key // is ignored. Existing effective final entries require no change. if (existing?.split("\n").at(-2) === PROJECT_ENV_KEY_FILENAME) { diff --git a/tests/env-migration-journal.test.ts b/tests/env-migration-journal.test.ts index 2014355e7..de80321c0 100644 --- a/tests/env-migration-journal.test.ts +++ b/tests/env-migration-journal.test.ts @@ -1125,6 +1125,116 @@ async function migrationFixtureGit(args: readonly string[]): Promise { expect(result.exitCode, result.stderr).toBe(0); } +test("secret migration refuses a symlinked root ignore file even when its target lists the key", async () => { + await addLegacySecret(); + await migrationFixtureGit(["init", "--quiet"]); + const ignore = join(project.projectRoot, ".gitignore"); + const target = join(root, "ignore-target"); + await writeFile(target, ".hack.secret.key\n"); + await symlink(target, ignore); + const before = await projectFiles(); + await expect(previewEnvMigration(project)).rejects.toThrow(); + await expect( + applyEnvMigration({ + project, + snapshots: fixtureEnvSnapshots(root), + acceptTrackedChanges: true, + }) + ).rejects.toThrow(); + expect(await projectFiles()).toEqual(before); + expect((await lstat(ignore)).isSymbolicLink()).toBe(true); + expect(await readFile(target, "utf8")).toBe(".hack.secret.key\n"); + expect( + await Bun.file(join(project.projectRoot, ".hack.secret.key")).exists() + ).toBe(false); + expect( + await Bun.file(envMigrationJournalPath(project.projectRoot)).exists() + ).toBe(false); +}); + +test("key publication rechecks effective ignores after waiting for its lock and resumes after repair", async () => { + await addLegacySecret(); + await migrationFixtureGit(["init", "--quiet"]); + const ignore = join(project.projectRoot, ".gitignore"); + await writeFile(ignore, ".hack.secret.key\n"); + const keyPath = join(project.projectRoot, ".hack.secret.key"); + const preview = await previewEnvMigration(project); + const keyIndex = preview.files.findIndex((file) => file.path === keyPath); + const location = await resolveProjectEnvSharedKeyLocation(project); + if (!location || keyIndex < 0) { + throw new Error("Fixture expected key publication"); + } + const entered = Promise.withResolvers(); + const release = Promise.withResolvers(); + const reached = Promise.withResolvers(); + const holder = withProjectsRegistryLock({ + lockPath: `${location.path}.lock`, + run: async () => { + entered.resolve(); + await release.promise; + }, + }); + await entered.promise; + const snapshots = fixtureEnvSnapshots(root); + const applying = applyEnvMigration({ + project, + snapshots, + acceptTrackedChanges: true, + hooks: { + boundary: async ({ phase, index }) => { + if (phase === "before_publish" && index === keyIndex) { + reached.resolve(); + } + }, + }, + }).then( + () => null, + (error: unknown) => error + ); + try { + await Promise.race([ + reached.promise, + applying.then(() => { + throw new Error("Migration settled before key publication"); + }), + ]); + await writeFile(ignore, ".hack.secret.key\n!.hack.secret.key\n"); + } finally { + release.resolve(); + await holder; + } + const error = await applying; + expect(error).toBeInstanceOf(Error); + if (error instanceof Error) { + expect(error.message).toContain("ignored"); + expect(error.message).not.toContain(SENTINEL); + } + expect(await Bun.file(keyPath).exists()).toBe(false); + expect( + await Bun.file(join(project.projectDir, "hack.env.default.yaml")).exists() + ).toBe(false); + await writeFile(ignore, ".hack.secret.key\n"); + expect( + (await apply({ project, snapshots, acceptTrackedChanges: true })).status + ).toBe("applied"); + await migrationFixtureGit([ + "check-ignore", + "--no-index", + "--", + ".hack.secret.key", + ]); + await migrationFixtureGit(["add", "--all"]); + const tracked = await runFixtureGit({ + cwd: project.projectRoot, + args: ["ls-files", "--", ".hack.secret.key"], + }); + expect(tracked.stdout).toBe(""); + expect( + (await resolveProjectEnvConfig({ ...project, envName: null }))?.serviceEnv + .api?.SEC + ).toBe(SENTINEL); +}, 20_000); + for (const trackedName of [".hack.secret.key", ".Hack.Secret.Key"]) { test(`secret migration refuses an absent key tracked as ${trackedName} before planning or writes`, async () => { await addLegacySecret(); From 5f83bc8f0677226304d483597b769e42b19e8a03 Mon Sep 17 00:00:00 2001 From: Kit <344842+roodboi@users.noreply.github.com> Date: Sat, 14 Nov 2026 14:52:30 +0000 Subject: [PATCH 26/32] fix(env): verify existing layer delivery before Compose repair --- docs/reference/env-migration.md | 5 ++ docs/versions/v5/guides/upgrade-from-v4.md | 4 ++ src/lib/project-env-config.ts | 22 +++++- tests/env-migration-journal.test.ts | 84 ++++++++++++++++++++++ tests/env-migration-provider.test.ts | 27 +++++++ 5 files changed, 140 insertions(+), 2 deletions(-) diff --git a/docs/reference/env-migration.md b/docs/reference/env-migration.md index 902183d2b..e7b2dab03 100644 --- a/docs/reference/env-migration.md +++ b/docs/reference/env-migration.md @@ -111,6 +111,11 @@ When implicit migration retained legacy Compose `env_file` references, a later explicit preview or `doctor --migrate-env-config` can offer their repair. This is a new journalled phase: completed env layers are not published again, tracked Compose edits require fresh consent, and migrated values must still resolve. +The first explicit migration also checks delivery before removing Compose inputs +when a contract and env layers already coexist without a journal. Missing, +changed or unavailable contract values refuse preview and apply without writes; +keep the legacy inputs and repair the layers before retrying. Implicit migration +continues to retain those Compose references. The Compose before image must be unchanged and covered by the original intact snapshot; the same manifest rolls back the whole migration, including this later repair. A fresh read revision permits unrelated additions in existing diff --git a/docs/versions/v5/guides/upgrade-from-v4.md b/docs/versions/v5/guides/upgrade-from-v4.md index e7ceb357e..dda9394b6 100644 --- a/docs/versions/v5/guides/upgrade-from-v4.md +++ b/docs/versions/v5/guides/upgrade-from-v4.md @@ -559,6 +559,10 @@ mode still cannot be combined with `--json`. - Migration can repair Compose `env_file` references and legacy config keys, but keeps every old contract, dotenv file, store, backend key and store pointer. Cleanup never runs as part of apply or Doctor migration. + If a contract and env layers already coexist without a journal, removing a + legacy Compose input requires the layers to deliver its contract values. + Preview and apply refuse missing or changed delivery without writes; keep the + legacy inputs and repair the layers before retrying. - Env-consuming commands can migrate implicitly. The implicit path keeps the legacy files and does not edit Compose, but can rewrite legacy config keys. It is still a write. An unfinished apply or rollback blocks env readers until diff --git a/src/lib/project-env-config.ts b/src/lib/project-env-config.ts index 4f1e9c6e6..ccdea89ad 100644 --- a/src/lib/project-env-config.ts +++ b/src/lib/project-env-config.ts @@ -6692,7 +6692,10 @@ async function prepareUndeclaredMigration( } async function prepareMigrationCompose( - opts: PreparedMigrationProject, + opts: PreparedMigrationProject & { + readonly hasContract: boolean; + readonly existingContractOverlays?: readonly string[]; + }, add: AddPreparedMigrationFile ): Promise { const composeFile = resolve(opts.projectDir, PROJECT_COMPOSE_FILENAME); @@ -6701,6 +6704,14 @@ async function prepareMigrationCompose( composeFile, }); if (references.length > 0) { + // Existing layers are kept, so they must already deliver every contract + // value before the first journal can remove its legacy Compose inputs. + if (opts.hasContract && opts.existingContractOverlays) { + await assertLegacyEnvMigrationDelivery({ + ...opts, + overlayNames: opts.existingContractOverlays, + }); + } const parsed = await readComposeFile({ composeFile }); if (!parsed) { throw new ClosedCliError( @@ -6907,7 +6918,14 @@ async function prepareLegacyProjectEnvMigrationInternal(opts: { } const composeFile = resolve(opts.projectDir, PROJECT_COMPOSE_FILENAME); if (opts.repairCompose !== false) { - await prepareMigrationCompose(opts, add); + await prepareMigrationCompose( + { + ...opts, + hasContract: contract.exists, + existingContractOverlays: layersExist ? overlayNames : undefined, + }, + add + ); } const composeEnvFileReferences = opts.repairCompose === false diff --git a/tests/env-migration-journal.test.ts b/tests/env-migration-journal.test.ts index de80321c0..91fb0c276 100644 --- a/tests/env-migration-journal.test.ts +++ b/tests/env-migration-journal.test.ts @@ -258,6 +258,90 @@ test("an already modern project without legacy inputs previews complete and appl expect(await Bun.file(preview.journalPath).exists()).toBe(false); }); +for (const delivery of [ + "missing-base", + "changed-base", + "missing-overlay", + "delivered", +] as const) { + test(`first Compose repair verifies existing contract layers (${delivery})`, async () => { + const compose = join(project.projectDir, "docker-compose.yml"); + await writeFile( + compose, + "services:\n api:\n image: alpine:3.22\n env_file: .env\n" + ); + const writeLayer = async (name: string, value: string | null) => { + await writeFile( + join(project.projectDir, `hack.env.${name}.yaml`), + JSON.stringify({ + version: 1, + environment: name, + secretsprovider: "project_key", + values: { global: value === null ? {} : { TOKEN: value } }, + }) + ); + }; + await writeLayer( + "default", + delivery === "missing-base" + ? null + : delivery === "changed-base" + ? QA + : SENTINEL + ); + if (delivery !== "missing-overlay") { + await writeLayer("qa", QA); + } + const before = await projectFiles(); + const path = envMigrationJournalPath(project.projectRoot); + if (delivery !== "delivered") { + await expect(previewEnvMigration(project)).rejects.toThrow(); + await expect( + applyEnvMigration({ + project, + snapshots: envMigrationSnapshots(), + acceptTrackedChanges: true, + }) + ).rejects.toThrow(); + expect(await projectFiles()).toEqual(before); + expect(await Bun.file(path).exists()).toBe(false); + // Keeping Compose references is safe even before the layers deliver them. + await migrateLegacyEnvForCommand({ ...project, materialize: false }); + expect(await readFile(compose, "base64")).toBe(before.get(compose)); + expect(await readFile(join(project.projectDir, ".env"), "utf8")).toBe( + `TOKEN=${SENTINEL}\n` + ); + return; + } + const preview = await previewEnvMigration(project); + expect(preview.files).toContainEqual( + expect.objectContaining({ path: compose, action: "change" }) + ); + expect(JSON.stringify(preview)).not.toContain(SENTINEL); + const snapshots = envMigrationSnapshots(); + const applied = await apply({ + project, + snapshots, + acceptTrackedChanges: true, + }); + expect(await readFile(compose, "utf8")).not.toContain("env_file"); + expect( + (await resolveProjectEnvConfig({ ...project, envName: null }))?.globalEnv + .TOKEN + ).toBe(SENTINEL); + expect( + (await resolveProjectEnvConfig({ ...project, envName: "qa" }))?.globalEnv + .TOKEN + ).toBe(QA); + await rollbackEnvMigration({ + project, + snapshots, + manifestPath: applied.manifestPath, + }); + expectRestored(await projectFiles(), before); + }); +} + test("preview makes no writes and exactly names what apply publishes; completed retry is a no-op", async () => { const before = await projectFiles(); const preview = await previewEnvMigration(project); diff --git a/tests/env-migration-provider.test.ts b/tests/env-migration-provider.test.ts index cccf90283..b417fd579 100644 --- a/tests/env-migration-provider.test.ts +++ b/tests/env-migration-provider.test.ts @@ -238,6 +238,33 @@ for (const location of ["HOME", "HACK_HOME", "missing-child"] as const) { }, 45_000); } +test("production CLI refuses first Compose repair when pre-existing layers do not deliver legacy contract values", async () => { + const project = await fixture(); + const composePath = join(project.projectDir, "docker-compose.yml"); + const compose = + "services:\n api:\n image: alpine:3.22\n env_file: .env\n"; + const layerPath = join(project.projectDir, "hack.env.default.yaml"); + const layer = + "version: 1\nenvironment: default\nsecretsprovider: project_key\nvalues:\n global: {}\n"; + await writeFile(composePath, compose); + await writeFile(layerPath, layer); + const before = (await readdir(project.projectDir)).sort(); + for (const args of [[], ["--apply"]]) { + const result = await cli(project.projectRoot, args); + expect(result.exitCode).toBe(1); + expect(result.envelope.ok).toBe(false); + expect(await readFile(composePath, "utf8")).toBe(compose); + expect(await readFile(layerPath, "utf8")).toBe(layer); + expect(await readFile(join(project.projectDir, ".env"), "utf8")).toBe( + `TOKEN=${sentinel}\n` + ); + expect((await readdir(project.projectDir)).sort()).toEqual(before); + expect( + await Bun.file(envMigrationJournalPath(project.projectRoot)).exists() + ).toBe(false); + } +}); + test("production CLI explicitly repairs Compose after implicit env exec, then rolls the whole migration back", async () => { const project = await fixture(); const composePath = join(project.projectDir, "docker-compose.yml"); From 57bb2f543d08d400b9b21bc6af16e28d2c370ce0 Mon Sep 17 00:00:00 2001 From: Kit <344842+roodboi@users.noreply.github.com> Date: Sat, 14 Nov 2026 14:53:25 +0000 Subject: [PATCH 27/32] refactor(env): keep Compose repair admission in its owner --- src/lib/project-env-config.ts | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/lib/project-env-config.ts b/src/lib/project-env-config.ts index ccdea89ad..2f4df27ac 100644 --- a/src/lib/project-env-config.ts +++ b/src/lib/project-env-config.ts @@ -6694,7 +6694,8 @@ async function prepareUndeclaredMigration( async function prepareMigrationCompose( opts: PreparedMigrationProject & { readonly hasContract: boolean; - readonly existingContractOverlays?: readonly string[]; + readonly layersExist: boolean; + readonly overlayNames: readonly string[]; }, add: AddPreparedMigrationFile ): Promise { @@ -6706,10 +6707,9 @@ async function prepareMigrationCompose( if (references.length > 0) { // Existing layers are kept, so they must already deliver every contract // value before the first journal can remove its legacy Compose inputs. - if (opts.hasContract && opts.existingContractOverlays) { + if (opts.hasContract && opts.layersExist) { await assertLegacyEnvMigrationDelivery({ ...opts, - overlayNames: opts.existingContractOverlays, }); } const parsed = await readComposeFile({ composeFile }); @@ -6922,7 +6922,8 @@ async function prepareLegacyProjectEnvMigrationInternal(opts: { { ...opts, hasContract: contract.exists, - existingContractOverlays: layersExist ? overlayNames : undefined, + layersExist, + overlayNames, }, add ); From a219fc97dc088b33fff5d59f8050bba7325e6140 Mon Sep 17 00:00:00 2001 From: Kit <344842+roodboi@users.noreply.github.com> Date: Sat, 14 Nov 2026 14:56:56 +0000 Subject: [PATCH 28/32] test(env): type the retained Compose byte comparison safely --- tests/env-migration-journal.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/env-migration-journal.test.ts b/tests/env-migration-journal.test.ts index 91fb0c276..66263d63c 100644 --- a/tests/env-migration-journal.test.ts +++ b/tests/env-migration-journal.test.ts @@ -307,7 +307,7 @@ for (const delivery of [ expect(await Bun.file(path).exists()).toBe(false); // Keeping Compose references is safe even before the layers deliver them. await migrateLegacyEnvForCommand({ ...project, materialize: false }); - expect(await readFile(compose, "base64")).toBe(before.get(compose)); + expect(await readFile(compose, "base64")).toEqual(before.get(compose)); expect(await readFile(join(project.projectDir, ".env"), "utf8")).toBe( `TOKEN=${SENTINEL}\n` ); From 00ee26a438cb48cb6b79d56e0dc3ba1ff9b8e600 Mon Sep 17 00:00:00 2001 From: Kit <344842+roodboi@users.noreply.github.com> Date: Sat, 14 Nov 2026 15:00:06 +0000 Subject: [PATCH 29/32] test(env): capture definite Compose bytes before repair --- tests/env-migration-journal.test.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/env-migration-journal.test.ts b/tests/env-migration-journal.test.ts index 66263d63c..c2ee71af9 100644 --- a/tests/env-migration-journal.test.ts +++ b/tests/env-migration-journal.test.ts @@ -293,6 +293,7 @@ for (const delivery of [ await writeLayer("qa", QA); } const before = await projectFiles(); + const composeBefore = await readFile(compose, "base64"); const path = envMigrationJournalPath(project.projectRoot); if (delivery !== "delivered") { await expect(previewEnvMigration(project)).rejects.toThrow(); @@ -307,7 +308,7 @@ for (const delivery of [ expect(await Bun.file(path).exists()).toBe(false); // Keeping Compose references is safe even before the layers deliver them. await migrateLegacyEnvForCommand({ ...project, materialize: false }); - expect(await readFile(compose, "base64")).toEqual(before.get(compose)); + expect(await readFile(compose, "base64")).toBe(composeBefore); expect(await readFile(join(project.projectDir, ".env"), "utf8")).toBe( `TOKEN=${SENTINEL}\n` ); From 66d1cf9c142d6453ec40c6dccc8ae4bfa533ba68 Mon Sep 17 00:00:00 2001 From: Kit <344842+roodboi@users.noreply.github.com> Date: Sat, 14 Nov 2026 16:46:02 +0000 Subject: [PATCH 30/32] docs(env): add next.18 migration notes to the v5 release notes The breaking env migration change ships in next.18. List what changed and what to do under a next.18 heading, and point the older Doctor-based migration section at the new commands. Co-Authored-By: Claude Opus 5.5 --- docs/guides/v5-release-notes.md | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/docs/guides/v5-release-notes.md b/docs/guides/v5-release-notes.md index 91a8f5c0c..11af785e2 100644 --- a/docs/guides/v5-release-notes.md +++ b/docs/guides/v5-release-notes.md @@ -14,6 +14,27 @@ beta. `hack dev`, added in 4.3.0, is part of the core. This page is the readable summary. The generated [CHANGELOG](../../CHANGELOG.md) lists every change by commit. +## next.18: what changed and what to do + +These are source changes intended for `5.0.0-next.18`, not a claim that the +release is published. An installed next.17 bundle keeps its original behavior. + +### Breaking change: env migration keeps legacy files until an explicit cleanup + +Env migration (HACK-2191) can now be previewed, resumed and rolled back, and it +no longer removes legacy inputs in the same step. + +| Change | What to do | +| --- | --- | +| **Migration keeps every legacy copy.** `hack env migrate --apply`, `hack doctor --migrate-env-config` and implicit migration now keep `.hack/hack.env.json`, legacy dotenv files and other legacy inputs. Implicit migration also keeps Compose `env_file` references. | Do not expect the old files to be gone after migration. Scripts that checked for their removal must run the separate cleanup below. | +| **Preview first.** `hack env migrate --json` is read-only and reports file actions, paths, key names and tracked status, never values (versioned contract in the [env migration protocol](../reference/env-migration.md)). | Review the preview before `--apply`. Add `--accept-tracked-env-changes` only to consent to the previewed tracked edits. | +| **Apply is journalled and resumable.** `--apply` snapshots originals outside the working tree and records each step. Repeating the same command after an interruption resumes; a completed migration is a no-op. Snapshot storage must be on the same filesystem as the project files. | After an interruption, rerun the same `--apply` command. Keep the snapshot and journal until the upgrade is accepted. | +| **Cleanup is a separate, later command.** `--cleanup --apply` runs only after a completed migration and a successful complete-project `hack up`. It rechecks delivery, the original snapshot and live inputs before each removal. | Run `hack up`, check the project, then `hack env migrate --cleanup --json` and `--cleanup --apply`. | +| **Rollback.** `--rollback-manifest ` restores originals and refuses when files changed later. `--preserve-changes` restores anyway and keeps those later edits privately for recovery. Project and shared keys are kept. | Keep a private key backup as before. Run `hack doctor` after a rollback to review the restored file modes. | +| **Compose repair refuses when existing layers would not deliver the values.** A checkout that already has env layers must deliver every contract value before Hack removes legacy Compose references. | Add the missing values to the layer, or keep the legacy references (implicit migration keeps them). | + +The full upgrade steps are in [Upgrade to v5](upgrade-to-v5.md#preview-resume-and-undo-env-migration). + ## next.17: what changed and what to do These are source changes intended for `5.0.0-next.17`, not a claim that the @@ -261,6 +282,9 @@ move to the layered YAML format with: hack doctor --migrate-env-config ``` +From next.18, prefer `hack env migrate` to preview, apply, clean up and roll back +in separate steps; see [next.18](#next18-what-changed-and-what-to-do). + Back up the project, env files and original project key before migration. On next.16 or earlier prereleases, explicit Doctor cleanup can remove the project key when a legacy store's `keyPath` names it (HACK-2172). Use a build containing From 969bf0721581cc7e58f56c0fc565e16a5f8d60b1 Mon Sep 17 00:00:00 2001 From: Kit <344842+roodboi@users.noreply.github.com> Date: Sun, 15 Nov 2026 04:46:25 +0000 Subject: [PATCH 31/32] test(env): use a complete activity record in the plan-home fixture Build the fixture's failed, unregistered up record with the production activity writer so it carries exactly the schema's nine keys and no path. It passes with the current shared helper and with the stricter key-set check proposed in #835, so merge order does not matter. Co-Authored-By: Claude Opus 5.5 --- tests/native-plan-home-state.test.ts | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/tests/native-plan-home-state.test.ts b/tests/native-plan-home-state.test.ts index e0dc074cf..425d322e7 100644 --- a/tests/native-plan-home-state.test.ts +++ b/tests/native-plan-home-state.test.ts @@ -6,10 +6,24 @@ import { assertNativePlanRefusalActivity, snapshotNativePlanHome, } from "../scripts/lib/native-plan-home-state.ts"; +import { buildActivityRecord } from "../src/lib/activity-log.ts"; const roots: string[] = []; const sentinel = "synthetic-native-plan-private-value"; -const record = { v: 1, action: "up", outcome: "failed", project: null }; +/** + * A complete failed, unregistered `up` record, built by the production + * writer so it has exactly the schema's keys and no path or value. + */ +const record = buildActivityRecord({ + action: "up", + startedAt: Date.UTC(2026, 0, 1), + durationMs: 42, + project: null, + branch: null, + exitCode: 1, + failedServices: [], + origin: "cli", +}); afterEach(async () => { for (const root of roots.splice(0)) { From 06522fcc3a2ac596163af142cc65ade37efa6591 Mon Sep 17 00:00:00 2001 From: Kit <344842+roodboi@users.noreply.github.com> Date: Sun, 15 Nov 2026 04:55:38 +0000 Subject: [PATCH 32/32] fix(env): look up legacy secrets under the project's effective name Env migration and Doctor's migration used the literal namespace "project" when the config has no name, so an unnamed legacy project could migrate without its stored secrets, or import another project's values from that namespace. Use the same effective name as the env commands that stored them: the configured name, else the checkout directory, as a slug. Co-Authored-By: Claude Opus 5.5 --- docs/reference/env-migration.md | 2 +- src/commands/doctor.ts | 10 ++--- src/commands/env-migrate.ts | 8 ++-- src/commands/env.ts | 9 +---- src/lib/project.ts | 15 ++++++++ tests/env-migration-provider.test.ts | 57 ++++++++++++++++++++++++++-- 6 files changed, 80 insertions(+), 21 deletions(-) diff --git a/docs/reference/env-migration.md b/docs/reference/env-migration.md index e7b2dab03..9ca44ed82 100644 --- a/docs/reference/env-migration.md +++ b/docs/reference/env-migration.md @@ -35,7 +35,7 @@ The preview's `data` has these stable fields: | Field | Meaning | | --- | --- | | `version` | Integer `1`; consumers must reject unsupported versions. | -| `project` | `projectRoot`, `projectDir`, `projectName` and `serviceNames`; paths are absolute and canonical. | +| `project` | `projectRoot`, `projectDir`, `projectName` and `serviceNames`; paths are absolute and canonical. `projectName` is the configured `name`, else the checkout directory, as a slug: the name legacy secret stores are keyed by. | | `status` | `ready` for unfinished migration work; `complete` when migration has completed; `recovery_required` for interrupted cleanup/rollback or an unavailable deferred Compose repair. | | `composeRepair` | Optional `"unavailable"` when completed env migration still has Compose references but their repair cannot be verified. The preview exits 0 with no file actions; apply still refuses. Existing env layers remain usable. | | `action` | `preview`, or `cleanup` with `--cleanup`. | diff --git a/src/commands/doctor.ts b/src/commands/doctor.ts index 753504909..92bcbaa7c 100644 --- a/src/commands/doctor.ts +++ b/src/commands/doctor.ts @@ -245,6 +245,7 @@ import { readProjectConfig, readProjectDefaultEnvConfig, readProjectDevHost, + resolveEffectiveProjectName, } from "../lib/project.ts"; import { runProjectDomainCommand } from "../lib/project-domain-command.ts"; import { @@ -4036,11 +4037,10 @@ export async function assertTrackedEnvMigrationConsent(opts: { if (!project) { return; } - const config = await readProjectConfig(project); const preview = await previewEnvMigration({ projectRoot: project.projectRoot, projectDir: project.projectDir, - projectName: config.name ?? "project", + projectName: await resolveEffectiveProjectName(project), serviceNames: await discoverComposeServiceNames({ composeFile: project.composeFile, }), @@ -4084,11 +4084,10 @@ async function maybeMigrateProjectEnvConfig(opts: { if (!project) { return; } - const config = await readProjectConfig(project); const migrationProject = { projectRoot: project.projectRoot, projectDir: project.projectDir, - projectName: config.name ?? "project", + projectName: await resolveEffectiveProjectName(project), serviceNames: await discoverComposeServiceNames({ composeFile: project.composeFile, }), @@ -4343,12 +4342,11 @@ async function buildEnvMigrationPlanStep(opts: { throw error; } if (modernExists) { - const config = await readProjectConfig(opts.project); const composeFile = opts.project.composeFile; const preview = await previewEnvMigration({ projectRoot: opts.project.projectRoot, projectDir: opts.project.projectDir, - projectName: config.name ?? "project", + projectName: await resolveEffectiveProjectName(opts.project), serviceNames: await discoverComposeServiceNames({ composeFile, }), diff --git a/src/commands/env-migrate.ts b/src/commands/env-migrate.ts index e1f430619..2c9cc215a 100644 --- a/src/commands/env-migrate.ts +++ b/src/commands/env-migrate.ts @@ -18,7 +18,10 @@ import { } from "../lib/env-migration.ts"; import { envMigrationSnapshots } from "../lib/env-migration-snapshot.ts"; import { MissingProjectContextError } from "../lib/missing-project-error.ts"; -import { findProjectContext, readProjectConfig } from "../lib/project.ts"; +import { + findProjectContext, + resolveEffectiveProjectName, +} from "../lib/project.ts"; import { discoverComposeServiceNames } from "../lib/project-env-config.ts"; const optApply = defineOption({ @@ -109,11 +112,10 @@ export async function handleEnvMigrate({ if (!context) { throw new MissingProjectContextError(); } - const config = await readProjectConfig(context); const project = { projectRoot: context.projectRoot, projectDir: context.projectDir, - projectName: config.name ?? "project", + projectName: await resolveEffectiveProjectName(context), serviceNames: await discoverComposeServiceNames({ composeFile: context.composeFile, }), diff --git a/src/commands/env.ts b/src/commands/env.ts index bfad1693e..174101e20 100644 --- a/src/commands/env.ts +++ b/src/commands/env.ts @@ -75,10 +75,8 @@ import { appendHackHostTrustEnvironment } from "../lib/local-ca.ts"; import { MissingProjectContextError } from "../lib/missing-project-error.ts"; import type { ProjectContext } from "../lib/project.ts"; import { - defaultProjectSlugFromPath, findProjectContext, - readProjectConfig, - sanitizeProjectSlug, + resolveEffectiveProjectName, } from "../lib/project.ts"; import { assertMachineLocalFileGitPrivate, @@ -763,10 +761,7 @@ async function resolveProjectForEnv(opts: { } async function resolveProjectName(project: ProjectContext): Promise { - const cfg = await readProjectConfig(project); - const derived = defaultProjectSlugFromPath(project.projectRoot); - const raw = (cfg.name ?? derived).trim(); - return sanitizeProjectSlug(raw.length > 0 ? raw : derived); + return await resolveEffectiveProjectName(project); } async function resolveConfiguredSecretStore(input: { diff --git a/src/lib/project.ts b/src/lib/project.ts index 968dac685..261c538e3 100644 --- a/src/lib/project.ts +++ b/src/lib/project.ts @@ -196,6 +196,21 @@ export function defaultProjectSlugFromPath(repoRoot: string): string { return sanitizeProjectSlug(basename(repoRoot)); } +/** + * The project's effective name: its configured `name`, else the checkout + * directory, as a sanitized slug. Legacy secret stores (keychain service and + * encrypted-file scope) are keyed by this name, so every env read, write and + * migration route must use it to find the same stored values. + */ +export async function resolveEffectiveProjectName( + ctx: ProjectContext +): Promise { + const cfg = await readProjectConfig(ctx); + const derived = defaultProjectSlugFromPath(ctx.projectRoot); + const raw = (cfg.name ?? derived).trim(); + return sanitizeProjectSlug(raw.length > 0 ? raw : derived); +} + export async function readProjectDevHost( ctx: ProjectContext ): Promise { diff --git a/tests/env-migration-provider.test.ts b/tests/env-migration-provider.test.ts index b417fd579..1a4edbc89 100644 --- a/tests/env-migration-provider.test.ts +++ b/tests/env-migration-provider.test.ts @@ -65,14 +65,21 @@ afterEach(async () => { } }); -async function fixture() { - const projectRoot = join(root, "project"); +async function fixture( + opts: { + /** Checkout directory name; the legacy store is keyed by it when unnamed. */ + readonly dir?: string; + /** Configured project name; null omits `name` from the config. */ + readonly name?: string | null; + } = {} +) { + const projectRoot = join(root, opts.dir ?? "project"); const projectDir = join(projectRoot, ".hack"); await mkdir(projectDir, { recursive: true }); await writeFile( join(projectDir, "hack.config.json"), JSON.stringify({ - name: "provider", + ...(opts.name === null ? {} : { name: opts.name ?? "provider" }), controlPlane: { secrets: { backend: "encrypted_file", @@ -99,7 +106,8 @@ async function fixture() { const project = { projectRoot, projectDir, - projectName: "provider", + projectName: + opts.name === null ? (opts.dir ?? "project") : (opts.name ?? "provider"), serviceNames: ["api"], }; await (await resolveSecretStore(project)).set({ @@ -516,6 +524,47 @@ for (const damage of ["manifest", "missing-blob", "changed-blob"] as const) { }); } +for (const route of ["env migrate", "doctor"] as const) { + test( + `${route}: an unnamed legacy project migrates the secrets stored under its directory name`, + async () => { + // Legacy `hack env set` keyed the store by the directory slug when the + // config has no name. A decoy under the generic "project" scope must not + // be used. + const project = await fixture({ dir: "unnamed-app", name: null }); + await ( + await resolveSecretStore({ ...project, projectName: "project" }) + ).set({ key: "SECRET", value: "synthetic-decoy-value" }); + if (route === "env migrate") { + const applied = await cli(project.projectRoot, [ + "--apply", + "--accept-tracked-env-changes", + ]); + expect(applied.exitCode, applied.stderr).toBe(0); + expect(applied.envelope).toMatchObject({ + ok: true, + data: { status: "applied" }, + }); + } else { + // Doctor may exit 1 for unrelated host checks in this fixture; the + // migrated layer is the evidence. + await runCliCommand(project.projectRoot, [ + "doctor", + "--migrate-env-config", + "--accept-tracked-env-changes", + "--path", + project.projectRoot, + ]); + } + expect( + (await resolveProjectEnvConfig({ ...project, envName: null })) + ?.serviceEnv.api?.SECRET + ).toBe(sentinel); + }, + DOCTOR_WORKFLOW_TIMEOUT_MS + ); +} + test("production snapshot provider backs source CLI apply, no-op and guarded rollback", async () => { const project = await fixture(); const preview = await cli(project.projectRoot, []);