diff --git a/.changeset/pull-flow-ids.md b/.changeset/pull-flow-ids.md new file mode 100644 index 000000000..e74675858 --- /dev/null +++ b/.changeset/pull-flow-ids.md @@ -0,0 +1,5 @@ +--- +"@qawolf/cli": minor +--- + +Cache flow IDs when pulling an environment and include them in local JSON flow lists. Preserve previously cached IDs when the platform listing is unavailable. diff --git a/src/core/flowMeta.ts b/src/core/flowMeta.ts index 218adc0a8..d47f2bea7 100644 --- a/src/core/flowMeta.ts +++ b/src/core/flowMeta.ts @@ -75,3 +75,12 @@ export type PeekFlowMetaFn = (filePath: string) => Promise; export function extractFlowMeta(source: string): FlowCallMeta { return parseFlowCall(source); } + +const flowExtensions = [".flow.ts", ".flow.js"]; +const sourceExtensions = [".ts", ".js", ".mts", ".cts", ".mjs", ".cjs"]; + +export const isFlowFile = (name: string): boolean => + flowExtensions.some((extension) => name.endsWith(extension)); + +export const isSourceFile = (name: string): boolean => + sourceExtensions.some((extension) => name.endsWith(extension)); diff --git a/src/domains/flows/list.agent.test.ts b/src/domains/flows/list.agent.test.ts index 5ac4e561f..dd6c4f402 100644 --- a/src/domains/flows/list.agent.test.ts +++ b/src/domains/flows/list.agent.test.ts @@ -46,8 +46,8 @@ function makeDeps(overrides?: { target: metaByFile[file]?.target, }), ), - readCachedTags: mock(() => - Promise.resolve(new Map()), + readCachedFlows: mock(() => + Promise.resolve(new Map()), ), readEnvLabel: mock((dir: string) => Promise.resolve(dir), diff --git a/src/domains/flows/list.env.test.ts b/src/domains/flows/list.env.test.ts index 64a59ca04..52c99c9b2 100644 --- a/src/domains/flows/list.env.test.ts +++ b/src/domains/flows/list.env.test.ts @@ -9,6 +9,7 @@ import { makeNoopLogger } from "~/shell/logger.testUtils.js"; import { makeMemoryFs } from "~/shell/fs.testUtils.js"; import { type FlowsListDeps, flowsList } from "./list.js"; +import { cachedFlowsWithTags } from "./list.testUtils.js"; import { callsOf, makeFakeUI } from "~/shell/commandContext.testUtils.js"; afterEach(() => { @@ -44,8 +45,8 @@ function makeDeps( peekFlowMeta: mock(() => Promise.resolve({ name: undefined, target: undefined }), ), - readCachedTags: mock(() => - Promise.resolve(new Map(Object.entries(tagsByFile))), + readCachedFlows: mock(() => + Promise.resolve(cachedFlowsWithTags(tagsByFile)), ), readEnvLabel: mock((dir: string) => Promise.resolve(dir), @@ -189,10 +190,8 @@ describe("flowsList --env against a pulled environment", () => { const ctx = makeCtx(); const deps = { ...envDeps([stagingFlow, prodFlow]), - readCachedTags: mock(() => - Promise.resolve( - new Map([[stagingFlow, ["auth"]]]), - ), + readCachedFlows: mock(() => + Promise.resolve(cachedFlowsWithTags({ [stagingFlow]: ["auth"] })), ), }; diff --git a/src/domains/flows/list.json.test.ts b/src/domains/flows/list.json.test.ts index 9c3862b1b..efca93744 100644 --- a/src/domains/flows/list.json.test.ts +++ b/src/domains/flows/list.json.test.ts @@ -8,6 +8,7 @@ import { makeNoopLogger } from "~/shell/logger.testUtils.js"; import { makeMemoryFs } from "~/shell/fs.testUtils.js"; import { type FlowsListDeps, flowsList } from "./list.js"; +import { cachedFlowsWithTags } from "./list.testUtils.js"; import { makeFakeUI } from "~/shell/commandContext.testUtils.js"; const noopSignals = makeNoopSignals(); @@ -51,8 +52,8 @@ function makeDeps(overrides?: { target: metaByFile[file]?.target, }), ), - readCachedTags: mock(() => - Promise.resolve(new Map(Object.entries(cachedTags))), + readCachedFlows: mock(() => + Promise.resolve(cachedFlowsWithTags(cachedTags)), ), readEnvLabel: mock((dir: string) => Promise.resolve(dir), @@ -98,6 +99,22 @@ describe("flowsList json mode output", () => { expect(ui.outro).not.toHaveBeenCalled(); }); + it("includes a pulled flow’s cached ID in JSON", async () => { + const ui = makeFakeUI(); + const file = "/proj/.qawolf/staging/src/flows/a.flow.ts"; + const deps = makeDeps({ files: [file] }); + const cachedDeps = { + ...deps, + readCachedFlows: mock(() => + Promise.resolve(new Map([[file, { flowId: "flow-a", tags: [] }]])), + ), + }; + await flowsList(makeCtx(ui, "json"), undefined, cachedDeps); + expect(ui.json).toHaveBeenCalledWith([ + expect.objectContaining({ flowId: "flow-a", tags: [] }), + ]); + }); + it("falls back to basename for name when meta.name is undefined", async () => { const ui = makeFakeUI(); const deps = makeDeps({ diff --git a/src/domains/flows/list.selectors.test.ts b/src/domains/flows/list.selectors.test.ts index 5bd9aad11..d3fe13a10 100644 --- a/src/domains/flows/list.selectors.test.ts +++ b/src/domains/flows/list.selectors.test.ts @@ -9,6 +9,7 @@ import { makeNoopLogger } from "~/shell/logger.testUtils.js"; import { makeMemoryFs } from "~/shell/fs.testUtils.js"; import { type FlowsListDeps, flowsList } from "./list.js"; +import { cachedFlowsWithTags } from "./list.testUtils.js"; import { callsOf, makeFakeUI } from "~/shell/commandContext.testUtils.js"; afterEach(() => { @@ -45,8 +46,8 @@ function makeDeps( peekFlowMeta: mock(() => Promise.resolve({ name: undefined, target: undefined }), ), - readCachedTags: mock(() => - Promise.resolve(new Map(Object.entries(tagsByFile))), + readCachedFlows: mock(() => + Promise.resolve(cachedFlowsWithTags(tagsByFile)), ), readEnvLabel: mock((dir: string) => Promise.resolve(dir), diff --git a/src/domains/flows/list.test.ts b/src/domains/flows/list.test.ts index 8c0ad79de..f258f6cdc 100644 --- a/src/domains/flows/list.test.ts +++ b/src/domains/flows/list.test.ts @@ -49,8 +49,8 @@ function makeDeps(overrides?: { target: metaByFile[file]?.target, }), ), - readCachedTags: mock(() => - Promise.resolve(new Map()), + readCachedFlows: mock(() => + Promise.resolve(new Map()), ), readEnvLabel: mock((dir: string) => Promise.resolve(dir), diff --git a/src/domains/flows/list.testUtils.ts b/src/domains/flows/list.testUtils.ts new file mode 100644 index 000000000..5335c8f2d --- /dev/null +++ b/src/domains/flows/list.testUtils.ts @@ -0,0 +1,15 @@ +import type { CachedFlow } from "./readCachedFlows.js"; + +/** What `readCachedFlows` returns when the pulls recorded only these tags. */ +export const cachedFlowsWithTags = ( + tagsByFile: Record, +): Map => + new Map( + Object.entries(tagsByFile).map(([file, tags]) => [ + file, + { + tags, + flowId: undefined, + }, + ]), + ); diff --git a/src/domains/flows/list.ts b/src/domains/flows/list.ts index 804142b7f..be9b825b4 100644 --- a/src/domains/flows/list.ts +++ b/src/domains/flows/list.ts @@ -2,6 +2,7 @@ import path from "node:path"; import type { CommandContext, CommandResult } from "~/shell/commandContext.js"; import { flowsMessages, runnerMessages } from "~/core/messages/index.js"; +import type { CachedFlow } from "./readCachedFlows.js"; import type { BrowserName } from "~/core/types.js"; import { batchMap, flowBatchSize } from "~/core/batchMap.js"; @@ -23,10 +24,10 @@ export type FlowsListDeps = { cwd: string, ) => Promise; readonly peekFlowMeta: PeekFlowMetaFn; - /** Tags cached at pull time, keyed by absolute flow path. */ - readonly readCachedTags: ( + /** What each flow’s pull recorded, keyed by absolute flow path. */ + readonly readCachedFlows: ( files: readonly string[], - ) => Promise>; + ) => Promise>; /** Human label for a pulled env dir — its slug, name, or id. */ readonly readEnvLabel: (envDir: string) => Promise; /** Resolves an id, slug, or name to a pulled env, without the API. */ @@ -40,6 +41,7 @@ export type FlowsListDeps = { type FlowsListItem = { file: string; name: string; + flowId: string | undefined; // The pulled environment the flow came from. Undefined for project flows, // which belong to no environment. env: string | undefined; @@ -72,7 +74,11 @@ export async function flowsList( if (selection.kind === "unknown") return selection.result; files = selection.files; } - const cachedTags = await deps.readCachedTags(files); + const cached = await deps.readCachedFlows(files); + const cachedTags = new Map(); + for (const [file, flow] of cached) { + if (flow.tags !== undefined) cachedTags.set(file, flow.tags); + } const envLabels = await readEnvLabels(files, deps.readEnvLabel); const notCached = tagsNotCachedResult(selectors, cachedTags); @@ -87,6 +93,7 @@ export async function flowsList( all.push({ file: path.relative(deps.cwd, file), name: meta.name ?? flowBasename(file), + flowId: cached.get(file)?.flowId, env: envLabelFor(file, envLabels), tags: cachedTags.get(file), target: meta.target, diff --git a/src/domains/flows/listDefaults.ts b/src/domains/flows/listDefaults.ts index 3d0f5eeea..dd9226e8b 100644 --- a/src/domains/flows/listDefaults.ts +++ b/src/domains/flows/listDefaults.ts @@ -10,7 +10,7 @@ import { makePeekFlowMeta, } from "./expand.js"; import { flowsList } from "./list.js"; -import { readCachedTags as defaultReadCachedTags } from "./readCachedTags.js"; +import { readCachedFlows as defaultReadCachedFlows } from "./readCachedFlows.js"; import { readEnvLabel as defaultReadEnvLabel } from "./readEnvLabel.js"; export function handleFlowsList( @@ -27,7 +27,7 @@ export function handleFlowsList( expandPatterns: (patterns, cwd) => defaultExpandPatterns(patterns, cwd, undefined, fs), peekFlowMeta: makePeekFlowMeta(fs), - readCachedTags: (files) => defaultReadCachedTags(files, fs), + readCachedFlows: (files) => defaultReadCachedFlows(files, fs), readEnvLabel: (envDir) => defaultReadEnvLabel(envDir, fs), findPulledEnv: (ref) => defaultFindPulledEnv(ref, process.cwd(), fs), listPulledEnvDirs: () => defaultListPulledEnvDirs(process.cwd(), fs), diff --git a/src/domains/flows/manifestEntries.ts b/src/domains/flows/manifestEntries.ts new file mode 100644 index 000000000..02340e19c --- /dev/null +++ b/src/domains/flows/manifestEntries.ts @@ -0,0 +1,45 @@ +import { relative } from "node:path"; + +import { findPulledEnvDir, toPosix } from "~/core/repoRelativePath.js"; +import { makeDefaultFs, type Fs } from "~/shell/fs.js"; +import { readManifest } from "~/shell/manifest/io.js"; +import type { Manifest } from "~/shell/manifest/types.js"; + +type RecordedFlow = { + readonly entry: Manifest["flows"][number]; + /** Undefined when no tag fetch ever succeeded for the flow's environment. */ + readonly tagsFetchedAt: string | undefined; +}; + +/** Keyed by absolute path; flows outside a pulled manifest are absent. */ +export async function readManifestEntries( + files: readonly string[], + fs: Fs = makeDefaultFs(), +): Promise> { + // Group by env dir so a listing of many flows reads each manifest once + // rather than once per flow. + const filesByEnvDir = new Map(); + for (const file of files) { + const envDir = findPulledEnvDir(file); + if (envDir === undefined) continue; + const group = filesByEnvDir.get(envDir); + if (group) group.push(file); + else filesByEnvDir.set(envDir, [file]); + } + + const entries = new Map(); + for (const [envDir, envFiles] of filesByEnvDir) { + const manifest = await readManifest(envDir, fs); + if (typeof manifest === "string") continue; + // Compared posix on both sides: a manifest written on win32 by an older + // CLI may hold `\` paths. + const byPath = new Map(manifest.flows.map((f) => [toPosix(f.path), f])); + for (const file of envFiles) { + const entry = byPath.get(toPosix(relative(envDir, file))); + if (entry !== undefined) { + entries.set(file, { entry, tagsFetchedAt: manifest.tagsFetchedAt }); + } + } + } + return entries; +} diff --git a/src/domains/flows/pull/applyTeamStorageRewrite.ts b/src/domains/flows/pull/applyTeamStorageRewrite.ts index c210ef510..3c68321d8 100644 --- a/src/domains/flows/pull/applyTeamStorageRewrite.ts +++ b/src/domains/flows/pull/applyTeamStorageRewrite.ts @@ -1,39 +1,17 @@ -import { join, relative } from "node:path"; +import { relative } from "node:path"; +import { isFlowFile, isSourceFile } from "~/core/flowMeta.js"; import { makeDefaultFs } from "~/shell/fs.js"; import type { Fs } from "~/shell/fs.js"; +import { walkFiles } from "~/shell/walkFiles.js"; import { rewriteTeamStorage } from "./rewriteTeamStorage.js"; -const sourceExtensions = [".ts", ".js", ".mts", ".cts", ".mjs", ".cjs"]; -const flowExtensions = [".flow.ts", ".flow.js"]; - -function isSourceFile(name: string): boolean { - return sourceExtensions.some((ext) => name.endsWith(ext)); -} - -function isFlowFile(name: string): boolean { - return flowExtensions.some((ext) => name.endsWith(ext)); -} - -async function walk(dir: string, out: string[], fs: Fs): Promise { - const entries = await fs.readdirWithTypes(dir); - for (const e of entries) { - const abs = join(dir, e.name); - if (e.isDirectory()) { - await walk(abs, out, fs); - } else if (e.isFile() && isSourceFile(e.name)) { - out.push(abs); - } - } -} - export async function applyTeamStorageRewrite( rootDir: string, fs: Fs = makeDefaultFs(), ): Promise<{ flowsWithTeamStorageRefs: string[] }> { - const files: string[] = []; - await walk(rootDir, files, fs); + const files = await walkFiles(rootDir, isSourceFile, fs); const results = await Promise.all( files.map(async (file): Promise => { const source = await fs.readFile(file); diff --git a/src/domains/flows/pull/bundle.test.ts b/src/domains/flows/pull/bundle.test.ts index b5edeaf1b..c0fedf0ad 100644 --- a/src/domains/flows/pull/bundle.test.ts +++ b/src/domains/flows/pull/bundle.test.ts @@ -116,6 +116,7 @@ describe("buildManifest", () => { wrapperName: string | undefined; qawolfCommittedAt: string | undefined; tags: undefined; + flowIds: undefined; } => ({ envId: "env-x", bundleDir: workDir, @@ -125,6 +126,7 @@ describe("buildManifest", () => { wrapperName: undefined, qawolfCommittedAt: undefined, tags: undefined, + flowIds: undefined, }); it("walks .flow.ts and .flow.js files, ignores other extensions", async () => { diff --git a/src/domains/flows/pull/bundle.ts b/src/domains/flows/pull/bundle.ts index d894c775f..14ae35d7f 100644 --- a/src/domains/flows/pull/bundle.ts +++ b/src/domains/flows/pull/bundle.ts @@ -1,6 +1,8 @@ import { join, relative } from "node:path"; +import { isFlowFile } from "~/core/flowMeta.js"; import { toPosix } from "~/core/repoRelativePath.js"; +import { walkFiles } from "~/shell/walkFiles.js"; import { hashFile } from "~/shell/manifest/io.js"; import type { Manifest } from "~/shell/manifest/types.js"; @@ -38,6 +40,13 @@ export async function flattenSingleWrapper( return innerName; } +// Flow files under `root`, relative to it and sorted, so the manifest lists +// them in the same order on every pull. +async function flowPathsIn(root: string, fs: Fs): Promise { + const found = await walkFiles(root, isFlowFile, fs); + return found.map((path) => toPosix(relative(root, path))).sort(); +} + // GitHub's tarball archives wrap content in `--/`, where // the trailing 40 hex chars are the commit SHA. Defensive: returns undefined // when the wrapper name doesn't match — keeps manifest writes infallible. @@ -48,8 +57,6 @@ function extractQawolfCommitSha( return /-([0-9a-f]{40})$/i.exec(wrapperName)?.[1]; } -const flowExtensions = [".flow.ts", ".flow.js"]; - /** * Tags fetched for an env at pull time, keyed by repo-relative flow path. * Undefined when the fetch did not happen or failed. @@ -71,10 +78,11 @@ export async function buildManifest( wrapperName: string | undefined; qawolfCommittedAt: string | undefined; tags: FetchedTags | undefined; + flowIds: ReadonlyMap | undefined; }, fs: Fs = makeDefaultFs(), ): Promise { - const flowPaths = await walkForFlows(args.bundleDir, fs); + const flowPaths = await flowPathsIn(args.bundleDir, fs); const flows = await Promise.all( flowPaths.map(async (rel) => ({ // Stored posix so a manifest written on one platform resolves on @@ -85,6 +93,7 @@ export async function buildManifest( // Left unset when the fetch did not cover this file — unknown, not // untagged. tags: args.tags?.byPath.get(toPosix(rel)), + flowId: args.flowIds?.get(toPosix(rel)), })), ); @@ -102,32 +111,6 @@ export async function buildManifest( }; } -async function walkForFlows(root: string, fs: Fs): Promise { - const out: string[] = []; - await walk(root, root, out, fs); - return out.sort(); -} - -async function walk( - current: string, - root: string, - out: string[], - fs: Fs, -): Promise { - const entries = await fs.readdirWithTypes(current); - for (const e of entries) { - const abs = join(current, e.name); - if (e.isDirectory()) { - await walk(abs, root, out, fs); - } else if ( - e.isFile() && - flowExtensions.some((ext) => e.name.endsWith(ext)) - ) { - out.push(relative(root, abs)); - } - } -} - // Samples the mtime of any flow file in the bundle. GitHub-archive bundles // share one mtime across all entries (preserved by extract.ts). Returns // undefined when the bundle has no flow files. Sample BEFORE any local @@ -136,7 +119,7 @@ export async function sampleQawolfCommittedAt( bundleDir: string, fs: Fs = makeDefaultFs(), ): Promise { - const flowPaths = await walkForFlows(bundleDir, fs); + const flowPaths = await flowPathsIn(bundleDir, fs); const sample = flowPaths[0]; if (!sample) return undefined; return (await fs.stat(join(bundleDir, sample))).mtime.toISOString(); diff --git a/src/domains/flows/pull/bundleFlowIds.test.ts b/src/domains/flows/pull/bundleFlowIds.test.ts new file mode 100644 index 000000000..44deff232 --- /dev/null +++ b/src/domains/flows/pull/bundleFlowIds.test.ts @@ -0,0 +1,103 @@ +import { mkdtemp, mkdir, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { dirname, join } from "node:path"; +import { afterEach, beforeEach, describe, expect, it } from "bun:test"; + +import { readManifest } from "~/shell/manifest/io.js"; +import { buildManifest } from "./bundle.js"; +import { buildBundle } from "./pull.fixtures.js"; +import { stageBundle } from "./stage.js"; + +let workDir = ""; + +beforeEach(async () => { + workDir = await mkdtemp(join(tmpdir(), "qawolf-bundle-flow-ids-")); +}); + +afterEach(async () => { + await rm(workDir, { recursive: true, force: true }); +}); + +const flow = "src/flows/a.flow.ts"; + +const baseArgs = () => ({ + flowIds: undefined, + envId: "env-x", + envSlug: undefined, + envName: undefined, + bundleDir: workDir, + cliFlowsVersion: "0.4.0", + now: new Date("2026-05-10T12:00:00.000Z"), + envVarsFetchedAt: undefined, + wrapperName: undefined, + qawolfCommittedAt: undefined, + tags: undefined, +}); + +async function stage(names: string[]): Promise { + for (const name of names) { + const p = join(workDir, name); + await mkdir(dirname(p), { recursive: true }); + await writeFile(p, "// flow", "utf8"); + } +} + +describe("buildManifest flow ids", () => { + it("records the id the listing gave each flow", async () => { + await stage([flow]); + + const manifest = await buildManifest({ + ...baseArgs(), + flowIds: new Map([[flow, "flow-a"]]), + }); + + expect(manifest.flows.find((f) => f.path === flow)?.flowId).toBe("flow-a"); + }); + + it("leaves a flow the listing did not cover without an id", async () => { + await stage([flow]); + + const manifest = await buildManifest({ ...baseArgs(), flowIds: new Map() }); + + expect(manifest.flows.find((f) => f.path === flow)?.flowId).toBeUndefined(); + }); +}); + +describe("stageBundle flow ids", () => { + const stageArgs = (destDir: string, archive: string) => ({ + flowIds: undefined, + tmpArchive: archive, + destAbs: destDir, + assetsAbs: join(destDir, "..", "assets"), + envId: "env-abc", + envSlug: undefined, + envName: undefined, + cliFlowsVersion: "0.4.0", + now: new Date("2026-05-10T12:00:00.000Z"), + envVars: {}, + envVarsFetchedAt: new Date("2026-05-10T12:00:00.000Z"), + tags: undefined, + }); + + // A failed listing fetch must not erase ids a previous pull stored — the + // same guarantee tags have. + it("carries ids forward when the listing could not be fetched", async () => { + const destDir = join(workDir, "env"); + const first = join(workDir, "first.tar.gz"); + await buildBundle(first, { flows: [{ name: flow, data: "// a" }] }); + await stageBundle({ + ...stageArgs(destDir, first), + flowIds: new Map([[flow, "flow-a"]]), + }); + + const second = join(workDir, "second.tar.gz"); + await buildBundle(second, { + flows: [{ name: flow, data: "// a, edited" }], + }); + await stageBundle({ ...stageArgs(destDir, second), flowIds: undefined }); + + const manifest = await readManifest(destDir); + if (typeof manifest === "string") throw new Error(manifest); + expect(manifest.flows.find((f) => f.path === flow)?.flowId).toBe("flow-a"); + }); +}); diff --git a/src/domains/flows/pull/bundleOrder.test.ts b/src/domains/flows/pull/bundleOrder.test.ts new file mode 100644 index 000000000..c22f49f1b --- /dev/null +++ b/src/domains/flows/pull/bundleOrder.test.ts @@ -0,0 +1,38 @@ +import { expect, it } from "bun:test"; + +import { makeMemoryFs } from "~/shell/fs.testUtils.js"; + +import { buildManifest } from "./bundle.js"; + +it("sorts manifest paths after normalizing separators", async () => { + const fs = makeMemoryFs(); + await fs.mkdir("/bundle/a", { recursive: true }); + await fs.writeFile("/bundle/a/z.flow.ts", "export default () => 1;"); + await fs.writeFile("/bundle/a[.flow.ts", "export default () => 2;"); + const manifest = await buildManifest( + { + bundleDir: "/bundle", + envId: "env", + cliFlowsVersion: "1.0.0", + now: new Date(0), + envVarsFetchedAt: undefined, + wrapperName: undefined, + qawolfCommittedAt: undefined, + tags: undefined, + flowIds: undefined, + }, + { + ...fs, + readdirWithTypes: async () => + ["a\\z.flow.ts", "a[.flow.ts"].map((name) => ({ + name, + isFile: () => true, + isDirectory: () => false, + })), + }, + ); + expect(manifest.flows.map(({ path }) => path)).toEqual([ + "a/z.flow.ts", + "a[.flow.ts", + ]); +}); diff --git a/src/domains/flows/pull/bundleTags.test.ts b/src/domains/flows/pull/bundleTags.test.ts index 6290b5769..516f42409 100644 --- a/src/domains/flows/pull/bundleTags.test.ts +++ b/src/domains/flows/pull/bundleTags.test.ts @@ -27,6 +27,7 @@ async function stage(names: string[]): Promise { } const baseArgs = () => ({ + flowIds: undefined, envId: "env-x", envSlug: undefined, envName: undefined, @@ -104,6 +105,7 @@ describe("stageBundle tag preservation", () => { await buildBundle(first, { flows }); await stageBundle({ + flowIds: undefined, tmpArchive: first, destAbs: dest, assetsAbs: join(workDir, "assets"), @@ -124,6 +126,7 @@ describe("stageBundle tag preservation", () => { const second = join(workDir, "second.tar.gz"); await buildBundle(second, { flows }); await stageBundle({ + flowIds: undefined, tmpArchive: second, destAbs: dest, assetsAbs: join(workDir, "assets"), @@ -153,6 +156,7 @@ describe("stageBundle tag preservation", () => { await buildBundle(first, { flows }); await stageBundle({ + flowIds: undefined, tmpArchive: first, destAbs: dest, assetsAbs: join(workDir, "assets"), @@ -183,6 +187,7 @@ describe("stageBundle tag preservation", () => { const second = join(workDir, "second.tar.gz"); await buildBundle(second, { flows }); await stageBundle({ + flowIds: undefined, tmpArchive: second, destAbs: dest, assetsAbs: join(workDir, "assets"), diff --git a/src/domains/flows/pull/fetchPhase.ts b/src/domains/flows/pull/fetchPhase.ts index f95dbe1fb..d4905cd58 100644 --- a/src/domains/flows/pull/fetchPhase.ts +++ b/src/domains/flows/pull/fetchPhase.ts @@ -15,14 +15,18 @@ type FetchedBundle = { // Undefined when the tag fetch did not succeed. Tags enrich a pull; they are // never a precondition for one, so a failure here leaves the pull intact. tags: FetchedTags | undefined; + // From the same listing as the tags, and missing whenever they are. + flowIds: ReadonlyMap | undefined; }; +type FetchedListing = { tags: FetchedTags; flowIds: Map }; + // Drafts are included so the cache covers every flow the bundle can contain; // a flow missing from the response keeps unknown tags rather than empty ones. -async function fetchTags( +async function fetchListing( ctx: AuthCommandContext, envId: string, -): Promise { +): Promise { try { const result = await ctx.platformClient.callPublicApi( publicContractsV1.flow.list, @@ -30,8 +34,12 @@ async function fetchTags( ); if (!result.ok) return undefined; return { - fetchedAt: new Date(), - byPath: new Map(result.value.flows.map((f) => [f.path, [...f.tags]])), + tags: { + fetchedAt: new Date(), + byPath: new Map(result.value.flows.map((f) => [f.path, [...f.tags]])), + }, + // Kept so a pulled flow can be named by id offline, as --remote does. + flowIds: new Map(result.value.flows.map((f) => [f.path, f.flowId])), }; } catch { return undefined; @@ -48,7 +56,7 @@ export async function fetchBundleAndEnvVars( let envVars: Record | undefined; let envVarsFetchedAt: Date | undefined; let teamId: string | undefined; - let tags: FetchedTags | undefined; + let listing: FetchedListing | undefined; await ctx.ui.withProgress( [ @@ -75,7 +83,7 @@ export async function fetchBundleAndEnvVars( { message: flowsMessages.pull.fetchingTags, task: async () => { - tags = await fetchTags(ctx, envId); + listing = await fetchListing(ctx, envId); }, }, ], @@ -100,6 +108,7 @@ export async function fetchBundleAndEnvVars( envVars, envVarsFetchedAt, teamId, - tags, + tags: listing?.tags, + flowIds: listing?.flowIds, }; } diff --git a/src/domains/flows/pull/handler.ts b/src/domains/flows/pull/handler.ts index 0ab2a93db..6e5459135 100644 --- a/src/domains/flows/pull/handler.ts +++ b/src/domains/flows/pull/handler.ts @@ -93,6 +93,7 @@ export async function handleFlowsPull( envVars: fetched.envVars, envVarsFetchedAt: fetched.envVarsFetchedAt, tags: fetched.tags, + flowIds: fetched.flowIds, }, resolvedDeps.fs, ), diff --git a/src/domains/flows/pull/previousPull.ts b/src/domains/flows/pull/previousPull.ts new file mode 100644 index 000000000..57ea544a3 --- /dev/null +++ b/src/domains/flows/pull/previousPull.ts @@ -0,0 +1,37 @@ +import { toPosix } from "~/core/repoRelativePath.js"; +import type { Fs } from "~/shell/fs.js"; +import { readManifest } from "~/shell/manifest/io.js"; + +import type { FetchedTags } from "./bundle.js"; + +// A failed listing fetch must not erase cached IDs or break offline tag queries. +export async function carriedFromPreviousPull( + envDir: string, + fs: Fs, +): Promise<{ + tags: FetchedTags | undefined; + flowIds: Map | undefined; +}> { + const previous = await readManifest(envDir, fs); + if (typeof previous === "string") { + return { tags: undefined, flowIds: undefined }; + } + + const tagsByPath = new Map(); + const flowIds = new Map(); + for (const flow of previous.flows) { + // A manifest written by an older CLI on win32 may hold `\` paths; the new + // manifest looks entries up by posix path, so normalize or the carried + // values never match and vanish silently. + const path = toPosix(flow.path); + if (flow.tags !== undefined) tagsByPath.set(path, [...flow.tags]); + if (flow.flowId !== undefined) flowIds.set(path, flow.flowId); + } + return { + tags: + previous.tagsFetchedAt === undefined + ? undefined + : { fetchedAt: new Date(previous.tagsFetchedAt), byPath: tagsByPath }, + flowIds, + }; +} diff --git a/src/domains/flows/pull/pullSafety.test.ts b/src/domains/flows/pull/pullSafety.test.ts index 4066521cb..a5f712d59 100644 --- a/src/domains/flows/pull/pullSafety.test.ts +++ b/src/domains/flows/pull/pullSafety.test.ts @@ -8,6 +8,7 @@ import type { Manifest } from "~/shell/manifest/types.js"; import { buildBundle } from "./pull.fixtures.js"; import { checkSafety } from "./pull.js"; import { stageBundle } from "./stage.js"; +import { makeManifestFlow } from "~/shell/manifest/manifest.testUtils.js"; let workDir = ""; let bundleArchive = ""; @@ -38,11 +39,10 @@ describe("safety + staging integration", () => { qawolfCommittedAt: undefined, tagsFetchedAt: undefined, flows: [ - { + makeManifestFlow({ path: "a.flow.ts", contentHash: await hashFile(join(destDir, "a.flow.ts")), - tags: undefined, - }, + }), ], }; await writeManifest(destDir, manifest); @@ -61,6 +61,7 @@ describe("safety + staging integration", () => { expect(safety).toBe("proceed"); await stageBundle({ + flowIds: undefined, tmpArchive: bundleArchive, destAbs: destDir, assetsAbs: join(destDir, "..", "assets"), diff --git a/src/domains/flows/pull/safety.test.ts b/src/domains/flows/pull/safety.test.ts index 9c845b7d4..b2807cb90 100644 --- a/src/domains/flows/pull/safety.test.ts +++ b/src/domains/flows/pull/safety.test.ts @@ -8,6 +8,7 @@ import { detectLocalModifications, promptOverwriteIfModified, } from "./safety.js"; +import { makeManifestFlow } from "~/shell/manifest/manifest.testUtils.js"; let workDir = ""; @@ -19,9 +20,7 @@ afterEach(async () => { await rm(workDir, { recursive: true, force: true }); }); -const baseManifest = ( - flows: { path: string; contentHash: string; tags: undefined }[], -): Manifest => ({ +const baseManifest = (flows: Manifest["flows"]): Manifest => ({ envId: "env-abc", envSlug: undefined, envName: undefined, @@ -34,6 +33,11 @@ const baseManifest = ( flows, }); +const flowEntry = ( + path: string, + contentHash: string, +): Manifest["flows"][number] => makeManifestFlow({ path, contentHash }); + // sha256("hello") = 2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e73043362938b9824 const helloHash = "2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e73043362938b9824"; @@ -41,26 +45,20 @@ const helloHash = describe("detectLocalModifications", () => { it("returns [] when every file matches its manifest hash", async () => { await writeFile(join(workDir, "a.flow.ts"), "hello", "utf8"); - const manifest = baseManifest([ - { path: "a.flow.ts", contentHash: helloHash, tags: undefined }, - ]); + const manifest = baseManifest([flowEntry("a.flow.ts", helloHash)]); expect(await detectLocalModifications(workDir, manifest)).toEqual([]); }); it("flags a file whose hash differs as 'modified'", async () => { await writeFile(join(workDir, "a.flow.ts"), "edited", "utf8"); - const manifest = baseManifest([ - { path: "a.flow.ts", contentHash: helloHash, tags: undefined }, - ]); + const manifest = baseManifest([flowEntry("a.flow.ts", helloHash)]); expect(await detectLocalModifications(workDir, manifest)).toEqual([ { path: "a.flow.ts", reason: "modified" }, ]); }); it("flags a missing file as 'missing-from-disk'", async () => { - const manifest = baseManifest([ - { path: "gone.flow.ts", contentHash: helloHash, tags: undefined }, - ]); + const manifest = baseManifest([flowEntry("gone.flow.ts", helloHash)]); expect(await detectLocalModifications(workDir, manifest)).toEqual([ { path: "gone.flow.ts", reason: "missing-from-disk" }, ]); @@ -73,9 +71,7 @@ describe("detectLocalModifications", () => { }); it("rejects a manifest containing an absolute path entry", async () => { - const manifest = baseManifest([ - { path: "/etc/passwd", contentHash: helloHash, tags: undefined }, - ]); + const manifest = baseManifest([flowEntry("/etc/passwd", helloHash)]); let caught: unknown; try { await detectLocalModifications(workDir, manifest); @@ -87,9 +83,7 @@ describe("detectLocalModifications", () => { }); it("rejects a manifest entry that escapes the env directory", async () => { - const manifest = baseManifest([ - { path: "../escape.flow.ts", contentHash: helloHash, tags: undefined }, - ]); + const manifest = baseManifest([flowEntry("../escape.flow.ts", helloHash)]); let caught: unknown; try { await detectLocalModifications(workDir, manifest); @@ -120,9 +114,7 @@ describe("promptOverwriteIfModified", () => { it("proceeds without prompt when there are no modifications", async () => { await writeFile(join(workDir, "a.flow.ts"), "hello", "utf8"); - const manifest = baseManifest([ - { path: "a.flow.ts", contentHash: helloHash, tags: undefined }, - ]); + const manifest = baseManifest([flowEntry("a.flow.ts", helloHash)]); const confirm = makeFakeConfirm(false); const log = makeLog(); @@ -139,9 +131,7 @@ describe("promptOverwriteIfModified", () => { }); it("proceeds without prompt when only missing-from-disk entries exist", async () => { - const manifest = baseManifest([ - { path: "gone.flow.ts", contentHash: helloHash, tags: undefined }, - ]); + const manifest = baseManifest([flowEntry("gone.flow.ts", helloHash)]); const confirm = makeFakeConfirm(false); const log = makeLog(); @@ -159,9 +149,7 @@ describe("promptOverwriteIfModified", () => { it("proceeds without prompt and logs a notice when yes is true and mods exist", async () => { await writeFile(join(workDir, "a.flow.ts"), "edited", "utf8"); - const manifest = baseManifest([ - { path: "a.flow.ts", contentHash: helloHash, tags: undefined }, - ]); + const manifest = baseManifest([flowEntry("a.flow.ts", helloHash)]); const confirm = makeFakeConfirm(true); const log = makeLog(); @@ -181,9 +169,7 @@ describe("promptOverwriteIfModified", () => { it("prompts via confirm and proceeds when the user accepts", async () => { await writeFile(join(workDir, "a.flow.ts"), "edited", "utf8"); - const manifest = baseManifest([ - { path: "a.flow.ts", contentHash: helloHash, tags: undefined }, - ]); + const manifest = baseManifest([flowEntry("a.flow.ts", helloHash)]); const confirm = makeFakeConfirm(true); const log = makeLog(); @@ -202,9 +188,7 @@ describe("promptOverwriteIfModified", () => { it("aborts when the user declines the prompt", async () => { await writeFile(join(workDir, "a.flow.ts"), "edited", "utf8"); - const manifest = baseManifest([ - { path: "a.flow.ts", contentHash: helloHash, tags: undefined }, - ]); + const manifest = baseManifest([flowEntry("a.flow.ts", helloHash)]); const confirm = makeFakeConfirm(false); const log = makeLog(); diff --git a/src/domains/flows/pull/stage.test.ts b/src/domains/flows/pull/stage.test.ts index a354e65b4..ad101152d 100644 --- a/src/domains/flows/pull/stage.test.ts +++ b/src/domains/flows/pull/stage.test.ts @@ -33,6 +33,7 @@ describe("stageBundle", () => { }); const result = await stageBundle({ + flowIds: undefined, tmpArchive: bundleArchive, destAbs: destDir, assetsAbs: join(workDir, "assets"), @@ -74,6 +75,7 @@ describe("stageBundle", () => { }); const result = await stageBundle({ + flowIds: undefined, tmpArchive: bundleArchive, destAbs: destDir, assetsAbs: join(workDir, "assets"), @@ -102,6 +104,7 @@ describe("stageBundle", () => { }); await stageBundle({ + flowIds: undefined, tmpArchive: bundleArchive, destAbs: destDir, assetsAbs: join(workDir, "assets"), @@ -133,6 +136,7 @@ describe("stageBundle", () => { const assetsAbs = join(workDir, "assets"); const result = await stageBundle({ + flowIds: undefined, tmpArchive: bundleArchive, destAbs: destDir, assetsAbs, @@ -187,6 +191,7 @@ describe("stageBundle", () => { const assetsDir = join(workDir, "assets"); const stageResult = await stageBundle({ + flowIds: undefined, tmpArchive: bundleArchive, destAbs: destDir, assetsAbs: assetsDir, diff --git a/src/domains/flows/pull/stage.ts b/src/domains/flows/pull/stage.ts index b617fdd97..53e40734b 100644 --- a/src/domains/flows/pull/stage.ts +++ b/src/domains/flows/pull/stage.ts @@ -1,12 +1,12 @@ -import { toPosix } from "~/core/repoRelativePath.js"; import { makeDefaultFs, type Fs } from "~/shell/fs.js"; -import { readManifest, writeManifest } from "~/shell/manifest/io.js"; +import { writeManifest } from "~/shell/manifest/io.js"; import { buildManifest, flattenSingleWrapper, sampleQawolfCommittedAt, type FetchedTags, } from "./bundle.js"; +import { carriedFromPreviousPull } from "./previousPull.js"; import { applyTeamStorageRewrite } from "./applyTeamStorageRewrite.js"; import { writeEnvFile } from "./envVars.js"; import { extractTarGz } from "./extract.js"; @@ -28,6 +28,7 @@ type StageBundleArgs = { envVars: Record; envVarsFetchedAt: Date; tags: FetchedTags | undefined; + flowIds: ReadonlyMap | undefined; }; type StageBundleResult = { @@ -67,10 +68,16 @@ export async function stageBundle( TEAM_STORAGE_DIR: args.assetsAbs, }; await writeEnvFile(tmpDir, effectiveEnvVars, fs); + // A failed listing fetch must not erase cached tags or IDs. + const carried = + args.tags === undefined || args.flowIds === undefined + ? await carriedFromPreviousPull(args.destAbs, fs) + : undefined; const manifest = await buildManifest( { envId: args.envId, - tags: args.tags ?? (await carriedTags(args.destAbs, fs)), + tags: args.tags ?? carried?.tags, + flowIds: args.flowIds ?? carried?.flowIds, envSlug: args.envSlug, envName: args.envName, bundleDir: tmpDir, @@ -109,28 +116,3 @@ export async function stageBundle( throw err; } } - -/** - * Tags kept from the previous pull of this environment. - * - * A pull rebuilds the manifest from the bundle, so a failed tag fetch would - * otherwise erase tags that were cached successfully earlier. Stale tags are - * reported as stale; losing them silently would break every offline query. - */ -async function carriedTags( - envDir: string, - fs: Fs, -): Promise { - const previous = await readManifest(envDir, fs); - if (typeof previous === "string") return undefined; - if (previous.tagsFetchedAt === undefined) return undefined; - - const byPath = new Map(); - for (const flow of previous.flows) { - // A manifest written by an older CLI on win32 may hold `\` paths; the new - // manifest looks entries up by posix path, so normalize or the carried - // tags never match and vanish silently. - if (flow.tags !== undefined) byPath.set(toPosix(flow.path), [...flow.tags]); - } - return { fetchedAt: new Date(previous.tagsFetchedAt), byPath }; -} diff --git a/src/domains/flows/readCachedFlows.test.ts b/src/domains/flows/readCachedFlows.test.ts new file mode 100644 index 000000000..a1b59f7a7 --- /dev/null +++ b/src/domains/flows/readCachedFlows.test.ts @@ -0,0 +1,105 @@ +import { describe, expect, it } from "bun:test"; + +import type { Fs } from "~/shell/fs.js"; +import { makeMemoryFs } from "~/shell/fs.testUtils.js"; +import { manifestFilename } from "~/shell/manifest/io.js"; +import { + makeManifest, + makeManifestFlow, +} from "~/shell/manifest/manifest.testUtils.js"; +import type { Manifest } from "~/shell/manifest/types.js"; + +import { readCachedFlows } from "./readCachedFlows.js"; + +const envDir = "/proj/.qawolf/staging"; +const flowA = `${envDir}/src/flows/a.flow.ts`; +const flowB = `${envDir}/src/flows/b.flow.ts`; +const fetchedAt = "2026-05-10T12:00:00.000Z"; + +async function fsWith(manifest: Manifest): Promise { + const fs = makeMemoryFs(); + await fs.mkdir(envDir, { recursive: true }); + await fs.writeFile(`${envDir}/${manifestFilename}`, JSON.stringify(manifest)); + return fs; +} + +describe("readCachedFlows", () => { + it("returns what the pull recorded for each flow", async () => { + const fs = await fsWith( + makeManifest({ + tagsFetchedAt: fetchedAt, + flows: [ + makeManifestFlow({ + path: "src/flows/a.flow.ts", + tags: ["smoke"], + flowId: "flow-a", + }), + makeManifestFlow({ + path: "src/flows/b.flow.ts", + tags: [], + }), + ], + }), + ); + + const result = await readCachedFlows([flowA, flowB], fs); + + expect(result.get(flowA)).toEqual({ + tags: ["smoke"], + flowId: "flow-a", + }); + expect(result.get(flowB)).toEqual({ + tags: [], + flowId: undefined, + }); + }); + + // Older manifests leave IDs unknown. + it("leaves what the pull did not record undefined", async () => { + const fs = await fsWith(makeManifest({ flows: [makeManifestFlow()] })); + + const result = await readCachedFlows([flowA], fs); + + expect(result.get(flowA)).toEqual({ + tags: undefined, + flowId: undefined, + }); + }); + + it("treats tags as unknown until a fetch has succeeded", async () => { + const fs = await fsWith( + makeManifest({ flows: [makeManifestFlow({ tags: ["smoke"] })] }), + ); + + const result = await readCachedFlows([flowA], fs); + + expect(result.get(flowA)?.tags).toBeUndefined(); + }); + + it("omits a flow that is not in the manifest", async () => { + const fs = await fsWith(makeManifest({ flows: [makeManifestFlow()] })); + + const result = await readCachedFlows([flowA, flowB], fs); + + expect(result.has(flowB)).toBe(false); + }); + + it("returns nothing for flows outside a pulled env tree", async () => { + const result = await readCachedFlows( + ["/proj/src/flows/a.flow.ts"], + makeMemoryFs(), + ); + + expect(result.size).toBe(0); + }); + + it("returns nothing when the manifest is unreadable", async () => { + const fs = makeMemoryFs(); + await fs.mkdir(envDir, { recursive: true }); + await fs.writeFile(`${envDir}/${manifestFilename}`, "not json"); + + const result = await readCachedFlows([flowA], fs); + + expect(result.size).toBe(0); + }); +}); diff --git a/src/domains/flows/readCachedFlows.ts b/src/domains/flows/readCachedFlows.ts new file mode 100644 index 000000000..1ae969144 --- /dev/null +++ b/src/domains/flows/readCachedFlows.ts @@ -0,0 +1,32 @@ +import { makeDefaultFs, type Fs } from "~/shell/fs.js"; + +import { readManifestEntries } from "./manifestEntries.js"; + +/** What a pull recorded about one flow; each field is undefined when unknown. */ +export type CachedFlow = { + /** Unknown, not untagged, until a tag fetch has succeeded for the env. */ + readonly tags: readonly string[] | undefined; + /** Unknown when the flow was pulled before ids were kept. */ + readonly flowId: string | undefined; +}; + +/** + * What the pull recorded for each flow, keyed by absolute path, reading each + * environment's manifest once. Flows that were never pulled are absent. + */ +export async function readCachedFlows( + files: readonly string[], + fs: Fs = makeDefaultFs(), +): Promise> { + const flows = new Map(); + for (const [file, { entry, tagsFetchedAt }] of await readManifestEntries( + files, + fs, + )) { + flows.set(file, { + tags: tagsFetchedAt === undefined ? undefined : entry.tags, + flowId: entry.flowId, + }); + } + return flows; +} diff --git a/src/domains/flows/readCachedTags.test.ts b/src/domains/flows/readCachedTags.test.ts index 6b26ba41e..d3a8a7ad1 100644 --- a/src/domains/flows/readCachedTags.test.ts +++ b/src/domains/flows/readCachedTags.test.ts @@ -6,6 +6,7 @@ import { manifestFilename } from "~/shell/manifest/io.js"; import type { Manifest } from "~/shell/manifest/types.js"; import { readCachedTags } from "./readCachedTags.js"; +import { makeManifestFlow } from "~/shell/manifest/manifest.testUtils.js"; const envDir = "/proj/.qawolf/staging"; const flowA = `${envDir}/src/flows/a.flow.ts`; @@ -43,8 +44,16 @@ describe("readCachedTags", () => { const fs = await fsWith( manifest({ flows: [ - { path: "src/flows/a.flow.ts", contentHash: "h1", tags: ["auth"] }, - { path: "src/flows/b.flow.ts", contentHash: "h2", tags: [] }, + makeManifestFlow({ + path: "src/flows/a.flow.ts", + contentHash: "h1", + tags: ["auth"], + }), + makeManifestFlow({ + path: "src/flows/b.flow.ts", + contentHash: "h2", + tags: [], + }), ], }), ); @@ -62,7 +71,7 @@ describe("readCachedTags", () => { manifest({ tagsFetchedAt: undefined, flows: [ - { path: "src/flows/a.flow.ts", contentHash: "h1", tags: undefined }, + makeManifestFlow({ path: "src/flows/a.flow.ts", contentHash: "h1" }), ], }), ); @@ -78,8 +87,12 @@ describe("readCachedTags", () => { const fs = await fsWith( manifest({ flows: [ - { path: "src/flows/a.flow.ts", contentHash: "h1", tags: ["auth"] }, - { path: "src/flows/b.flow.ts", contentHash: "h2", tags: undefined }, + makeManifestFlow({ + path: "src/flows/a.flow.ts", + contentHash: "h1", + tags: ["auth"], + }), + makeManifestFlow({ path: "src/flows/b.flow.ts", contentHash: "h2" }), ], }), ); @@ -94,8 +107,16 @@ describe("readCachedTags", () => { const fs = await fsWith( manifest({ flows: [ - { path: "src/flows/a.flow.ts", contentHash: "h1", tags: ["auth"] }, - { path: "src/flows/b.flow.ts", contentHash: "h2", tags: ["smoke"] }, + makeManifestFlow({ + path: "src/flows/a.flow.ts", + contentHash: "h1", + tags: ["auth"], + }), + makeManifestFlow({ + path: "src/flows/b.flow.ts", + contentHash: "h2", + tags: ["smoke"], + }), ], }), ); @@ -120,11 +141,11 @@ describe("readCachedTags", () => { const fs = await fsWith( manifest({ flows: [ - { + makeManifestFlow({ path: "src\\flows\\a.flow.ts", contentHash: "h1", tags: ["auth"], - }, + }), ], }), ); diff --git a/src/domains/flows/readCachedTags.ts b/src/domains/flows/readCachedTags.ts index fa4376a3d..0d8873d82 100644 --- a/src/domains/flows/readCachedTags.ts +++ b/src/domains/flows/readCachedTags.ts @@ -1,10 +1,6 @@ -import { relative } from "node:path"; - -import { toPosix } from "~/core/repoRelativePath.js"; - -import { findPulledEnvDir } from "~/core/repoRelativePath.js"; import { makeDefaultFs, type Fs } from "~/shell/fs.js"; -import { readManifest } from "~/shell/manifest/io.js"; + +import { readManifestEntries } from "./manifestEntries.js"; /** * Reads the tags cached at pull time for each flow, keyed by absolute path. @@ -17,31 +13,13 @@ export async function readCachedTags( files: readonly string[], fs: Fs = makeDefaultFs(), ): Promise> { - // Group by env dir so a listing of many flows reads each manifest once - // rather than once per flow. - const filesByEnvDir = new Map(); - for (const file of files) { - const envDir = findPulledEnvDir(file); - if (envDir === undefined) continue; - const group = filesByEnvDir.get(envDir); - if (group) group.push(file); - else filesByEnvDir.set(envDir, [file]); - } - const tagsByFile = new Map(); - for (const [envDir, envFiles] of filesByEnvDir) { - const manifest = await readManifest(envDir, fs); - if (typeof manifest === "string") continue; - // No fetch ever happened for this env, so every entry is unknown. - if (manifest.tagsFetchedAt === undefined) continue; - - const entryByPath = new Map( - manifest.flows.map((f) => [toPosix(f.path), f]), - ); - for (const file of envFiles) { - const tags = entryByPath.get(toPosix(relative(envDir, file)))?.tags; - if (tags !== undefined) tagsByFile.set(file, tags); - } + for (const [file, { entry, tagsFetchedAt }] of await readManifestEntries( + files, + fs, + )) { + if (tagsFetchedAt === undefined || entry.tags === undefined) continue; + tagsByFile.set(file, entry.tags); } return tagsByFile; } diff --git a/src/domains/flows/resolveTags.test.ts b/src/domains/flows/resolveTags.test.ts index e53aaaaab..4ef217cb7 100644 --- a/src/domains/flows/resolveTags.test.ts +++ b/src/domains/flows/resolveTags.test.ts @@ -13,6 +13,7 @@ import { } from "~/shell/platform/createPlatformClient.testUtils.js"; import { resolveTags } from "./resolveTags.js"; +import { makeManifestFlow } from "~/shell/manifest/manifest.testUtils.js"; afterEach(() => { mock.restore(); @@ -52,7 +53,11 @@ async function fsWithManifest(over: Partial): Promise { qawolfCommittedAt: undefined, tagsFetchedAt: "2026-05-01T12:00:00.000Z", flows: [ - { path: "src/flows/a.flow.ts", contentHash: "h1", tags: ["cached-tag"] }, + makeManifestFlow({ + path: "src/flows/a.flow.ts", + contentHash: "h1", + tags: ["cached-tag"], + }), ], ...over, }; @@ -146,7 +151,7 @@ describe("resolveTags fallback", () => { envDir, await fsWithManifest({ flows: [ - { path: "src/flows/a.flow.ts", contentHash: "h1", tags: undefined }, + makeManifestFlow({ path: "src/flows/a.flow.ts", contentHash: "h1" }), ], }), ); @@ -164,11 +169,11 @@ describe("resolveTags fallback", () => { envDir, await fsWithManifest({ flows: [ - { + makeManifestFlow({ path: "src\\flows\\a.flow.ts", contentHash: "h1", tags: ["cached-tag"], - }, + }), ], }), ); diff --git a/src/domains/runner/run.manifestStamp.test.ts b/src/domains/runner/run.manifestStamp.test.ts index 1ac2ea43c..cc56ed57e 100644 --- a/src/domains/runner/run.manifestStamp.test.ts +++ b/src/domains/runner/run.manifestStamp.test.ts @@ -9,6 +9,7 @@ import type { Manifest } from "~/shell/manifest/types.js"; import { defaultFlags, makeDeps, passResult } from "./run.fixtures.js"; import { dispatchFlow } from "./dispatchFlow.js"; +import { makeManifestFlow } from "~/shell/manifest/manifest.testUtils.js"; let workDir = ""; let envDir = ""; @@ -35,7 +36,7 @@ const sampleManifest = (): Manifest => ({ tagsFetchedAt: undefined, envVarsFetchedAt: undefined, flows: [ - { path: "login.flow.ts", contentHash: "hash-login", tags: undefined }, + makeManifestFlow({ path: "login.flow.ts", contentHash: "hash-login" }), ], }); diff --git a/src/shell/manifest/io.test.ts b/src/shell/manifest/io.test.ts index aa8701cc0..61a050c6c 100644 --- a/src/shell/manifest/io.test.ts +++ b/src/shell/manifest/io.test.ts @@ -12,6 +12,7 @@ import { writeManifest, } from "./io.js"; import type { Manifest } from "./types.js"; +import { makeManifestFlow } from "./manifest.testUtils.js"; const envDir = "/qawolf/manifest-test"; @@ -26,7 +27,7 @@ const sample: Manifest = { tagsFetchedAt: undefined, envVarsFetchedAt: "2026-05-10T12:30:00.000Z", flows: [ - { path: "src/checkout.flow.ts", contentHash: "deadbeef", tags: undefined }, + makeManifestFlow({ path: "src/checkout.flow.ts", contentHash: "deadbeef" }), ], }; @@ -79,7 +80,6 @@ describe("readManifest", () => { { path: "src/checkout.flow.ts", contentHash: "deadbeef", - tags: undefined, }, ], }), @@ -99,12 +99,16 @@ describe("readManifest", () => { ...sample, tagsFetchedAt: "2026-05-10T12:45:00.000Z", flows: [ - { + makeManifestFlow({ path: "src/checkout.flow.ts", contentHash: "deadbeef", tags: ["smoke", "auth"], - }, - { path: "src/untagged.flow.ts", contentHash: "cafe", tags: [] }, + }), + makeManifestFlow({ + path: "src/untagged.flow.ts", + contentHash: "cafe", + tags: [], + }), ], }; diff --git a/src/shell/manifest/io.ts b/src/shell/manifest/io.ts index ae8e21944..96318fc0c 100644 --- a/src/shell/manifest/io.ts +++ b/src/shell/manifest/io.ts @@ -15,6 +15,7 @@ const flowEntrySchema = z.object({ // Absent on manifests written before tags existed, and on flows the tag // fetch did not return. Both optional so an older manifest still parses. tags: z.array(z.string()).optional(), + flowId: z.string().optional(), }); const manifestSchema = z.object({ @@ -70,6 +71,7 @@ export async function readManifest( path: flow.path, contentHash: flow.contentHash, tags: flow.tags, + flowId: flow.flowId, })), }; } diff --git a/src/shell/manifest/lookup.test.ts b/src/shell/manifest/lookup.test.ts index fbbdfe1a1..52186cfd9 100644 --- a/src/shell/manifest/lookup.test.ts +++ b/src/shell/manifest/lookup.test.ts @@ -6,6 +6,7 @@ import { afterEach, beforeEach, describe, expect, it } from "bun:test"; import { manifestFilename, writeManifest } from "./io.js"; import { findFlowStamp } from "./lookup.js"; import type { Manifest } from "./types.js"; +import { makeManifestFlow } from "./manifest.testUtils.js"; let workDir = ""; @@ -44,12 +45,14 @@ const sampleManifest: Manifest = { tagsFetchedAt: undefined, envVarsFetchedAt: undefined, flows: [ - { + makeManifestFlow({ path: join("src", "login.flow.ts"), contentHash: "hash-login", - tags: undefined, - }, - { path: "checkout.flow.ts", contentHash: "hash-checkout", tags: undefined }, + }), + makeManifestFlow({ + path: "checkout.flow.ts", + contentHash: "hash-checkout", + }), ], }; diff --git a/src/shell/manifest/manifest.testUtils.ts b/src/shell/manifest/manifest.testUtils.ts new file mode 100644 index 000000000..bcd4cb604 --- /dev/null +++ b/src/shell/manifest/manifest.testUtils.ts @@ -0,0 +1,29 @@ +import type { Manifest } from "./types.js"; + +type ManifestFlow = Manifest["flows"][number]; + +/** A flow entry with every optional field unset, then `over`. */ +export const makeManifestFlow = ( + over: Partial = {}, +): ManifestFlow => ({ + path: "src/flows/a.flow.ts", + contentHash: "hash", + tags: undefined, + flowId: undefined, + ...over, +}); + +/** A manifest with nothing fetched and no flows, then `over`. */ +export const makeManifest = (over: Partial = {}): Manifest => ({ + envId: "env-1", + envSlug: undefined, + envName: undefined, + fetchedAt: "2026-05-10T12:00:00.000Z", + envVarsFetchedAt: undefined, + cliFlowsVersion: "0.1.0", + qawolfCommitSha: undefined, + qawolfCommittedAt: undefined, + tagsFetchedAt: undefined, + flows: [], + ...over, +}); diff --git a/src/shell/manifest/types.ts b/src/shell/manifest/types.ts index bc1c0c7f9..cc1580128 100644 --- a/src/shell/manifest/types.ts +++ b/src/shell/manifest/types.ts @@ -5,6 +5,8 @@ type ManifestFlowEntry = { // absent both on pre-tags manifests and on flows the tag fetch skipped — // `Manifest.tagsFetchedAt` distinguishes those from a genuinely untagged flow. tags: string[] | undefined; + // Absent on older manifests and flows the listing did not cover. + flowId: string | undefined; }; // Identifies a flow run against a pulled env: derived by walking the diff --git a/src/shell/walkFiles.ts b/src/shell/walkFiles.ts new file mode 100644 index 000000000..c8c0aa4d9 --- /dev/null +++ b/src/shell/walkFiles.ts @@ -0,0 +1,18 @@ +import { join } from "node:path"; + +import type { Fs } from "./fs.js"; + +export async function walkFiles( + dir: string, + include: (name: string) => boolean, + fs: Fs, +): Promise { + const found: string[] = []; + for (const entry of await fs.readdirWithTypes(dir)) { + const path = join(dir, entry.name); + if (entry.isDirectory()) + found.push(...(await walkFiles(path, include, fs))); + else if (entry.isFile() && include(entry.name)) found.push(path); + } + return found; +}