feat(flows): cache flow IDs during pull

.changeset/pull-flow-ids.md

@@ -0,0 +1,5 @@
+---
+"@qawolf/cli": minor
+---
+
+Cache flow IDs when pulling an environment and include them in local JSON flow lists. Preserve previously cached IDs when the platform listing is unavailable.

src/core/flowMeta.ts

@@ -75,3 +75,12 @@ export type PeekFlowMetaFn = (filePath: string) => Promise<FlowCallMeta>;
 export function extractFlowMeta(source: string): FlowCallMeta {
   return parseFlowCall(source);
 }
+
+const flowExtensions = [".flow.ts", ".flow.js"];
+const sourceExtensions = [".ts", ".js", ".mts", ".cts", ".mjs", ".cjs"];
+
+export const isFlowFile = (name: string): boolean =>
+  flowExtensions.some((extension) => name.endsWith(extension));
+
+export const isSourceFile = (name: string): boolean =>
+  sourceExtensions.some((extension) => name.endsWith(extension));

src/domains/flows/list.agent.test.ts

@@ -46,8 +46,8 @@ function makeDeps(overrides?: {
         target: metaByFile[file]?.target,
       }),
     ),
-    readCachedTags: mock<FlowsListDeps["readCachedTags"]>(() =>
-      Promise.resolve(new Map<string, readonly string[]>()),
+    readCachedFlows: mock<FlowsListDeps["readCachedFlows"]>(() =>
+      Promise.resolve(new Map()),
     ),
     readEnvLabel: mock<FlowsListDeps["readEnvLabel"]>((dir: string) =>
       Promise.resolve(dir),

src/domains/flows/list.env.test.ts

@@ -9,6 +9,7 @@ import { makeNoopLogger } from "~/shell/logger.testUtils.js";
 import { makeMemoryFs } from "~/shell/fs.testUtils.js";
 
 import { type FlowsListDeps, flowsList } from "./list.js";
+import { cachedFlowsWithTags } from "./list.testUtils.js";
 import { callsOf, makeFakeUI } from "~/shell/commandContext.testUtils.js";
 
 afterEach(() => {
@@ -44,8 +45,8 @@ function makeDeps(
     peekFlowMeta: mock<FlowsListDeps["peekFlowMeta"]>(() =>
       Promise.resolve({ name: undefined, target: undefined }),
     ),
-    readCachedTags: mock<FlowsListDeps["readCachedTags"]>(() =>
-      Promise.resolve(new Map(Object.entries(tagsByFile))),
+    readCachedFlows: mock<FlowsListDeps["readCachedFlows"]>(() =>
+      Promise.resolve(cachedFlowsWithTags(tagsByFile)),
     ),
     readEnvLabel: mock<FlowsListDeps["readEnvLabel"]>((dir: string) =>
       Promise.resolve(dir),
@@ -189,10 +190,8 @@ describe("flowsList --env against a pulled environment", () => {
     const ctx = makeCtx();
     const deps = {
       ...envDeps([stagingFlow, prodFlow]),
-      readCachedTags: mock<FlowsListDeps["readCachedTags"]>(() =>
-        Promise.resolve(
-          new Map<string, readonly string[]>([[stagingFlow, ["auth"]]]),
-        ),
+      readCachedFlows: mock<FlowsListDeps["readCachedFlows"]>(() =>
+        Promise.resolve(cachedFlowsWithTags({ [stagingFlow]: ["auth"] })),
       ),
     };
 

src/domains/flows/list.json.test.ts

@@ -8,6 +8,7 @@ import { makeNoopLogger } from "~/shell/logger.testUtils.js";
 import { makeMemoryFs } from "~/shell/fs.testUtils.js";
 
 import { type FlowsListDeps, flowsList } from "./list.js";
+import { cachedFlowsWithTags } from "./list.testUtils.js";
 import { makeFakeUI } from "~/shell/commandContext.testUtils.js";
 
 const noopSignals = makeNoopSignals();
@@ -51,8 +52,8 @@ function makeDeps(overrides?: {
         target: metaByFile[file]?.target,
       }),
     ),
-    readCachedTags: mock<FlowsListDeps["readCachedTags"]>(() =>
-      Promise.resolve(new Map(Object.entries(cachedTags))),
+    readCachedFlows: mock<FlowsListDeps["readCachedFlows"]>(() =>
+      Promise.resolve(cachedFlowsWithTags(cachedTags)),
     ),
     readEnvLabel: mock<FlowsListDeps["readEnvLabel"]>((dir: string) =>
       Promise.resolve(dir),
@@ -98,6 +99,22 @@ describe("flowsList json mode output", () => {
     expect(ui.outro).not.toHaveBeenCalled();
   });
 
+  it("includes a pulled flow’s cached ID in JSON", async () => {
+    const ui = makeFakeUI();
+    const file = "/proj/.qawolf/staging/src/flows/a.flow.ts";
+    const deps = makeDeps({ files: [file] });
+    const cachedDeps = {
+      ...deps,
+      readCachedFlows: mock(() =>
+        Promise.resolve(new Map([[file, { flowId: "flow-a", tags: [] }]])),
+      ),
+    };
+    await flowsList(makeCtx(ui, "json"), undefined, cachedDeps);
+    expect(ui.json).toHaveBeenCalledWith([
+      expect.objectContaining({ flowId: "flow-a", tags: [] }),
+    ]);
+  });
+
   it("falls back to basename for name when meta.name is undefined", async () => {
     const ui = makeFakeUI();
     const deps = makeDeps({

src/domains/flows/list.selectors.test.ts

@@ -9,6 +9,7 @@ import { makeNoopLogger } from "~/shell/logger.testUtils.js";
 import { makeMemoryFs } from "~/shell/fs.testUtils.js";
 
 import { type FlowsListDeps, flowsList } from "./list.js";
+import { cachedFlowsWithTags } from "./list.testUtils.js";
 import { callsOf, makeFakeUI } from "~/shell/commandContext.testUtils.js";
 
 afterEach(() => {
@@ -45,8 +46,8 @@ function makeDeps(
     peekFlowMeta: mock<FlowsListDeps["peekFlowMeta"]>(() =>
       Promise.resolve({ name: undefined, target: undefined }),
     ),
-    readCachedTags: mock<FlowsListDeps["readCachedTags"]>(() =>
-      Promise.resolve(new Map(Object.entries(tagsByFile))),
+    readCachedFlows: mock<FlowsListDeps["readCachedFlows"]>(() =>
+      Promise.resolve(cachedFlowsWithTags(tagsByFile)),
     ),
     readEnvLabel: mock<FlowsListDeps["readEnvLabel"]>((dir: string) =>
       Promise.resolve(dir),

src/domains/flows/list.test.ts

@@ -49,8 +49,8 @@ function makeDeps(overrides?: {
         target: metaByFile[file]?.target,
       }),
     ),
-    readCachedTags: mock<FlowsListDeps["readCachedTags"]>(() =>
-      Promise.resolve(new Map<string, readonly string[]>()),
+    readCachedFlows: mock<FlowsListDeps["readCachedFlows"]>(() =>
+      Promise.resolve(new Map()),
     ),
     readEnvLabel: mock<FlowsListDeps["readEnvLabel"]>((dir: string) =>
       Promise.resolve(dir),

src/domains/flows/list.testUtils.ts

@@ -0,0 +1,15 @@
+import type { CachedFlow } from "./readCachedFlows.js";
+
+/** What `readCachedFlows` returns when the pulls recorded only these tags. */
+export const cachedFlowsWithTags = (
+  tagsByFile: Record<string, readonly string[]>,
+): Map<string, CachedFlow> =>
+  new Map(
+    Object.entries(tagsByFile).map(([file, tags]) => [
+      file,
+      {
+        tags,
+        flowId: undefined,
+      },
+    ]),
+  );

src/domains/flows/list.ts

@@ -2,6 +2,7 @@ import path from "node:path";
 
 import type { CommandContext, CommandResult } from "~/shell/commandContext.js";
 import { flowsMessages, runnerMessages } from "~/core/messages/index.js";
+import type { CachedFlow } from "./readCachedFlows.js";
 import type { BrowserName } from "~/core/types.js";
 
 import { batchMap, flowBatchSize } from "~/core/batchMap.js";
@@ -23,10 +24,10 @@ export type FlowsListDeps = {
     cwd: string,
   ) => Promise<string[]>;
   readonly peekFlowMeta: PeekFlowMetaFn;
-  /** Tags cached at pull time, keyed by absolute flow path. */
-  readonly readCachedTags: (
+  /** What each flow’s pull recorded, keyed by absolute flow path. */
+  readonly readCachedFlows: (
     files: readonly string[],
-  ) => Promise<Map<string, readonly string[]>>;
+  ) => Promise<ReadonlyMap<string, CachedFlow>>;
   /** Human label for a pulled env dir — its slug, name, or id. */
   readonly readEnvLabel: (envDir: string) => Promise<string>;
   /** Resolves an id, slug, or name to a pulled env, without the API. */
@@ -40,6 +41,7 @@ export type FlowsListDeps = {
 type FlowsListItem = {
   file: string;
   name: string;
+  flowId: string | undefined;
   // The pulled environment the flow came from. Undefined for project flows,
   // which belong to no environment.
   env: string | undefined;
@@ -72,7 +74,11 @@ export async function flowsList(
     if (selection.kind === "unknown") return selection.result;
     files = selection.files;
   }
-  const cachedTags = await deps.readCachedTags(files);
+  const cached = await deps.readCachedFlows(files);
+  const cachedTags = new Map<string, readonly string[]>();
+  for (const [file, flow] of cached) {
+    if (flow.tags !== undefined) cachedTags.set(file, flow.tags);
+  }
   const envLabels = await readEnvLabels(files, deps.readEnvLabel);
 
   const notCached = tagsNotCachedResult(selectors, cachedTags);
@@ -87,6 +93,7 @@ export async function flowsList(
     all.push({
       file: path.relative(deps.cwd, file),
       name: meta.name ?? flowBasename(file),
+      flowId: cached.get(file)?.flowId,
       env: envLabelFor(file, envLabels),
       tags: cachedTags.get(file),
       target: meta.target,

src/domains/flows/listDefaults.ts

@@ -10,7 +10,7 @@ import {
   makePeekFlowMeta,
 } from "./expand.js";
 import { flowsList } from "./list.js";
-import { readCachedTags as defaultReadCachedTags } from "./readCachedTags.js";
+import { readCachedFlows as defaultReadCachedFlows } from "./readCachedFlows.js";
 import { readEnvLabel as defaultReadEnvLabel } from "./readEnvLabel.js";
 
 export function handleFlowsList(
@@ -27,7 +27,7 @@ export function handleFlowsList(
       expandPatterns: (patterns, cwd) =>
         defaultExpandPatterns(patterns, cwd, undefined, fs),
       peekFlowMeta: makePeekFlowMeta(fs),
-      readCachedTags: (files) => defaultReadCachedTags(files, fs),
+      readCachedFlows: (files) => defaultReadCachedFlows(files, fs),
       readEnvLabel: (envDir) => defaultReadEnvLabel(envDir, fs),
       findPulledEnv: (ref) => defaultFindPulledEnv(ref, process.cwd(), fs),
       listPulledEnvDirs: () => defaultListPulledEnvDirs(process.cwd(), fs),

src/domains/flows/manifestEntries.ts

@@ -0,0 +1,45 @@
+import { relative } from "node:path";
+
+import { findPulledEnvDir, toPosix } from "~/core/repoRelativePath.js";
+import { makeDefaultFs, type Fs } from "~/shell/fs.js";
+import { readManifest } from "~/shell/manifest/io.js";
+import type { Manifest } from "~/shell/manifest/types.js";
+
+type RecordedFlow = {
+  readonly entry: Manifest["flows"][number];
+  /** Undefined when no tag fetch ever succeeded for the flow's environment. */
+  readonly tagsFetchedAt: string | undefined;
+};
+
+/** Keyed by absolute path; flows outside a pulled manifest are absent. */
+export async function readManifestEntries(
+  files: readonly string[],
+  fs: Fs = makeDefaultFs(),
+): Promise<Map<string, RecordedFlow>> {
+  // Group by env dir so a listing of many flows reads each manifest once
+  // rather than once per flow.
+  const filesByEnvDir = new Map<string, string[]>();
+  for (const file of files) {
+    const envDir = findPulledEnvDir(file);
+    if (envDir === undefined) continue;
+    const group = filesByEnvDir.get(envDir);
+    if (group) group.push(file);
+    else filesByEnvDir.set(envDir, [file]);
+  }
+
+  const entries = new Map<string, RecordedFlow>();
+  for (const [envDir, envFiles] of filesByEnvDir) {
+    const manifest = await readManifest(envDir, fs);
+    if (typeof manifest === "string") continue;
+    // Compared posix on both sides: a manifest written on win32 by an older
+    // CLI may hold `\` paths.
+    const byPath = new Map(manifest.flows.map((f) => [toPosix(f.path), f]));
+    for (const file of envFiles) {
+      const entry = byPath.get(toPosix(relative(envDir, file)));
+      if (entry !== undefined) {
+        entries.set(file, { entry, tagsFetchedAt: manifest.tagsFetchedAt });
+      }
+    }
+  }
+  return entries;
+}

src/domains/flows/pull/applyTeamStorageRewrite.ts

@@ -1,39 +1,17 @@
-import { join, relative } from "node:path";
+import { relative } from "node:path";
 
+import { isFlowFile, isSourceFile } from "~/core/flowMeta.js";
 import { makeDefaultFs } from "~/shell/fs.js";
 import type { Fs } from "~/shell/fs.js";
+import { walkFiles } from "~/shell/walkFiles.js";
 
 import { rewriteTeamStorage } from "./rewriteTeamStorage.js";
 
-const sourceExtensions = [".ts", ".js", ".mts", ".cts", ".mjs", ".cjs"];
-const flowExtensions = [".flow.ts", ".flow.js"];
-
-function isSourceFile(name: string): boolean {
-  return sourceExtensions.some((ext) => name.endsWith(ext));
-}
-
-function isFlowFile(name: string): boolean {
-  return flowExtensions.some((ext) => name.endsWith(ext));
-}
-
-async function walk(dir: string, out: string[], fs: Fs): Promise<void> {
-  const entries = await fs.readdirWithTypes(dir);
-  for (const e of entries) {
-    const abs = join(dir, e.name);
-    if (e.isDirectory()) {
-      await walk(abs, out, fs);
-    } else if (e.isFile() && isSourceFile(e.name)) {
-      out.push(abs);
-    }
-  }
-}
-
 export async function applyTeamStorageRewrite(
   rootDir: string,
   fs: Fs = makeDefaultFs(),
 ): Promise<{ flowsWithTeamStorageRefs: string[] }> {
-  const files: string[] = [];
-  await walk(rootDir, files, fs);
+  const files = await walkFiles(rootDir, isSourceFile, fs);
   const results = await Promise.all(
     files.map(async (file): Promise<string | undefined> => {
       const source = await fs.readFile(file);

src/domains/flows/pull/bundle.test.ts

@@ -116,6 +116,7 @@ describe("buildManifest", () => {
     wrapperName: string | undefined;
     qawolfCommittedAt: string | undefined;
     tags: undefined;
+    flowIds: undefined;
   } => ({
     envId: "env-x",
     bundleDir: workDir,
@@ -125,6 +126,7 @@ describe("buildManifest", () => {
     wrapperName: undefined,
     qawolfCommittedAt: undefined,
     tags: undefined,
+    flowIds: undefined,
   });
 
   it("walks .flow.ts and .flow.js files, ignores other extensions", async () => {