From 999e6f615e1890a90e52d9f10e6fbe09dcc1eeaf Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Tue, 3 Nov 2026 12:24:47 -0800 Subject: [PATCH 1/3] improvement(executor): make run-from-block refusals actionable --- apps/sim/executor/execution/executor.test.ts | 12 +- apps/sim/executor/execution/executor.ts | 9 +- apps/sim/executor/utils/run-from-block.ts | 78 +++++++---- .../run-workflow-from-copilot.test.ts | 125 ++++++++++++++++++ .../application/run-workflow-from-copilot.ts | 65 ++++++++- 5 files changed, 255 insertions(+), 34 deletions(-) diff --git a/apps/sim/executor/execution/executor.test.ts b/apps/sim/executor/execution/executor.test.ts index c307535850b..584fbfce142 100644 --- a/apps/sim/executor/execution/executor.test.ts +++ b/apps/sim/executor/execution/executor.test.ts @@ -197,7 +197,10 @@ describe('DAGExecutor run-from-block snapshot metadata', () => { version: '1', blocks: [ createBlock('start', BlockType.STARTER), - createBlock('producer', BlockType.FUNCTION), + { + ...createBlock('producer', BlockType.FUNCTION), + metadata: { id: 'function', name: 'Fetch' }, + }, createBlock('consumer', BlockType.FUNCTION), ], connections: [ @@ -220,7 +223,12 @@ describe('DAGExecutor run-from-block snapshot metadata', () => { const error = await executor.executeFromBlock('wf', 'consumer', sourceSnapshot).catch((e) => e) expect(error).toBeInstanceOf(RunFromBlockValidationError) - expect(error.message).toBe('Upstream dependency not executed: producer') + expect(error.message).toBe('Upstream dependency not executed: Fetch') + expect(error.refusal).toEqual({ + kind: 'upstream_not_executed', + blockId: 'consumer', + upstreamBlockId: 'producer', + }) }) }) diff --git a/apps/sim/executor/execution/executor.ts b/apps/sim/executor/execution/executor.ts index d381c17c449..b2f20695c85 100644 --- a/apps/sim/executor/execution/executor.ts +++ b/apps/sim/executor/execution/executor.ts @@ -153,10 +153,15 @@ export class DAGExecutor { const dag = this.dagBuilder.build(this.workflow, { includeAllBlocks: true }) const executedBlocks = new Set(sourceSnapshot.executedBlocks) - const validation = validateRunFromBlock(startBlockId, dag, executedBlocks) + const blockNames = new Map() + for (const block of this.workflow.blocks) { + if (block.metadata?.name) blockNames.set(block.id, block.metadata.name) + } + const validation = validateRunFromBlock(startBlockId, dag, executedBlocks, blockNames) if (!validation.valid) { throw new RunFromBlockValidationError( - validation.error ?? `Cannot run from block: ${startBlockId}` + validation.error ?? `Cannot run from block: ${startBlockId}`, + validation.refusal ) } diff --git a/apps/sim/executor/utils/run-from-block.ts b/apps/sim/executor/utils/run-from-block.ts index 08300e3e481..6aa15d909b5 100644 --- a/apps/sim/executor/utils/run-from-block.ts +++ b/apps/sim/executor/utils/run-from-block.ts @@ -29,8 +29,24 @@ export function resolveContainerToSentinelStart(blockId: string, dag: DAG): stri export interface RunFromBlockValidation { valid: boolean error?: string + /** Why the start was refused, by id, for callers that phrase their own guidance. */ + refusal?: RunFromBlockRefusal } +/** + * The structured reason behind a refused run-from-block start. `blockId` is always the + * requested start block. + */ +export type RunFromBlockRefusal = + | { kind: 'block_not_found'; blockId: string } + | { + kind: 'inside_subflow' + blockId: string + subflowId: string + subflowType: 'loop' | 'parallel' + } + | { kind: 'upstream_not_executed'; blockId: string; upstreamBlockId: string } + /** * A run-from-block start the executor refused before running anything: the block is * missing, sits inside a loop or parallel, or has an upstream dependency the source @@ -38,9 +54,12 @@ export interface RunFromBlockValidation { * to the caller as a validation failure instead of the generic system-error fallback. */ export class RunFromBlockValidationError extends Error { - constructor(message: string) { + readonly refusal?: RunFromBlockRefusal + + constructor(message: string, refusal?: RunFromBlockRefusal) { super(message) this.name = 'RunFromBlockValidationError' + this.refusal = refusal } } @@ -158,13 +177,30 @@ export function computeExecutionSets(dag: DAG, startBlockId: string): ExecutionS * @param blockId - The block ID to validate * @param dag - The workflow DAG * @param executedBlocks - Set of blocks that were executed in the source run + * @param blockNames - Display names by block id, so the message names blocks instead of ids * @returns Validation result with error message if invalid */ export function validateRunFromBlock( blockId: string, dag: DAG, - executedBlocks: Set + executedBlocks: Set, + blockNames: ReadonlyMap = new Map() ): RunFromBlockValidation { + const label = (id: string) => blockNames.get(id) ?? id + const insideSubflow = ( + subflowId: string | undefined, + subflowType: 'loop' | 'parallel' + ): RunFromBlockValidation => { + if (!subflowId) { + return { valid: false, error: `Cannot run from block inside ${subflowType}: ${blockId}` } + } + return { + valid: false, + error: `Cannot run from block inside ${subflowType}: ${label(subflowId)}. Run the ${subflowType} or the whole workflow instead.`, + refusal: { kind: 'inside_subflow', blockId, subflowId, subflowType }, + } + } + const node = dag.nodes.get(blockId) const isLoopContainer = dag.loopConfigs.has(blockId) const isParallelContainer = dag.parallelConfigs.has(blockId) @@ -172,25 +208,19 @@ export function validateRunFromBlock( let validationNode = node if (!node && !isContainer) { - return { valid: false, error: `Block not found in workflow: ${blockId}` } + return { + valid: false, + error: `Block not found in workflow: ${blockId}`, + refusal: { kind: 'block_not_found', blockId }, + } } if (isContainer) { const parentLoopId = findParentLoop(blockId, dag) - if (parentLoopId) { - return { - valid: false, - error: `Cannot run from block inside loop: ${parentLoopId}`, - } - } + if (parentLoopId) return insideSubflow(parentLoopId, 'loop') const parentParallelId = findParentParallel(blockId, dag) - if (parentParallelId) { - return { - valid: false, - error: `Cannot run from block inside parallel: ${parentParallelId}`, - } - } + if (parentParallelId) return insideSubflow(parentParallelId, 'parallel') const sentinelStartId = resolveContainerToSentinelStart(blockId, dag) if (!sentinelStartId || !dag.nodes.has(sentinelStartId)) { @@ -203,19 +233,8 @@ export function validateRunFromBlock( } if (node) { - if (node.metadata.isLoopNode) { - return { - valid: false, - error: `Cannot run from block inside loop: ${node.metadata.subflowId}`, - } - } - - if (node.metadata.isParallelBranch) { - return { - valid: false, - error: `Cannot run from block inside parallel: ${node.metadata.subflowId}`, - } - } + if (node.metadata.isLoopNode) return insideSubflow(node.metadata.subflowId, 'loop') + if (node.metadata.isParallelBranch) return insideSubflow(node.metadata.subflowId, 'parallel') if (node.metadata.isSentinel) { return { valid: false, error: 'Cannot run from sentinel node' } @@ -236,7 +255,8 @@ export function validateRunFromBlock( if (!executedBlocks.has(sourceId)) { return { valid: false, - error: `Upstream dependency not executed: ${sourceId}`, + error: `Upstream dependency not executed: ${label(sourceId)}`, + refusal: { kind: 'upstream_not_executed', blockId, upstreamBlockId: sourceId }, } } } diff --git a/apps/sim/lib/workflows/application/run-workflow-from-copilot.test.ts b/apps/sim/lib/workflows/application/run-workflow-from-copilot.test.ts index f985d210379..cf3d8d8738a 100644 --- a/apps/sim/lib/workflows/application/run-workflow-from-copilot.test.ts +++ b/apps/sim/lib/workflows/application/run-workflow-from-copilot.test.ts @@ -306,6 +306,131 @@ describe('Copilot workflow run application commands', () => { expect(readAttemptedExecutionId(error)).toBe('child-execution-1') }) + describe('a refused run-from-block start tells the agent what to do next', () => { + const blocks = { + 'start-1': { id: 'start-1', name: 'Start', enabled: true }, + 'fetch-1': { id: 'fetch-1', name: 'Fetch', enabled: true }, + 'loop-1': { id: 'loop-1', name: 'Each row', enabled: true }, + 'agent-1': { id: 'agent-1', name: 'Summarize', enabled: true }, + 'note-1': { id: 'note-1', name: 'Notes', enabled: false }, + } + + async function refuse( + blockId: string, + error: RunFromBlockValidationError, + workflowBlocks: Record = blocks + ) { + mockLoadDraft.mockResolvedValue({ blocks: workflowBlocks, edges: [] }) + mocks.sourceState.mockResolvedValueOnce({ + blockStates: {}, + executedBlocks: [], + blockLogs: [], + decisions: {}, + completedLoops: [], + activeExecutionPath: [], + }) + mockExecuteWorkflow.mockRejectedValueOnce(error) + const thrown = await runFromBlockFromCopilot + .execute({ + principal, + input: { + workflowId: 'workflow-1', + useDraftState: true, + lifecycle, + blockId, + sourceExecutionId: 'source-execution-1', + }, + }) + .catch((caught) => caught) + expect(thrown).toBeInstanceOf(OrchestrationError) + expect(thrown.code).toBe('validation') + return thrown.message as string + } + + it('names the upstream block that never ran and the calls that get past it', async () => { + const message = await refuse( + 'agent-1', + new RunFromBlockValidationError('Upstream dependency not executed: Fetch', { + kind: 'upstream_not_executed', + blockId: 'agent-1', + upstreamBlockId: 'fetch-1', + }) + ) + + expect(message).toContain('"Summarize" (agent-1)') + expect(message).toContain('"Fetch" (fetch-1)') + expect(message).toContain('execution source-execution-1') + expect(message).toContain('run_workflow_until_block') + expect(message).toContain('stopAfterBlockId "agent-1"') + expect(message).toContain('executionId') + }) + + it('names the container a nested block sits in and points at it', async () => { + const message = await refuse( + 'agent-1', + new RunFromBlockValidationError('Cannot run from block inside loop: Each row', { + kind: 'inside_subflow', + blockId: 'agent-1', + subflowId: 'loop-1', + subflowType: 'loop', + }) + ) + + expect(message).toContain('inside loop "Each row" (loop-1)') + expect(message).toContain('block id "loop-1"') + expect(message).toContain('run_workflow') + }) + + it('lists the blocks the workflow does have when the id is unknown', async () => { + const message = await refuse( + 'agent-x', + new RunFromBlockValidationError('Block not found in workflow: agent-x', { + kind: 'block_not_found', + blockId: 'agent-x', + }) + ) + + expect(message).toContain('"agent-x" is not in the draft workflow') + expect(message).toContain('"Fetch" (fetch-1)') + expect(message).toContain('"Each row" (loop-1)') + }) + + it('bounds the block list on a large workflow', async () => { + const many = Object.fromEntries( + Array.from({ length: 25 }, (_, index) => [ + `b-${index}`, + { id: `b-${index}`, name: `Block ${index}`, enabled: true }, + ]) + ) + const message = await refuse( + 'missing', + new RunFromBlockValidationError('Block not found in workflow: missing', { + kind: 'block_not_found', + blockId: 'missing', + }), + many + ) + + expect(message).toContain('(b-19)') + expect(message).not.toContain('(b-20)') + expect(message).toContain('and 5 more') + }) + + it('says a known but unrunnable block is disabled instead of missing', async () => { + const message = await refuse( + 'note-1', + new RunFromBlockValidationError('Block not found in workflow: note-1', { + kind: 'block_not_found', + blockId: 'note-1', + }) + ) + + expect(message).toContain('"Notes" (note-1)') + expect(message).toContain('disabled') + expect(message).not.toContain('is not in the') + }) + }) + /** * A caller whose result was withheld decides about retry from one fact: whether a run * exists. This layer owns that answer, because it is the last place that can distinguish diff --git a/apps/sim/lib/workflows/application/run-workflow-from-copilot.ts b/apps/sim/lib/workflows/application/run-workflow-from-copilot.ts index dcf07c96f1b..ee466383500 100644 --- a/apps/sim/lib/workflows/application/run-workflow-from-copilot.ts +++ b/apps/sim/lib/workflows/application/run-workflow-from-copilot.ts @@ -3,6 +3,7 @@ import { createLogger } from '@sim/logger' import { toError } from '@sim/utils/errors' import { generateId } from '@sim/utils/id' import { mergeSubblockStateWithValues } from '@sim/workflow-persistence/subblocks' +import type { BlockState } from '@sim/workflow-types/workflow' import { releaseExecutionSlot } from '@/lib/billing/calculations/usage-reservation' import type { BillingAttributionSnapshot } from '@/lib/billing/core/attribution' import { OrchestrationError } from '@/lib/core/orchestration/types' @@ -244,6 +245,61 @@ async function resolveSourceSnapshot(input: SnapshotCopilotRunInput): Promise<{ ) } +/** How many workflow blocks a "block not found" refusal lists before summarizing the rest. */ +const MAX_LISTED_BLOCKS = 20 + +/** + * Rewrites a refused run-from-block start as guidance the agent can act on: blocks by name + * and id, and the tool call that gets past the refusal. The executor's message is written for + * the canvas, where tool names mean nothing. Never suggests anything that would run more than + * the caller asked for: every alternative is the agent's own next call. + */ +function describeRunFromBlockRefusal( + error: RunFromBlockValidationError, + blocks: Record, + source: { executionId?: string; useDraftState: boolean } +): string { + const refusal = error.refusal + if (!refusal) return error.message + const label = (id: string) => { + const name = blocks[id]?.name + return name ? `"${name}" (${id})` : `"${id}"` + } + + switch (refusal.kind) { + case 'upstream_not_executed': { + const execution = source.executionId + ? `execution ${source.executionId}` + : 'the source execution' + return ( + `Cannot run from ${label(refusal.blockId)}: its upstream block ${label(refusal.upstreamBlockId)} did not run in ${execution}, so there is no output to start from. ` + + `Run the workflow up to this block with run_workflow_until_block (stopAfterBlockId "${refusal.blockId}"), ` + + `or pass the executionId of a run that executed ${label(refusal.upstreamBlockId)}.` + ) + } + case 'inside_subflow': + return ( + `Cannot run ${label(refusal.blockId)} on its own: it is inside ${refusal.subflowType} ${label(refusal.subflowId)}. ` + + `Start from the ${refusal.subflowType} itself (block id "${refusal.subflowId}") or run the whole workflow with run_workflow.` + ) + case 'block_not_found': { + const state = source.useDraftState ? 'draft' : 'deployed' + const existing = blocks[refusal.blockId] + if (existing?.enabled === false) { + return `Block ${label(refusal.blockId)} is disabled in the ${state} workflow, so it cannot be run.` + } + if (existing) { + return `Block ${label(refusal.blockId)} is not an executable block, so it cannot be run.` + } + const all = Object.keys(blocks) + const listed = all.slice(0, MAX_LISTED_BLOCKS).map(label).join(', ') + const more = + all.length > MAX_LISTED_BLOCKS ? ` and ${all.length - MAX_LISTED_BLOCKS} more` : '' + return `Block "${refusal.blockId}" is not in the ${state} workflow. Its blocks are: ${listed}${more}.` + } + } +} + async function executeCopilotRun(params: { principal: Principal input: BaseCopilotRunInput @@ -257,6 +313,8 @@ async function executeCopilotRun(params: { sourceExecutionId?: string variableInputs?: Record } + /** Phrases a refused run-from-block start for the agent; the executor's message otherwise. */ + describeRefusal?: (error: RunFromBlockValidationError) => string }): Promise { if ( params.principal.kind === 'resource_delegated' || @@ -366,7 +424,7 @@ async function executeCopilotRun(params: { */ const error = caught instanceof RunFromBlockValidationError - ? new OrchestrationError('validation', caught.message) + ? new OrchestrationError('validation', params.describeRefusal?.(caught) ?? caught.message) : caught /** * `executeWorkflow` names the run itself once it crosses its own dispatch boundary, so @@ -491,6 +549,11 @@ function defineSnapshotRunUseCase( ...(input.variableInputs ? { variableInputs: input.variableInputs } : {}), }, stopAfterBlockId: stopAtStartBlock ? input.blockId : undefined, + describeRefusal: (error) => + describeRunFromBlockRefusal(error, state.blocks, { + executionId: source.executionId, + useDraftState: input.useDraftState, + }), }) }, }) From 085e0e5ea6a076b3e3441c03fb9d194fe8d4f094 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Tue, 3 Nov 2026 12:52:58 -0800 Subject: [PATCH 2/3] fix(executor): name the outermost container and catch parallel children by workflow id --- .../sim/executor/utils/run-from-block.test.ts | 47 +++++++++++++++++++ apps/sim/executor/utils/run-from-block.ts | 36 +++++++++++--- 2 files changed, 77 insertions(+), 6 deletions(-) diff --git a/apps/sim/executor/utils/run-from-block.test.ts b/apps/sim/executor/utils/run-from-block.test.ts index 316abc26414..759edd13c84 100644 --- a/apps/sim/executor/utils/run-from-block.test.ts +++ b/apps/sim/executor/utils/run-from-block.test.ts @@ -333,6 +333,53 @@ describe('validateRunFromBlock', () => { expect(result.error).toContain(outerParallelId) }) + it('names the outermost container for a block in a nested loop, since the inner loop is refused too', () => { + const dag = createDAG([ + createNode('B', [], { isLoopNode: true, subflowId: 'inner-loop', subflowType: 'loop' }), + ]) + dag.loopConfigs.set('outer-loop', { + id: 'outer-loop', + nodes: ['inner-loop'], + iterations: 2, + loopType: 'for', + } as any) + dag.loopConfigs.set('inner-loop', { + id: 'inner-loop', + nodes: ['B'], + iterations: 2, + loopType: 'for', + } as any) + + const result = validateRunFromBlock('B', dag, new Set()) + + expect(result.valid).toBe(false) + expect(result.refusal).toEqual({ + kind: 'inside_subflow', + blockId: 'B', + subflowId: 'outer-loop', + subflowType: 'loop', + }) + }) + + it('refuses a parallel child by its workflow id as inside the parallel, not as missing', () => { + // The DAG holds parallel children only as branch clones, never under the workflow id. + const dag = createDAG([ + createNode('Pā‚0ā‚Ž', [], { isParallelBranch: true, subflowId: 'par', subflowType: 'parallel' }), + ]) + dag.parallelConfigs.set('par', { id: 'par', nodes: ['P'], count: 2 } as any) + + const result = validateRunFromBlock('P', dag, new Set(), new Map([['par', 'Fan out']])) + + expect(result.valid).toBe(false) + expect(result.error).toContain('inside parallel: Fan out') + expect(result.refusal).toEqual({ + kind: 'inside_subflow', + blockId: 'P', + subflowId: 'par', + subflowType: 'parallel', + }) + }) + it('rejects container when sentinel-start upstream dependency was not executed', () => { const loopId = 'loop-container-1' const sentinelStartId = `loop-${loopId}-sentinel-start` diff --git a/apps/sim/executor/utils/run-from-block.ts b/apps/sim/executor/utils/run-from-block.ts index 6aa15d909b5..3fbfd20d998 100644 --- a/apps/sim/executor/utils/run-from-block.ts +++ b/apps/sim/executor/utils/run-from-block.ts @@ -201,6 +201,9 @@ export function validateRunFromBlock( } } + const container = findOutermostContainer(blockId, dag) + if (container) return insideSubflow(container.id, container.type) + const node = dag.nodes.get(blockId) const isLoopContainer = dag.loopConfigs.has(blockId) const isParallelContainer = dag.parallelConfigs.has(blockId) @@ -216,12 +219,6 @@ export function validateRunFromBlock( } if (isContainer) { - const parentLoopId = findParentLoop(blockId, dag) - if (parentLoopId) return insideSubflow(parentLoopId, 'loop') - - const parentParallelId = findParentParallel(blockId, dag) - if (parentParallelId) return insideSubflow(parentParallelId, 'parallel') - const sentinelStartId = resolveContainerToSentinelStart(blockId, dag) if (!sentinelStartId || !dag.nodes.has(sentinelStartId)) { return { @@ -265,6 +262,33 @@ export function validateRunFromBlock( return { valid: true } } +/** + * The top-level loop or parallel a block sits in, however deeply nested. A nested container + * is refused as a start too, so naming the immediate parent would point at another refusal. + * Parallel children are matched by their workflow id, which the DAG only holds as branch clones. + */ +function findOutermostContainer( + blockId: string, + dag: DAG +): { id: string; type: 'loop' | 'parallel' } | undefined { + let outermost: { id: string; type: 'loop' | 'parallel' } | undefined + const visited = new Set([blockId]) + let current = blockId + while (true) { + const loopId = findParentLoop(current, dag) + const parallelId = loopId ? undefined : findParentParallel(current, dag) + const parent = loopId + ? { id: loopId, type: 'loop' as const } + : parallelId + ? { id: parallelId, type: 'parallel' as const } + : undefined + if (!parent || visited.has(parent.id)) return outermost + visited.add(parent.id) + outermost = parent + current = parent.id + } +} + function findParentLoop(blockId: string, dag: DAG): string | undefined { for (const [loopId, config] of dag.loopConfigs) { if (loopId !== blockId && config.nodes?.includes(blockId)) { From 495445578f38943cd48aba626688e243779371d3 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Tue, 3 Nov 2026 13:12:05 -0800 Subject: [PATCH 3/3] chore(executor): type subflow configs in new run-from-block tests --- apps/sim/executor/utils/run-from-block.test.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/apps/sim/executor/utils/run-from-block.test.ts b/apps/sim/executor/utils/run-from-block.test.ts index 759edd13c84..3de2d7de8e3 100644 --- a/apps/sim/executor/utils/run-from-block.test.ts +++ b/apps/sim/executor/utils/run-from-block.test.ts @@ -342,13 +342,13 @@ describe('validateRunFromBlock', () => { nodes: ['inner-loop'], iterations: 2, loopType: 'for', - } as any) + }) dag.loopConfigs.set('inner-loop', { id: 'inner-loop', nodes: ['B'], iterations: 2, loopType: 'for', - } as any) + }) const result = validateRunFromBlock('B', dag, new Set()) @@ -366,7 +366,7 @@ describe('validateRunFromBlock', () => { const dag = createDAG([ createNode('Pā‚0ā‚Ž', [], { isParallelBranch: true, subflowId: 'par', subflowType: 'parallel' }), ]) - dag.parallelConfigs.set('par', { id: 'par', nodes: ['P'], count: 2 } as any) + dag.parallelConfigs.set('par', { id: 'par', nodes: ['P'], count: 2 }) const result = validateRunFromBlock('P', dag, new Set(), new Map([['par', 'Fan out']]))