fix(files): persist large Agiloft and Cursor downloads before serialization

Author: Bill Leoutsakos

.agents/skills/add-integration/SKILL.md

@@ -523,46 +523,31 @@ registry/direct-handler test. There is no HTTP fallback.
 
 ### File Output Pattern (Downloads)
 
-For tools that return files, use `FileToolProcessor` to store files and return `UserFile` objects.
-
-#### In Tool transformResponse
-
-```typescript
-import { FileToolProcessor } from '@/executor/utils/file-tool-processor'
-
-transformResponse: async (response, context) => {
-  const data = await response.json()
-
-  // Process file outputs to UserFile objects
-  const fileProcessor = new FileToolProcessor(context)
-  const file = await fileProcessor.processFileData({
-    data: data.content,      // base64 or buffer
-    mimeType: data.mimeType,
-    filename: data.filename,
-  })
-
-  return {
-    success: true,
-    output: { file },
-  }
-}
-```
-
-#### In the operation handler (for complex file handling)
-
-```typescript
-// Return file data that FileToolProcessor can handle. No API route is involved.
-return Response.json({
-  success: true,
-  output: {
-    file: {
-      data: base64Content,
-      mimeType: 'application/pdf',
-      filename: 'document.pdf',
-    },
-  },
-})
-```
+Declare downloads as `file` / `file[]` outputs and return canonical `UserFile` objects.
+Internal operation responses are capped at 10 MiB **before** `transformResponse` and
+`FileToolProcessor` run. Inline base64 expands the bytes by roughly one third, so it
+cannot carry a download near that limit. Persist downloads in the server operation
+**before `Response.json`**, not in a response transform.
+
+Follow `executeQuickBooksDownloadDocument` or `executeAgiloftRetrieveAttachment`:
+
+- Derive storage scope only from trusted `request.context`, never tool parameters.
+  Use `uploadExecutionFile` for a complete workspace/workflow/execution scope;
+  otherwise use `uploadCopilotFile` with the trusted user identity. Reject missing
+  storage authority before downloading. Do not fabricate an `ExecutionContext`.
+- Keep provider authentication, DNS-pinned downloads, byte caps and cancellation.
+  Normalize image metadata with `resolveStoredFileMetadata` before uploading.
+- Return the stored file unchanged through the response schema and transform; use
+  `userFileSchema` / `UserFile` rather than rebuilding a base64-only shape.
+  `FileToolProcessor` passes stored files through; the executor records them for
+  execution consumers. Do not call its private `processFileData` method.
+- Surface storage failures instead of falling back to an oversized inline payload.
+  Storage helpers do not promise rollback when later execution steps fail.
+
+Test provider bytes larger than the inline JSON budget through the actual handler,
+bounded response reader, transform and file processor, mocking provider/storage
+boundaries only. Preserve explicit legacy base64 outputs when they are a separate
+versioned contract; do not silently convert those outputs or raise the global cap.
 
 ### Key Helpers Reference
 

apps/sim/executor/utils/file-tool-processor.ts

@@ -4,19 +4,12 @@ import { isCanonicalBase64 } from '@/lib/api/contracts/primitives'
 import { isUserFile } from '@/lib/core/utils/user-file'
 import { uploadExecutionFile, uploadFileFromRawData } from '@/lib/uploads/contexts/execution'
 import { downloadFileFromUrl } from '@/lib/uploads/utils/file-utils.server'
-import { MAX_FILE_SIZE, sniffImageContentType } from '@/lib/uploads/utils/validation'
+import { MAX_FILE_SIZE, resolveStoredFileMetadata } from '@/lib/uploads/utils/validation'
 import type { ExecutionContext, UserFile } from '@/executor/types'
 import type { ToolDefinition, ToolFileData } from '@/tools/types'
 
 const logger = createLogger('FileToolProcessor')
 
-const IMAGE_FILE_EXTENSIONS: Record<string, string> = {
-  'image/gif': 'gif',
-  'image/jpeg': 'jpg',
-  'image/png': 'png',
-  'image/webp': 'webp',
-}
-
 /**
  * Strip a base64 `data:` URI prefix, leaving the encoded payload. An empty payload is
  * a legitimate zero-byte file; a payload that only looks empty after normalization is
@@ -43,30 +36,6 @@ function assertFileSize(size: number, fileName: string): void {
   }
 }
 
-function resolveStoredFileMetadata(
-  fileName: string,
-  declaredMimeType: string,
-  buffer: Buffer
-): { fileName: string; mimeType: string } {
-  if (!declaredMimeType.startsWith('image/')) {
-    return { fileName, mimeType: declaredMimeType }
-  }
-
-  const mimeType = sniffImageContentType(buffer)
-  if (!mimeType) {
-    return {
-      fileName: `${fileName.replace(/\.[^.]+$/, '')}.bin`,
-      mimeType: 'application/octet-stream',
-    }
-  }
-
-  const extension = IMAGE_FILE_EXTENSIONS[mimeType]
-  return {
-    fileName: extension ? `${fileName.replace(/\.[^.]+$/, '')}.${extension}` : fileName,
-    mimeType,
-  }
-}
-
 /**
  * Processes tool outputs and converts file-typed outputs to UserFile objects.
  * This enables tools to return file data that gets automatically stored in the

apps/sim/lib/api/contracts/tools/agiloft.ts

@@ -1,4 +1,5 @@
 import { z } from 'zod'
+import { userFileSchema } from '@/lib/api/contracts/primitives'
 import type {
   ContractBody,
   ContractBodyInput,
@@ -18,17 +19,10 @@ const optionalText = z
   .nullish()
   .transform((value) => value ?? undefined)
 
-const agiloftFileOutputSchema = z.object({
-  name: z.string(),
-  mimeType: z.string(),
-  data: z.string(),
-  size: z.number(),
-})
-
 export const agiloftRetrieveResponseSchema = z.object({
   success: z.literal(true),
   output: z.object({
-    file: agiloftFileOutputSchema,
+    file: userFileSchema,
   }),
 })
 

apps/sim/lib/internal/agiloft/execute-tool.test.ts

@@ -185,11 +185,14 @@ describe('executeAgiloftTool', () => {
       })
     )
 
-    expect(operationMocks.executeAgiloftCreateRecord).toHaveBeenCalledWith(input, {
-      requestId: 'request-1',
-      userId: 'user-origin',
-      signal: controller.signal,
-    })
+    expect(operationMocks.executeAgiloftCreateRecord).toHaveBeenCalledWith(
+      input,
+      expect.objectContaining({
+        requestId: 'request-1',
+        userId: 'user-origin',
+        signal: controller.signal,
+      })
+    )
   })
 
   it('preserves non-object input and canonical validation envelopes', async () => {

apps/sim/lib/internal/agiloft/execute-tool.ts

@@ -77,6 +77,9 @@ async function executeOperation<C extends AnyApiRouteContract>(
     const result = await operation(parsed.data, {
       requestId: request.requestId,
       userId: request.context.executorDelegationOrigin?.subjectUserId ?? request.context.userId,
+      workspaceId: request.context.workspaceId,
+      workflowId: request.context.workflowId,
+      executionId: request.context.executionId,
       signal: request.signal,
     })
     request.signal?.throwIfAborted()

apps/sim/lib/internal/agiloft/operations.test.ts

@@ -20,6 +20,7 @@ const providerMocks = vi.hoisted(() => ({
 
 const fileMocks = vi.hoisted(() => ({
   resolveAgiloftAttachmentFile: vi.fn(),
+  uploadCopilotFile: vi.fn(),
 }))
 
 vi.mock('@/lib/core/security/input-validation.server', () => ({
@@ -28,6 +29,8 @@ vi.mock('@/lib/core/security/input-validation.server', () => ({
 }))
 vi.mock('@/lib/internal/agiloft/client', () => clientMocks)
 vi.mock('@/lib/internal/agiloft/file-input', () => fileMocks)
+vi.mock('@/lib/uploads/contexts/copilot/copilot-file-manager', () => fileMocks)
+vi.mock('@/lib/uploads/contexts/execution', () => ({ uploadExecutionFile: vi.fn() }))
 
 import {
   executeAgiloftCreateRecord,
@@ -148,6 +151,15 @@ describe('Agiloft operations', () => {
 
   it('bounds attachment downloads and preserves binary metadata', async () => {
     const controller = new AbortController()
+    const storedFile = {
+      id: 'file-1',
+      key: 'copilot/user-1/file-1',
+      url: '/api/files/serve/file-1',
+      name: 'evidence.txt',
+      type: 'text/plain',
+      size: 5,
+    }
+    fileMocks.uploadCopilotFile.mockResolvedValue(storedFile)
     providerMocks.secureFetchWithPinnedIP.mockResolvedValue(
       createResponse({
         bytes: new TextEncoder().encode('hello'),
@@ -160,20 +172,21 @@ describe('Agiloft operations', () => {
 
     const result = await executeAgiloftRetrieveAttachment(
       { ...BASE, recordId: '1', fieldName: 'files', position: '0' },
-      { requestId: 'request-1', signal: controller.signal }
+      { requestId: 'request-1', userId: 'user-1', signal: controller.signal }
     )
 
     expect(result).toEqual({
       success: true,
       output: {
-        file: {
-          name: 'evidence.txt',
-          mimeType: 'text/plain',
-          data: Buffer.from('hello').toString('base64'),
-          size: 5,
-        },
+        file: storedFile,
       },
     })
+    expect(fileMocks.uploadCopilotFile).toHaveBeenCalledWith({
+      buffer: Buffer.from('hello'),
+      fileName: 'evidence.txt',
+      contentType: 'text/plain',
+      userId: 'user-1',
+    })
     expect(providerMocks.secureFetchWithPinnedIP).toHaveBeenCalledWith(
       expect.stringContaining('/ewws/EWRetrieve'),
       '203.0.113.10',

apps/sim/lib/internal/agiloft/operations.ts

@@ -65,7 +65,10 @@ import {
   getLockHttpMethod,
   parseFieldList,
 } from '@/lib/internal/agiloft/urls'
+import { uploadCopilotFile } from '@/lib/uploads/contexts/copilot/copilot-file-manager'
+import { uploadExecutionFile } from '@/lib/uploads/contexts/execution'
 import { resolveEffectiveMimeType } from '@/lib/uploads/utils/file-utils'
+import { resolveStoredFileMetadata } from '@/lib/uploads/utils/validation'
 import type {
   AgiloftAsyncStatusResponse,
   AgiloftAttachmentInfoResponse,
@@ -88,6 +91,9 @@ import type { ToolResponse } from '@/tools/types'
 export interface AgiloftOperationContext {
   requestId: string
   userId?: string
+  workspaceId?: string
+  workflowId?: string
+  executionId?: string
   signal?: AbortSignal
 }
 
@@ -894,6 +900,20 @@ export async function executeAgiloftRetrieveAttachment(
   input: AgiloftRetrieveBody,
   context: AgiloftOperationContext
 ): Promise<ToolResponse> {
+  const executionContext =
+    context.workspaceId && context.workflowId && context.executionId
+      ? {
+          workspaceId: context.workspaceId,
+          workflowId: context.workflowId,
+          executionId: context.executionId,
+        }
+      : null
+  if (!executionContext && !context.userId) {
+    throw new AgiloftOperationError(401, {
+      success: false,
+      error: 'User context is required to store attachments',
+    })
+  }
   let resolvedIP: string
   try {
     resolvedIP = await resolveAgiloftInstance(input.instanceUrl, context.signal)
@@ -930,15 +950,25 @@ export async function executeAgiloftRetrieveAttachment(
       error: `Agiloft error: ${buffer.toString('utf8').slice(0, 300)}`,
     })
   }
-  return {
-    success: true,
-    output: {
-      file: {
-        name: fileName,
-        mimeType: resolveEffectiveMimeType(contentType, fileName),
-        data: buffer.toString('base64'),
-        size: buffer.length,
-      },
-    },
-  }
+  const metadata = resolveStoredFileMetadata(
+    fileName,
+    resolveEffectiveMimeType(contentType, fileName),
+    buffer
+  )
+  const file = executionContext
+    ? await uploadExecutionFile(
+        executionContext,
+        buffer,
+        metadata.fileName,
+        metadata.mimeType,
+        context.userId
+      )
+    : await uploadCopilotFile({
+        buffer,
+        fileName: metadata.fileName,
+        contentType: metadata.mimeType,
+        userId: context.userId!,
+      })
+  context.signal?.throwIfAborted()
+  return { success: true, output: { file } }
 }

apps/sim/lib/internal/cursor/execute-tool.test.ts

@@ -47,7 +47,7 @@ describe('executeCursorTool', () => {
       expect(response.status).toBe(200)
       expect(mocks.downloadCursorArtifact).toHaveBeenCalledWith(
         { apiKey: 'cursor-key', agentId: 'agent-1', path: '/src/index.ts' },
-        { requestId: 'request-1', signal: controller.signal }
+        expect.objectContaining({ requestId: 'request-1', signal: controller.signal })
       )
     }
   )

apps/sim/lib/internal/cursor/execute-tool.ts

@@ -33,6 +33,16 @@ export const executeCursorTool: InternalToolOperationHandler = async (request) =
       await downloadCursorArtifact(parsed.data, {
         requestId: request.requestId,
         signal: request.signal,
+        ...(request.toolId === 'cursor_download_artifact_v2'
+          ? {
+              persistFile: true,
+              userId:
+                request.context.executorDelegationOrigin?.subjectUserId ?? request.context.userId,
+              workspaceId: request.context.workspaceId,
+              workflowId: request.context.workflowId,
+              executionId: request.context.executionId,
+            }
+          : {}),
       })
     )
   } catch (error) {

apps/sim/lib/internal/cursor/operations.test.ts

@@ -12,6 +12,10 @@ vi.mock('@/lib/core/security/input-validation.server', () => ({
   secureFetchWithPinnedIP: mocks.secureFetchWithPinnedIP,
   validateUrlWithDNS: mocks.validateUrlWithDNS,
 }))
+vi.mock('@/lib/uploads/contexts/copilot/copilot-file-manager', () => ({
+  uploadCopilotFile: vi.fn(),
+}))
+vi.mock('@/lib/uploads/contexts/execution', () => ({ uploadExecutionFile: vi.fn() }))
 
 import { downloadCursorArtifact } from '@/lib/internal/cursor/operations'