From 1050e8a7adb40e07c91eddd613aaa2bf1944129d Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Thu, 17 Sep 2026 14:25:11 -0700 Subject: [PATCH 1/2] fix(embeddings): preserve safe provider failure diagnostics --- .../background/knowledge-processing.test.ts | 3 +- apps/sim/lib/embeddings/api-error.ts | 22 ++++ apps/sim/lib/embeddings/client.test.ts | 109 +++++++++++++++++- apps/sim/lib/embeddings/client.ts | 37 +++--- .../lib/embeddings/error-diagnostics.test.ts | 75 ++++++++++++ apps/sim/lib/embeddings/error-diagnostics.ts | 84 ++++++++++++++ .../connectors/connector-error.test.ts | 15 +++ .../knowledge/connectors/connector-error.ts | 10 +- .../document-processing-source.test.ts | 3 +- .../processing-provider-deferral.test.ts | 3 +- 10 files changed, 331 insertions(+), 30 deletions(-) create mode 100644 apps/sim/lib/embeddings/api-error.ts create mode 100644 apps/sim/lib/embeddings/error-diagnostics.test.ts create mode 100644 apps/sim/lib/embeddings/error-diagnostics.ts diff --git a/apps/sim/background/knowledge-processing.test.ts b/apps/sim/background/knowledge-processing.test.ts index 2623409ec68..b200f9a0dbd 100644 --- a/apps/sim/background/knowledge-processing.test.ts +++ b/apps/sim/background/knowledge-processing.test.ts @@ -34,7 +34,8 @@ vi.mock('@/lib/knowledge/documents/service', () => ({ })) import { ProviderCapacityDeferredError } from '@/lib/core/rate-limiter/provider-capacity-error' -import { EmbeddingAPIError, EmbeddingQuotaExhaustedError } from '@/lib/embeddings/client' +import { EmbeddingAPIError } from '@/lib/embeddings/api-error' +import { EmbeddingQuotaExhaustedError } from '@/lib/embeddings/client' import { EMBEDDING_QUOTA_CIRCUIT_TTL_MS } from '@/lib/embeddings/quota-circuit' import { OcrRequestRejectedError, diff --git a/apps/sim/lib/embeddings/api-error.ts b/apps/sim/lib/embeddings/api-error.ts new file mode 100644 index 00000000000..aa7cece606a --- /dev/null +++ b/apps/sim/lib/embeddings/api-error.ts @@ -0,0 +1,22 @@ +export class EmbeddingAPIError extends Error { + public status: number + + /** True when the rejected request used a customer-managed credential. */ + public readonly isBYOK: boolean + + /** Rejected for an exhausted balance rather than a recoverable rate limit. */ + public quotaExhausted?: boolean + + /** + * Wait the provider asked for, read from the rejected response. Consumed by + * {@link retryWithExponentialBackoff}, which prefers it over its own backoff. + */ + public retryAfterMs?: number + + constructor(message: string, status: number, isBYOK = false) { + super(message) + this.name = 'EmbeddingAPIError' + this.status = status + this.isBYOK = isBYOK + } +} diff --git a/apps/sim/lib/embeddings/client.test.ts b/apps/sim/lib/embeddings/client.test.ts index 5efc3104276..2e807434593 100644 --- a/apps/sim/lib/embeddings/client.test.ts +++ b/apps/sim/lib/embeddings/client.test.ts @@ -2,16 +2,16 @@ * @vitest-environment node */ -import { resetEnvMock, setEnv } from '@sim/testing' +import { createMockLogger, resetEnvMock, setEnv } from '@sim/testing' import { interruptibleSleep } from '@sim/utils/helpers' import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import { ProviderCapacityDeferredError } from '@/lib/core/rate-limiter/provider-capacity-error' +import { EmbeddingAPIError } from '@/lib/embeddings/api-error' import { assertKnowledgeEmbeddingCapacityForDeployment, clampEmbeddingConcurrency, EMBEDDING_MAX_RETRIES, EMBEDDING_RETRY_BUDGET_MS, - EmbeddingAPIError, EmbeddingOutputLimitError, EmbeddingQuotaExhaustedError, embed, @@ -28,6 +28,11 @@ const { mockGetBYOKKey } = vi.hoisted(() => ({ mockGetBYOKKey: vi.fn(), })) +const { mockDiagnosticWarn } = vi.hoisted(() => ({ mockDiagnosticWarn: vi.fn() })) +vi.mock('@sim/logger', () => ({ + createLogger: () => ({ ...createMockLogger(), warn: mockDiagnosticWarn }), +})) + const { quotaGates, mockAdmit, mockCooldown, mockQuotaCheck } = vi.hoisted(() => ({ quotaGates: new Set(), mockAdmit: vi.fn(), @@ -116,6 +121,7 @@ function oversizedChunkedSuccessResponse(): Response { let fetchMock: ReturnType beforeEach(() => { + mockDiagnosticWarn.mockClear() mockQuotaCheck .mockReset() .mockImplementation(async (identity: { credentialFingerprint: string }) => @@ -154,6 +160,105 @@ afterEach(() => { resetEnvMock() }) +describe('embedding HTTP failure diagnostics', () => { + const options = { model: 'text-embedding-3-small', projectInputs: null } as const + + it('logs safe OpenAI context internally while preserving the public error and retry policy', async () => { + fetchMock.mockResolvedValue( + jsonResponse( + { error: { code: 'model_not_found', message: 'private document and private-key' } }, + 404, + { 'x-request-id': 'req_test', authorization: 'Bearer private-key' } + ) + ) + const error = await embed(['private document'], { ...options, apiKey: 'private-key' }).catch( + (caught) => caught + ) + expect(error).toBeInstanceOf(EmbeddingAPIError) + expect(error.message).toBe('Embedding API failed: 404') + expect(isTransientEmbeddingError(error)).toBe(false) + expect(fetchMock).toHaveBeenCalledTimes(1) + expect(mockDiagnosticWarn).toHaveBeenCalledWith('Embedding provider request failed', { + providerId: 'openai', + modelName: 'text-embedding-3-small', + status: 404, + providerRequestId: 'req_test', + providerErrorCode: 'model_not_found', + providerErrorType: null, + bodyFormat: 'json', + }) + expect(JSON.stringify(mockDiagnosticWarn.mock.calls)).not.toContain('private') + expect(JSON.stringify(error)).not.toContain('req_test') + expect(JSON.stringify(error)).not.toContain('model_not_found') + }) + + it('identifies the actual Azure transport and deployment selected for a catalog model', async () => { + setEnv({ + AZURE_OPENAI_API_KEY: 'private-azure-key', + AZURE_OPENAI_ENDPOINT: 'https://azure.example', + AZURE_OPENAI_API_VERSION: '2024-02-01', + KB_OPENAI_MODEL_NAME: 'test-embedding-deployment', + }) + fetchMock.mockResolvedValue( + jsonResponse({ error: { code: 'DeploymentNotFound' } }, 404, { + 'apim-request-id': 'azure-request-test', + }) + ) + await expect(embed(['text'], options)).rejects.toThrow('Embedding API failed: 404') + expect(mockDiagnosticWarn).toHaveBeenCalledWith( + 'Embedding provider request failed', + expect.objectContaining({ + providerId: 'azure-openai', + modelName: 'test-embedding-deployment', + providerRequestId: 'azure-request-test', + providerErrorCode: 'DeploymentNotFound', + }) + ) + expect(JSON.stringify(mockDiagnosticWarn.mock.calls)).not.toContain('private-azure-key') + expect(JSON.stringify(mockDiagnosticWarn.mock.calls)).not.toContain('https://azure.example') + expect(fetchMock).toHaveBeenCalledTimes(1) + }) + + it('keeps the HTTP failure diagnosable when its response body cannot be read', async () => { + fetchMock.mockImplementation( + async () => + new Response( + new ReadableStream({ + start(controller) { + controller.error(new Error('private body failure')) + }, + }), + { status: 404, headers: { 'x-request-id': 'req_unreadable' } } + ) + ) + await expect(embed(['text'], { ...options, apiKey: 'key' })).rejects.toThrow( + 'Embedding API failed: 404' + ) + expect(mockDiagnosticWarn).toHaveBeenCalledWith( + 'Embedding provider request failed', + expect.objectContaining({ + status: 404, + providerRequestId: 'req_unreadable', + bodyFormat: 'unavailable', + }) + ) + expect(fetchMock).toHaveBeenCalledTimes(1) + expect(JSON.stringify(mockDiagnosticWarn.mock.calls)).not.toContain('private') + }) + + it('logs quota rejection before the existing quota circuit wraps the HTTP error', async () => { + fetchMock.mockResolvedValue(jsonResponse({ error: { code: 'insufficient_quota' } }, 429)) + await expect(embed(['text'], { ...options, apiKey: 'key' })).rejects.toBeInstanceOf( + EmbeddingQuotaExhaustedError + ) + expect(mockDiagnosticWarn).toHaveBeenCalledWith( + 'Embedding provider request failed', + expect.objectContaining({ status: 429, providerErrorCode: 'insufficient_quota' }) + ) + expect(fetchMock).toHaveBeenCalledTimes(1) + }) +}) + describe('embedding cancellation', () => { it('cancels a stalled response body after headers arrive without retrying', async () => { vi.useFakeTimers() diff --git a/apps/sim/lib/embeddings/client.ts b/apps/sim/lib/embeddings/client.ts index aea273979a7..c13e25cb5db 100644 --- a/apps/sim/lib/embeddings/client.ts +++ b/apps/sim/lib/embeddings/client.ts @@ -1,6 +1,7 @@ import { createLogger } from '@sim/logger' import { sha256Hex } from '@sim/security/hash' import { chunkArray } from '@sim/utils/helpers' +import { truncate } from '@sim/utils/string' import { getBYOKKey } from '@/lib/api-key/byok' import { getRotatingApiKey } from '@/lib/core/config/api-keys' import { env, envNumber } from '@/lib/core/config/env' @@ -23,6 +24,7 @@ import { readResponseTextWithLimit, } from '@/lib/core/utils/stream-limits' import { getOllamaUrl } from '@/lib/core/utils/urls' +import { EmbeddingAPIError } from '@/lib/embeddings/api-error' import { DEFAULT_EMBEDDING_MODEL, type EmbeddingModelInfo, @@ -31,6 +33,7 @@ import { ollamaEmbeddingModelName, resolveDimensions, } from '@/lib/embeddings/catalog' +import { getEmbeddingResponseDiagnostic } from '@/lib/embeddings/error-diagnostics' import { resolveProviderKey } from '@/lib/embeddings/keys' import { isOllamaServerConfigured } from '@/lib/embeddings/ollama-model-catalog.server' import { DEFAULT_OPENROUTER_EMBEDDING_MODEL } from '@/lib/embeddings/openrouter-models' @@ -168,29 +171,6 @@ export const EMBEDDING_RETRY_BUDGET_MS = EMBEDDING_MAX_RETRIES * EMBEDDING_MAX_R */ export const KNOWLEDGE_EMBEDDING_ADMISSION_WAIT_MS = 60_000 -export class EmbeddingAPIError extends Error { - public status: number - - /** True when the rejected request used a customer-managed credential. */ - public readonly isBYOK: boolean - - /** Rejected for an exhausted balance rather than a recoverable rate limit. */ - public quotaExhausted?: boolean - - /** - * Wait the provider asked for, read from the rejected response. Consumed by - * {@link retryWithExponentialBackoff}, which prefers it over its own backoff. - */ - public retryAfterMs?: number - - constructor(message: string, status: number, isBYOK = false) { - super(message) - this.name = 'EmbeddingAPIError' - this.status = status - this.isBYOK = isBYOK - } -} - class EmbeddingResponseValidationError extends EmbeddingAPIError { constructor(message: string) { super(`Embedding API returned an invalid success response: ${message}`, 502) @@ -293,7 +273,7 @@ function isQuotaExhaustionBody(errorText: string): boolean { } } -/** Reads a bounded provider body only for internal quota classification. */ +/** Reads a bounded provider body for internal diagnostics and quota classification. */ async function readEmbeddingErrorBody(response: Response, signal?: AbortSignal): Promise { try { return await readResponseTextWithLimit(response, { @@ -542,6 +522,7 @@ async function callEmbeddingAPI( tokenizerProvider: string, taskType: EmbeddingTaskType, providerId: EmbeddingProviderKind, + modelName: string, quotaCircuitIdentity: EmbeddingQuotaCircuitIdentity, /** * The caller's explicit reduction, or undefined when none was requested. Kept @@ -605,6 +586,12 @@ async function callEmbeddingAPI( if (!response.ok) { const classificationBody = await readEmbeddingErrorBody(response, controller.signal) + logger.warn('Embedding provider request failed', { + providerId, + modelName: truncate(modelName, 256), + status: response.status, + ...getEmbeddingResponseDiagnostic(response.headers, classificationBody), + }) const error = new EmbeddingAPIError( `Embedding API failed: ${response.status}`, response.status, @@ -856,6 +843,7 @@ async function callCheckpointedEmbeddingBatch( provider.info.tokenizerProvider, taskType, provider.providerId, + provider.modelName, provider.quotaCircuitIdentity, requestedDimensions, provider.dimensions, @@ -1081,6 +1069,7 @@ export async function embedOpenRouter( limits.tokenizerProvider, 'document', 'openrouter', + model, quotaCircuitIdentity, options.dimensions, expectedDimensions, diff --git a/apps/sim/lib/embeddings/error-diagnostics.test.ts b/apps/sim/lib/embeddings/error-diagnostics.test.ts new file mode 100644 index 00000000000..c733cd1d2b6 --- /dev/null +++ b/apps/sim/lib/embeddings/error-diagnostics.test.ts @@ -0,0 +1,75 @@ +/** @vitest-environment node */ +import { describe, expect, it } from 'vitest' +import { getEmbeddingResponseDiagnostic } from '@/lib/embeddings/error-diagnostics' + +describe('embedding response diagnostics', () => { + it.each([ + ['model_not_found', 'invalid_request_error'], + ['DeploymentNotFound', undefined], + [404, 'NOT_FOUND'], + ])('retains safe machine codes for %s without free-form provider text', (code, type) => { + const diagnostic = getEmbeddingResponseDiagnostic( + new Headers({ 'x-request-id': 'req_test', authorization: 'Bearer private-key' }), + JSON.stringify({ + error: { code, type, message: 'private document', param: 'private input' }, + }) + ) + expect(diagnostic).toEqual({ + providerRequestId: 'req_test', + bodyFormat: 'json', + providerErrorCode: String(code), + providerErrorType: type ?? null, + }) + expect(JSON.stringify(diagnostic)).not.toContain('private') + }) + + it('reads Gemini status independently of its numeric error code', () => { + expect( + getEmbeddingResponseDiagnostic( + new Headers(), + JSON.stringify({ error: { code: 404, status: 'NOT_FOUND' } }) + ) + ).toMatchObject({ providerErrorCode: '404', providerErrorType: 'NOT_FOUND' }) + }) + + it.each(['apim-request-id', 'x-ms-request-id'])('reads the Azure %s header', (header) => { + expect( + getEmbeddingResponseDiagnostic(new Headers({ [header]: 'request-test' }), '') + ).toMatchObject({ providerRequestId: 'request-test', bodyFormat: 'unavailable' }) + }) + + it.each(['', 'private gateway error', '{"error":', 'null', '[]', '"private"'])( + 'tolerates an unavailable, non-JSON, or unexpected body: %s', + (body) => { + const diagnostic = getEmbeddingResponseDiagnostic(new Headers(), body) + expect(diagnostic.providerErrorCode).toBeNull() + expect(diagnostic.providerErrorType).toBeNull() + expect(JSON.stringify(diagnostic)).not.toContain('private') + } + ) + + it.each([ + 'private-key', + 'private document text', + { private: true }, + ['private'], + 'x'.repeat(1000), + ])('does not copy unknown error fields into logs: %j', (value) => { + expect( + getEmbeddingResponseDiagnostic( + new Headers(), + JSON.stringify({ error: { code: value, type: value } }) + ) + ).toMatchObject({ providerErrorCode: 'unrecognized', providerErrorType: 'unrecognized' }) + }) + + it.each(['Bearer private-key', 'https://private.example', 'x'.repeat(129)])( + 'omits malformed or oversized request IDs: %s', + (requestId) => { + expect( + getEmbeddingResponseDiagnostic(new Headers({ 'x-request-id': requestId }), '') + .providerRequestId + ).toBeNull() + } + ) +}) diff --git a/apps/sim/lib/embeddings/error-diagnostics.ts b/apps/sim/lib/embeddings/error-diagnostics.ts new file mode 100644 index 00000000000..7d0d6422335 --- /dev/null +++ b/apps/sim/lib/embeddings/error-diagnostics.ts @@ -0,0 +1,84 @@ +/** Only recognized machine codes may leave a provider's untrusted response body. */ +const SAFE_ERROR_CODES = new Set([ + 'model_not_found', + 'invalid_api_key', + 'invalid_request_error', + 'authentication_error', + 'permission_error', + 'rate_limit_error', + 'rate_limit_exceeded', + 'insufficient_quota', + 'context_length_exceeded', + 'server_error', + 'DeploymentNotFound', + 'ResourceNotFound', + 'OperationNotSupported', + 'InvalidRequest', + 'Unauthorized', + 'Forbidden', + 'TooManyRequests', + 'InternalServerError', + 'ServiceUnavailable', + 'INVALID_ARGUMENT', + 'NOT_FOUND', + 'PERMISSION_DENIED', + 'UNAUTHENTICATED', + 'RESOURCE_EXHAUSTED', + 'INTERNAL', + 'UNAVAILABLE', +]) + +function safeErrorCode(value: unknown): string | null { + if (value === undefined || value === null) return null + if (typeof value === 'string' && SAFE_ERROR_CODES.has(value)) return value + if (typeof value === 'number' && Number.isInteger(value) && value >= 400 && value <= 599) { + return String(value) + } + return 'unrecognized' +} + +function safeRequestId(value: string | null): string | null { + return value && /^[a-zA-Z0-9_-]{1,128}$/.test(value) ? value : null +} + +interface EmbeddingResponseDiagnostic { + providerRequestId: string | null + bodyFormat: 'json' | 'non_json' | 'unavailable' + providerErrorCode: string | null + providerErrorType: string | null +} + +/** + * Internal log metadata only. Never retain free-form messages, request inputs, + * response bodies, or the full header bag on errors that can reach callers. + */ +export function getEmbeddingResponseDiagnostic( + headers: Headers, + body: string +): EmbeddingResponseDiagnostic { + const diagnostic: EmbeddingResponseDiagnostic = { + providerRequestId: + safeRequestId(headers.get('x-request-id')) ?? + safeRequestId(headers.get('apim-request-id')) ?? + safeRequestId(headers.get('x-ms-request-id')), + bodyFormat: body ? 'non_json' : 'unavailable', + providerErrorCode: null, + providerErrorType: null, + } + if (!body) return diagnostic + let parsed: unknown + try { + parsed = JSON.parse(body) + } catch { + return diagnostic + } + diagnostic.bodyFormat = 'json' + if (!parsed || typeof parsed !== 'object' || !('error' in parsed)) return diagnostic + const error = parsed.error + if (!error || typeof error !== 'object') return diagnostic + diagnostic.providerErrorCode = safeErrorCode('code' in error ? error.code : undefined) + diagnostic.providerErrorType = safeErrorCode( + 'type' in error ? error.type : 'status' in error ? error.status : undefined + ) + return diagnostic +} diff --git a/apps/sim/lib/knowledge/connectors/connector-error.test.ts b/apps/sim/lib/knowledge/connectors/connector-error.test.ts index a94305e22b0..d2645383eef 100644 --- a/apps/sim/lib/knowledge/connectors/connector-error.test.ts +++ b/apps/sim/lib/knowledge/connectors/connector-error.test.ts @@ -1,6 +1,7 @@ /** @vitest-environment node */ import { DrizzleQueryError } from 'drizzle-orm/errors' import { describe, expect, it } from 'vitest' +import { EmbeddingAPIError } from '@/lib/embeddings/api-error' import { getConnectorFailureDiagnostic } from '@/lib/knowledge/connectors/connector-error' import { GoogleDriveApiError, @@ -9,6 +10,20 @@ import { import { ConnectorDirectoryError } from '@/connectors/source-error' describe('connector failure diagnostics', () => { + it.each([401, 403, 404, 429, 502, 503])( + 'does not attribute a wrapped embedding HTTP %s to the source', + (status) => { + const error = new Error('private wrapper', { + cause: new EmbeddingAPIError('private provider details', status), + }) + expect(getConnectorFailureDiagnostic(error)).toEqual({ + category: 'embedding', + status, + message: `Embedding service request failed (HTTP ${status}).`, + }) + } + ) + it('retains the SQLSTATE while discarding SQL, bound values and driver detail', () => { const error = new DrizzleQueryError( 'select private_column from private_source where id = $1', diff --git a/apps/sim/lib/knowledge/connectors/connector-error.ts b/apps/sim/lib/knowledge/connectors/connector-error.ts index e82d9715f2c..3c6b00c02f0 100644 --- a/apps/sim/lib/knowledge/connectors/connector-error.ts +++ b/apps/sim/lib/knowledge/connectors/connector-error.ts @@ -1,5 +1,6 @@ import { findCause, getPostgresErrorCode } from '@sim/utils/errors' import { DrizzleQueryError } from 'drizzle-orm/errors' +import { EmbeddingAPIError } from '@/lib/embeddings/api-error' import { ConnectorDirectoryError, ConnectorSourceError, @@ -8,7 +9,7 @@ import { } from '@/connectors/source-error' export interface ConnectorFailureDiagnostic { - category: 'directory' | 'database' | ConnectorSourceFailureCategory | 'transport' + category: 'directory' | 'database' | 'embedding' | ConnectorSourceFailureCategory | 'transport' message: string status?: number code?: string @@ -77,6 +78,13 @@ function classifyFailure(error: unknown): ConnectorFailureDiagnostic | null { ) if (!httpError) return null const { status } = httpError + if (httpError instanceof EmbeddingAPIError) { + return { + category: 'embedding', + status, + message: `Embedding service request failed (HTTP ${status}).`, + } + } const category = httpError instanceof ConnectorSourceError ? httpError.category : undefined if (category === 'authorization' || (!category && (status === 401 || status === 403))) { return { diff --git a/apps/sim/lib/knowledge/documents/document-processing-source.test.ts b/apps/sim/lib/knowledge/documents/document-processing-source.test.ts index 1be1439f8f8..beb7e61e66a 100644 --- a/apps/sim/lib/knowledge/documents/document-processing-source.test.ts +++ b/apps/sim/lib/knowledge/documents/document-processing-source.test.ts @@ -85,8 +85,9 @@ import { BYOK_EMBEDDING_CREDENTIAL_REJECTION_MESSAGE, EMBEDDING_QUOTA_EXHAUSTED_MESSAGE, } from '@/lib/embeddings' +import { EmbeddingAPIError } from '@/lib/embeddings/api-error' import * as embeddingClient from '@/lib/embeddings/client' -import { EmbeddingAPIError, EmbeddingQuotaExhaustedError } from '@/lib/embeddings/client' +import { EmbeddingQuotaExhaustedError } from '@/lib/embeddings/client' import { SYSTEM_ACCESS_SCOPE } from '@/lib/knowledge/access/types' import { PermanentDocumentProcessingError, diff --git a/apps/sim/lib/knowledge/documents/processing-provider-deferral.test.ts b/apps/sim/lib/knowledge/documents/processing-provider-deferral.test.ts index 49ff2d46e20..57348e4adf7 100644 --- a/apps/sim/lib/knowledge/documents/processing-provider-deferral.test.ts +++ b/apps/sim/lib/knowledge/documents/processing-provider-deferral.test.ts @@ -5,7 +5,8 @@ import { ProviderAdmissionTimeoutError, } from '@/lib/core/rate-limiter/provider-admission' import { ProviderCapacityDeferredError } from '@/lib/core/rate-limiter/provider-capacity-error' -import { EmbeddingAPIError, EmbeddingQuotaExhaustedError } from '@/lib/embeddings/client' +import { EmbeddingAPIError } from '@/lib/embeddings/api-error' +import { EmbeddingQuotaExhaustedError } from '@/lib/embeddings/client' import { OcrRequestRejectedError, PermanentDocumentProcessingError, From 17c18a15f1741ecf11bbcfb75afcbebd61caab76 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Thu, 17 Sep 2026 14:32:19 -0700 Subject: [PATCH 2/2] fix(embeddings): classify aggregated batch failures --- apps/sim/lib/embeddings/api-error.ts | 15 ++++++++ .../connectors/connector-error.test.ts | 36 +++++++++++++++++++ .../knowledge/connectors/connector-error.ts | 22 +++++++----- 3 files changed, 65 insertions(+), 8 deletions(-) diff --git a/apps/sim/lib/embeddings/api-error.ts b/apps/sim/lib/embeddings/api-error.ts index aa7cece606a..ff447db3b32 100644 --- a/apps/sim/lib/embeddings/api-error.ts +++ b/apps/sim/lib/embeddings/api-error.ts @@ -20,3 +20,18 @@ export class EmbeddingAPIError extends Error { this.isBYOK = isBYOK } } + +/** Finds an embedding failure through bounded aggregate/cause wrappers. */ +export function getEmbeddingAPIError(error: unknown): EmbeddingAPIError | null { + const pending = [error] + const seen = new Set() + while (pending.length > 0 && seen.size < 32) { + const current = pending.pop() + if (!(current instanceof Error) || seen.has(current)) continue + seen.add(current) + if (current instanceof EmbeddingAPIError) return current + if (current.cause !== undefined) pending.push(current.cause) + if (current instanceof AggregateError) pending.push(...current.errors.slice(0, 32)) + } + return null +} diff --git a/apps/sim/lib/knowledge/connectors/connector-error.test.ts b/apps/sim/lib/knowledge/connectors/connector-error.test.ts index d2645383eef..e75747a877a 100644 --- a/apps/sim/lib/knowledge/connectors/connector-error.test.ts +++ b/apps/sim/lib/knowledge/connectors/connector-error.test.ts @@ -10,6 +10,42 @@ import { import { ConnectorDirectoryError } from '@/connectors/source-error' describe('connector failure diagnostics', () => { + it('finds embedding failures through concurrent batches and nested cause wrappers', () => { + const error = new Error('private outer wrapper', { + cause: new AggregateError([ + new EmbeddingAPIError('private upstream message', 503), + new Error('private inner wrapper', { + cause: new AggregateError([new EmbeddingAPIError('private upstream message', 503)]), + }), + new Error('private sibling'), + ]), + }) + expect(getConnectorFailureDiagnostic(error)).toEqual({ + category: 'embedding', + status: 503, + message: 'Embedding service request failed (HTTP 503).', + }) + }) + + it('terminates cyclic aggregate wrappers and still finds an embedding sibling', () => { + const error = new AggregateError([]) + error.errors.push(new EmbeddingAPIError('private upstream message', 429), error) + error.cause = error + expect(getConnectorFailureDiagnostic(error)).toMatchObject({ + category: 'embedding', + status: 429, + }) + const cycle = new AggregateError([]) + cycle.errors.push(cycle) + expect(getConnectorFailureDiagnostic(cycle)).toBeNull() + }) + + it('bounds traversal of deeply nested aggregate wrappers', () => { + let error: Error = new EmbeddingAPIError('private upstream message', 503) + for (let i = 0; i < 40; i++) error = new AggregateError([error]) + expect(getConnectorFailureDiagnostic(error)).toBeNull() + }) + it.each([401, 403, 404, 429, 502, 503])( 'does not attribute a wrapped embedding HTTP %s to the source', (status) => { diff --git a/apps/sim/lib/knowledge/connectors/connector-error.ts b/apps/sim/lib/knowledge/connectors/connector-error.ts index 3c6b00c02f0..b244bd808d9 100644 --- a/apps/sim/lib/knowledge/connectors/connector-error.ts +++ b/apps/sim/lib/knowledge/connectors/connector-error.ts @@ -1,6 +1,6 @@ import { findCause, getPostgresErrorCode } from '@sim/utils/errors' import { DrizzleQueryError } from 'drizzle-orm/errors' -import { EmbeddingAPIError } from '@/lib/embeddings/api-error' +import { getEmbeddingAPIError } from '@/lib/embeddings/api-error' import { ConnectorDirectoryError, ConnectorSourceError, @@ -66,6 +66,19 @@ function classifyFailure(error: unknown): ConnectorFailureDiagnostic | null { if (databaseError) { return { category: 'database', message: 'Database request failed without a driver error code.' } } + const embeddingError = getEmbeddingAPIError(error) + if ( + embeddingError && + Number.isInteger(embeddingError.status) && + embeddingError.status >= 400 && + embeddingError.status <= 599 + ) { + return { + category: 'embedding', + status: embeddingError.status, + message: `Embedding service request failed (HTTP ${embeddingError.status}).`, + } + } const httpError = findCause( error, (value): value is Error & { status: number } => @@ -78,13 +91,6 @@ function classifyFailure(error: unknown): ConnectorFailureDiagnostic | null { ) if (!httpError) return null const { status } = httpError - if (httpError instanceof EmbeddingAPIError) { - return { - category: 'embedding', - status, - message: `Embedding service request failed (HTTP ${status}).`, - } - } const category = httpError instanceof ConnectorSourceError ? httpError.category : undefined if (category === 'authorization' || (!category && (status === 401 || status === 403))) { return {