From 6bfc5a5b6519b54c6518a12f778725f1f9281838 Mon Sep 17 00:00:00 2001 From: Vikhyath Mondreti Date: Thu, 17 Sep 2026 12:25:37 -0700 Subject: [PATCH 1/2] improvement(provenance): enforce tracked durable reads --- .../self-hosting/environment-variables.mdx | 3 +- .../knowledge/search/route.provenance.test.ts | 76 ++---- .../executor/handlers/agent/memory.test.ts | 50 +--- apps/sim/executor/handlers/agent/memory.ts | 58 +---- apps/sim/lib/copilot/tools/handlers/vfs.ts | 1 - apps/sim/lib/core/config/env.ts | 1 - ...able-secret-provenance-enforcement.test.ts | 202 --------------- .../durable-secret-provenance-enforcement.ts | 237 ------------------ ...urable-secret-provenance-telemetry.test.ts | 58 +++++ .../durable-secret-provenance-telemetry.ts | 67 +++++ .../durable-secret-provenance.test.ts | 46 +--- .../execution/durable-secret-provenance.ts | 31 +-- .../lib/function-execution/execute-request.ts | 1 - .../file/operations.provenance.test.ts | 78 +++--- apps/sim/lib/internal/file/operations.ts | 2 - apps/sim/lib/internal/memory/provenance.ts | 4 +- .../external-file-provenance.integration.ts | 3 - .../upload-read-provenance.integration.ts | 2 - .../lib/knowledge/api/secret-provenance.ts | 2 - .../application/add-workspace-files.ts | 2 +- apps/sim/lib/knowledge/application/chunks.ts | 2 +- .../application/read-indexed-document.ts | 9 +- .../application/read-search-document.ts | 10 +- apps/sim/lib/knowledge/application/search.ts | 34 +-- .../lib/knowledge/secret-provenance.test.ts | 58 ++--- apps/sim/lib/knowledge/secret-provenance.ts | 74 +----- .../lib/memory/application/use-cases.test.ts | 14 -- apps/sim/lib/memory/application/use-cases.ts | 20 +- .../message-provenance.postgres.test.ts | 15 +- .../sim/lib/memory/message-provenance.test.ts | 14 +- apps/sim/lib/memory/secret-provenance.ts | 2 +- .../rows/secret-provenance.postgres.test.ts | 78 ++---- .../lib/table/rows/secret-provenance.test.ts | 70 ++---- apps/sim/lib/table/rows/secret-provenance.ts | 37 +-- .../workspace-file-secret-provenance.test.ts | 131 ++-------- .../workspace-file-secret-provenance.ts | 112 ++------- 36 files changed, 346 insertions(+), 1258 deletions(-) delete mode 100644 apps/sim/lib/execution/durable-secret-provenance-enforcement.test.ts delete mode 100644 apps/sim/lib/execution/durable-secret-provenance-enforcement.ts create mode 100644 apps/sim/lib/execution/durable-secret-provenance-telemetry.test.ts create mode 100644 apps/sim/lib/execution/durable-secret-provenance-telemetry.ts diff --git a/apps/docs/content/docs/platform/self-hosting/environment-variables.mdx b/apps/docs/content/docs/platform/self-hosting/environment-variables.mdx index f97b6f246df..d870ef0e354 100644 --- a/apps/docs/content/docs/platform/self-hosting/environment-variables.mdx +++ b/apps/docs/content/docs/platform/self-hosting/environment-variables.mdx @@ -234,9 +234,10 @@ width. Check what you pick there, or upgrade Ollama. | `COPILOT_API_KEY` | API key for Chat. Without it the Sim Chat block, scheduled prompt jobs, and Inbox cannot run | | `NEXT_PUBLIC_CHAT_DISABLED` | Set to `true` to hide the Chat module: the workspace lands on your first workflow, with no chats list, scheduled tasks, or editor Chat panel. Chat is shown when unset; `npx sim-setup` sets it for you if you skip the chat key | | `PII_URL` | Base URL of the Presidio service backing PII detection and redaction. The Helm chart wires it to its own `pii` Service when `pii.enabled`; on Compose point it at the PII service on your network. The default `http://localhost:5001` exists only in local development, and leaving it makes redaction fail | -| `DURABLE_SECRET_PROVENANCE_ENFORCED_SURFACES` | Durable stores where a value whose secret provenance was never recorded fails the run instead of logging a warning. `all`, or a comma-separated subset of `memory`, `table-row`, `knowledge`, `workspace-file`. Unset (nothing enforced) by default | | `ADMIN_API_KEY` | Admin API key for GitOps operations and organization provisioning | +Tracked memory, table rows, knowledge content, and workspace files require valid secret provenance before entering a model or a trusted runtime. Records with a null provenance tracking marker retain legacy compatibility. + ## Enterprise Features Enterprise features are unlocked by configuration rather than billing on self-hosted deployments. One switch turns on the full set; per-feature flags below it override the switch either way. diff --git a/apps/sim/app/api/v2/knowledge/search/route.provenance.test.ts b/apps/sim/app/api/v2/knowledge/search/route.provenance.test.ts index 7e847cb2797..f295359c749 100644 --- a/apps/sim/app/api/v2/knowledge/search/route.provenance.test.ts +++ b/apps/sim/app/api/v2/knowledge/search/route.provenance.test.ts @@ -107,10 +107,6 @@ import { } from '@sim/testing' import { NextRequest } from 'next/server' import { env } from '@/lib/core/config/env' -import { - isDurableSecretProvenanceEnforced, - resetDurableSecretProvenanceEnforcementCache, -} from '@/lib/execution/durable-secret-provenance-enforcement' import { createKnowledgeDocumentSourceValue } from '@/lib/knowledge/secret-provenance' import { POST } from '@/app/api/v2/knowledge/search/route' import { ResolvedSecretTraceRegistry } from '@/executor/utils/resolved-secret-trace-registry' @@ -211,12 +207,6 @@ function providerPayload() { return JSON.parse(provider.fetch.mock.calls[0][1].body) } -function enforceKnowledge(enforced: boolean) { - env.DURABLE_SECRET_PROVENANCE_ENFORCED_SURFACES = enforced ? 'all' : '' - resetDurableSecretProvenanceEnforcementCache() - expect(isDurableSecretProvenanceEnforced('knowledge')).toBe(enforced) -} - async function requestSearch(overrides: Partial = {}) { return POST( new NextRequest('http://localhost/api/v2/knowledge/search', { @@ -230,7 +220,6 @@ async function requestSearch(overrides: Partial = {}) { beforeEach(() => { vi.clearAllMocks() resetDbChainMock() - enforceKnowledge(true) env.COHERE_API_KEY = 'synthetic-cohere-key' provider.decrypt.mockResolvedValue({ decrypted: SECRET }) provider.fetch.mockResolvedValue( @@ -264,26 +253,22 @@ beforeEach(() => { /** The route, use case, sidecar binding/import, registry, projection and provider request builder are real. */ describe('Knowledge search provenance through the V2 route and reranker HTTP boundary', () => { - it.each([false, true])( - 'redacts current known-secret chunks with enforcement=%s', - async (enforced) => { - enforceKnowledge(enforced) - seedSidecar('exact') - const response = await requestSearch() - const body = await response.json() - expect(response.status).toBe(200) - expect(body.data.rerankerStatus).toBe('applied') - expect(providerPayload().documents).toEqual([CONTENT.replace(SECRET, '{{TOKEN}}')]) - expect(body.data.results[0].content).toBe(CONTENT) - expect(provider.decrypt).toHaveBeenCalledWith('synthetic-encrypted-token') - expect(mocks.generateEmbedding).toHaveBeenCalledWith( - requestInput.query, - expect.anything(), - 'workspace-1', - undefined - ) - } - ) + it('redacts current known-secret chunks', async () => { + seedSidecar('exact') + const response = await requestSearch() + const body = await response.json() + expect(response.status).toBe(200) + expect(body.data.rerankerStatus).toBe('applied') + expect(providerPayload().documents).toEqual([CONTENT.replace(SECRET, '{{TOKEN}}')]) + expect(body.data.results[0].content).toBe(CONTENT) + expect(provider.decrypt).toHaveBeenCalledWith('synthetic-encrypted-token') + expect(mocks.generateEmbedding).toHaveBeenCalledWith( + requestInput.query, + expect.anything(), + 'workspace-1', + undefined + ) + }) it('does not assign a billing owner secret name to a workspace-key caller', async () => { seedSidecar('exact') @@ -314,7 +299,7 @@ describe('Knowledge search provenance through the V2 route and reranker HTTP bou }) it.each(['unknown', 'missing', 'stale', 'malformed'] as const)( - 'refuses %s tracked provenance before provider HTTP when enforcement is enabled', + 'refuses %s tracked provenance before provider HTTP', async (status) => { seedSidecar(status) const response = await requestSearch() @@ -326,27 +311,12 @@ describe('Knowledge search provenance through the V2 route and reranker HTTP bou } ) - it.each(['unknown', 'missing', 'stale', 'malformed'] as const)( - 'preserves existing flag-off compatibility for %s sidecars', - async (status) => { - enforceKnowledge(false) - seedSidecar(status) - const response = await requestSearch() - expect(response.status).toBe(200) - expect(providerPayload().documents).toEqual([CONTENT]) - } - ) - - it.each([false, true])( - 'keeps pre-tracking NULL rows readable with enforcement=%s', - async (enforced) => { - enforceKnowledge(enforced) - seedSidecar('legacy') - const response = await requestSearch() - expect(response.status).toBe(200) - expect(providerPayload().documents).toEqual([CONTENT]) - } - ) + it('keeps pre-tracking NULL rows readable', async () => { + seedSidecar('legacy') + const response = await requestSearch() + expect(response.status).toBe(200) + expect(providerPayload().documents).toEqual([CONTENT]) + }) it('does not subject a raw public read without reranking to durable-model enforcement', async () => { seedSidecar('unknown') diff --git a/apps/sim/executor/handlers/agent/memory.test.ts b/apps/sim/executor/handlers/agent/memory.test.ts index 96f629e62ba..497958954db 100644 --- a/apps/sim/executor/handlers/agent/memory.test.ts +++ b/apps/sim/executor/handlers/agent/memory.test.ts @@ -1,18 +1,9 @@ import { loggerMock, queueTableRows, resetDbChainMock, schemaMock } from '@sim/testing' import { beforeEach, describe, expect, it, vi } from 'vitest' -const { mockDecryptSecret, mockRedactObjectStrings, mockIsEnforced, mockReportUnrecorded } = - vi.hoisted(() => ({ - mockDecryptSecret: vi.fn(), - mockRedactObjectStrings: vi.fn(async (value: unknown) => value), - mockIsEnforced: vi.fn(() => false), - mockReportUnrecorded: vi.fn(), - })) - -vi.mock('@/lib/execution/durable-secret-provenance-enforcement', () => ({ - DURABLE_SECRET_PROVENANCE_SURFACES: ['memory', 'table-row', 'knowledge'], - isDurableSecretProvenanceEnforced: mockIsEnforced, - reportUnrecordedDurableProvenance: mockReportUnrecorded, +const { mockDecryptSecret, mockRedactObjectStrings } = vi.hoisted(() => ({ + mockDecryptSecret: vi.fn(), + mockRedactObjectStrings: vi.fn(async (value: unknown) => value), })) vi.mock('@/lib/core/security/encryption', () => ({ @@ -56,7 +47,6 @@ describe('Memory', () => { beforeEach(() => { vi.clearAllMocks() resetDbChainMock() - mockIsEnforced.mockReturnValue(false) mockDecryptSecret.mockImplementation(async (encryptedValue: string) => ({ decrypted: `decrypted:${encryptedValue}`, })) @@ -313,6 +303,7 @@ describe('Memory', () => { async ({ provider, render }) => { queueTableRows(schemaMock.memory, [ { + secretProvenanceVersion: null, data: [ { role: 'user', @@ -359,7 +350,7 @@ describe('Memory', () => { context: 'execution', } queueTableRows(schemaMock.memory, [ - { data: [{ role: 'user', content: 'File', files: [file] }] }, + { secretProvenanceVersion: null, data: [{ role: 'user', content: 'File', files: [file] }] }, ]) const context = { workspaceId: 'workspace-1', @@ -388,6 +379,7 @@ describe('Memory', () => { it('bounds historical file loading even when the messages contain no text', async () => { queueTableRows(schemaMock.memory, [ { + secretProvenanceVersion: null, data: Array.from({ length: MEMORY.MAX_REPLAY_FILE_REFERENCES + 1 }, () => ({ role: 'user', content: '', @@ -406,6 +398,7 @@ describe('Memory', () => { it('does not carry inline-only or malformed file objects into a later turn', async () => { queueTableRows(schemaMock.memory, [ { + secretProvenanceVersion: null, data: [ { role: 'user', @@ -695,33 +688,7 @@ describe('Memory', () => { expect(mockDecryptSecret).not.toHaveBeenCalled() }) - /** Trace 2's shape: a stored memory a previous run could not vouch for. */ - it('reads a memory with unrecorded provenance while the surface stays open', async () => { - const registry = new ResolvedSecretTraceRegistry([], { - userId: 'user-1', - workspaceId: 'workspace-1', - }) - vi.spyOn(memoryService as any, 'fetchMemory').mockResolvedValueOnce({ - messages: [{ role: 'user', content: 'how do i see my tickets?' }], - provenance: { status: 'unknown' }, - }) - - const messages = await memoryService.fetchMemoryMessages( - createContext(registry) as never, - inputs - ) - - expect(messages).toEqual([{ role: 'user', content: 'how do i see my tickets?' }]) - expect(registry.isPermanentlyIncomplete()).toBe(false) - expect(mockReportUnrecorded).toHaveBeenCalledWith({ - surface: 'memory', - cause: 'stored-memory-provenance-unknown', - workspaceId: 'workspace-1', - }) - }) - - it('refuses that same memory once the memory surface is closed', async () => { - mockIsEnforced.mockReturnValue(true) + it('refuses tracked memory with unknown provenance', async () => { const registry = new ResolvedSecretTraceRegistry([], { userId: 'user-1', workspaceId: 'workspace-1', @@ -734,7 +701,6 @@ describe('Memory', () => { await expect( memoryService.fetchMemoryMessages(createContext(registry) as never, inputs) ).rejects.toThrow() - expect(mockReportUnrecorded).not.toHaveBeenCalled() }) }) diff --git a/apps/sim/executor/handlers/agent/memory.ts b/apps/sim/executor/handlers/agent/memory.ts index 2da2bbec8d9..a55c02bc725 100644 --- a/apps/sim/executor/handlers/agent/memory.ts +++ b/apps/sim/executor/handlers/agent/memory.ts @@ -11,10 +11,6 @@ import { importDurableSecretProvenance, mergeDurableSecretProvenance, } from '@/lib/execution/durable-secret-provenance' -import { - isDurableSecretProvenanceEnforced, - reportUnrecordedDurableProvenance, -} from '@/lib/execution/durable-secret-provenance-enforcement' import { mergeFileKeys } from '@/lib/execution/payloads/access-keys' import { redactObjectStrings } from '@/lib/logs/execution/pii-redaction' import { lockMemoryConversationInTx } from '@/lib/memory/locks' @@ -101,12 +97,8 @@ export class Memory { const staged = new ResolvedSecretTraceRegistry([], scope, { staged: true }) staged.mergeToolCallRegistry(ctx.resolvedSecretTraceRegistry) includeRecovered = - (await importDurableSecretProvenance( - staged, - selection.select(messages, true), - messages, - 'memory' - )) && staged.getModelEgressSnapshot().complete + (await importDurableSecretProvenance(staged, selection.select(messages, true), messages)) && + staged.getModelEgressSnapshot().complete if (includeRecovered) { for (const message of messages) { const stagedMessage = new ResolvedSecretTraceRegistry([], scope, { staged: true }) @@ -114,8 +106,7 @@ export class Memory { !(await importDurableSecretProvenance( stagedMessage, selection.select([message], true), - message, - 'memory' + message )) || !stagedMessage.getModelEgressSnapshot().complete ) { @@ -146,31 +137,15 @@ export class Memory { const selectProvenance = (values: readonly unknown[]) => selection.select(values, includeRecovered) const selectedProvenance = selectProvenance(messages) - /** - * Unrecorded provenance is checked through the same policy the shared import uses, so stored - * memory written by a run that could not vouch does not permanently refuse every later turn. - */ - let refuseStoredProvenance: boolean - if (selectedProvenance.status === 'unknown') { - refuseStoredProvenance = isDurableSecretProvenanceEnforced('memory') - if (!refuseStoredProvenance) { - reportUnrecordedDurableProvenance({ - surface: 'memory', - cause: 'stored-memory-provenance-unknown', - ...(ctx.workspaceId ? { workspaceId: ctx.workspaceId } : {}), - }) - } - } else { - refuseStoredProvenance = - (selectedProvenance.entries.length > 0 && !ctx.resolvedSecretTraceRegistry) || - (ctx.resolvedSecretTraceRegistry !== undefined && - !(await importDurableSecretProvenance( - ctx.resolvedSecretTraceRegistry, - selectedProvenance, - messages, - 'memory' - ))) - } + const refuseStoredProvenance = + selectedProvenance.status === 'unknown' || + (selectedProvenance.entries.length > 0 && !ctx.resolvedSecretTraceRegistry) || + (ctx.resolvedSecretTraceRegistry !== undefined && + !(await importDurableSecretProvenance( + ctx.resolvedSecretTraceRegistry, + selectedProvenance, + messages + ))) if (refuseStoredProvenance) { refuseResolvedSecretProjection({ site: 'memory.storedProvenanceImport', @@ -187,14 +162,7 @@ export class Memory { [], ctx.resolvedSecretTraceRegistry?.exportProvenance().scope ) - if ( - !(await importDurableSecretProvenance( - modelRegistry, - messageProvenance, - message, - 'memory' - )) - ) { + if (!(await importDurableSecretProvenance(modelRegistry, messageProvenance, message))) { refuseResolvedSecretProjection({ site: 'memory.messageProvenanceImport', message: MEMORY_CONTENT_REFUSAL, diff --git a/apps/sim/lib/copilot/tools/handlers/vfs.ts b/apps/sim/lib/copilot/tools/handlers/vfs.ts index 91cdab45981..2f71288ebba 100644 --- a/apps/sim/lib/copilot/tools/handlers/vfs.ts +++ b/apps/sim/lib/copilot/tools/handlers/vfs.ts @@ -130,7 +130,6 @@ async function canReturnWorkspaceFileValue( registry: context.resolvedSecretTraceRegistry, view: provenanceView, value, - actorUserId: context.userId, })) ) { return false diff --git a/apps/sim/lib/core/config/env.ts b/apps/sim/lib/core/config/env.ts index 2fcfd17badb..0b45247cd8b 100644 --- a/apps/sim/lib/core/config/env.ts +++ b/apps/sim/lib/core/config/env.ts @@ -189,7 +189,6 @@ export const env = createEnv({ BILLING_CONCURRENCY_LIMIT_ENTERPRISE: z.string().optional(), // In-flight executions per Enterprise billing account (metadata-overridable) BILLING_ENABLED: z.boolean().optional(), // Enable billing enforcement and usage tracking TRIGGER_EU_REGION: z.boolean().optional(), // Route Trigger.dev runs to eu-central-1 instead of the default us-east-1 (fallback for the trigger-eu-region flag when AppConfig is not the source of truth) - DURABLE_SECRET_PROVENANCE_ENFORCED_SURFACES: z.string().optional(), // Durable surfaces where unrecorded secret provenance fails the run instead of logging a warning: "all", or a comma-separated subset of memory,table-row,knowledge,workspace-file (default: none enforced) // Table feature limits (per plan). Apply when billing is disabled (free tier defaults) or for billed plans. FREE_TABLES_LIMIT: z.number().optional(), // Max user tables per workspace on free tier (default: 5) diff --git a/apps/sim/lib/execution/durable-secret-provenance-enforcement.test.ts b/apps/sim/lib/execution/durable-secret-provenance-enforcement.test.ts deleted file mode 100644 index e4b1c092356..00000000000 --- a/apps/sim/lib/execution/durable-secret-provenance-enforcement.test.ts +++ /dev/null @@ -1,202 +0,0 @@ -/** - * @vitest-environment node - */ -import { beforeEach, describe, expect, it, vi } from 'vitest' - -const { mockEnv, mockLogger, mockPersistenceLogger, mockRecordAudit } = vi.hoisted(() => ({ - mockEnv: {} as Record, - mockLogger: { info: vi.fn(), warn: vi.fn(), error: vi.fn(), debug: vi.fn() }, - mockPersistenceLogger: { error: vi.fn() }, - mockRecordAudit: vi.fn(), -})) - -vi.mock('@/lib/core/config/env', () => ({ env: mockEnv })) -vi.mock('@sim/logger', () => ({ - createLogger: (module: string) => - module === 'DurableSecretProvenancePersistence' ? mockPersistenceLogger : mockLogger, -})) -/** Literal values rather than the real constants: these reach the database and the trail. */ -vi.mock('@sim/audit', () => ({ - recordAudit: mockRecordAudit, - AuditAction: { SECRET_PROVENANCE_UNRECORDED: 'secret_provenance.unrecorded' }, - AuditResourceType: { SECRET_PROVENANCE: 'secret_provenance' }, -})) - -import { - DURABLE_SECRET_PROVENANCE_SURFACES, - isDurableSecretProvenanceEnforced, - reportDurableSecretProvenanceRefusal, - reportDurableSecretProvenanceWrite, - reportUnrecordedDurableProvenance, - resetDurableSecretProvenanceEnforcementCache, -} from '@/lib/execution/durable-secret-provenance-enforcement' - -function configure(value: string | undefined): void { - mockEnv.DURABLE_SECRET_PROVENANCE_ENFORCED_SURFACES = value - resetDurableSecretProvenanceEnforcementCache() -} - -describe('durable secret provenance enforcement', () => { - beforeEach(() => { - vi.clearAllMocks() - configure(undefined) - }) - - it('enforces nothing by default, so unrecorded provenance warns instead of latching', () => { - for (const surface of DURABLE_SECRET_PROVENANCE_SURFACES) { - expect(isDurableSecretProvenanceEnforced(surface)).toBe(false) - } - }) - - it('closes one surface at a time without touching the others', () => { - configure('table-row') - - expect(isDurableSecretProvenanceEnforced('table-row')).toBe(true) - expect(isDurableSecretProvenanceEnforced('memory')).toBe(false) - expect(isDurableSecretProvenanceEnforced('knowledge')).toBe(false) - }) - - it('accepts a comma-separated subset, ignoring case and padding', () => { - configure(' Memory , TABLE-ROW ') - - expect(isDurableSecretProvenanceEnforced('memory')).toBe(true) - expect(isDurableSecretProvenanceEnforced('table-row')).toBe(true) - expect(isDurableSecretProvenanceEnforced('knowledge')).toBe(false) - }) - - it('closes every surface on "all"', () => { - configure('all') - - for (const surface of DURABLE_SECRET_PROVENANCE_SURFACES) { - expect(isDurableSecretProvenanceEnforced(surface)).toBe(true) - } - }) - - it('reports an unrecognized surface rather than silently enforcing nothing', () => { - configure('memory,not-a-surface') - - expect(isDurableSecretProvenanceEnforced('memory')).toBe(true) - expect(mockLogger.error).toHaveBeenCalledWith( - 'Ignoring unrecognized durable secret provenance surfaces', - expect.objectContaining({ unrecognized: ['not-a-surface'] }) - ) - }) - - /** - * `workspace-file` was this test's example of an unrecognized name while the env documentation - * already offered it — so anyone who set it got a silent no-op and the fail-closed posture they - * were trying to change. It is a real surface now. - */ - it('recognizes every surface its own configuration documents', () => { - configure('workspace-file') - - expect(isDurableSecretProvenanceEnforced('workspace-file')).toBe(true) - expect(isDurableSecretProvenanceEnforced('table-row')).toBe(false) - expect(mockLogger.error).not.toHaveBeenCalled() - }) - - it('reports at error with the surface, cause, and affected count so it survives every LOG_LEVEL default', () => { - reportUnrecordedDurableProvenance({ - surface: 'table-row', - cause: 'row-sidecar-not-exact', - affectedCount: 8, - workspaceId: 'workspace-1', - }) - - expect(mockLogger.warn).not.toHaveBeenCalled() - expect(mockLogger.error).toHaveBeenCalledWith( - 'Proceeding on unrecorded durable secret provenance', - { - surface: 'table-row', - cause: 'row-sidecar-not-exact', - enforced: false, - affectedCount: 8, - workspaceId: 'workspace-1', - } - ) - }) - it('records a workspace-visible audit entry so a fail-open read is not only in our logs', () => { - reportUnrecordedDurableProvenance({ - surface: 'table-row', - cause: 'row-sidecar-not-exact', - affectedCount: 8, - workspaceId: 'workspace-1', - actorUserId: 'user-1', - }) - - expect(mockRecordAudit).toHaveBeenCalledWith( - expect.objectContaining({ - workspaceId: 'workspace-1', - actorId: 'user-1', - action: 'secret_provenance.unrecorded', - resourceType: 'secret_provenance', - resourceId: 'table-row', - metadata: { surface: 'table-row', cause: 'row-sidecar-not-exact', affectedCount: 8 }, - }) - ) - }) - - it('still records the entry when the surface cannot name an actor', () => { - reportUnrecordedDurableProvenance({ - surface: 'memory', - cause: 'durable-provenance-unknown', - workspaceId: 'workspace-1', - }) - - expect(mockRecordAudit).toHaveBeenCalledWith(expect.objectContaining({ actorId: null })) - }) - - /** An entry with no workspace names nobody it concerns; the log line still carries it. */ - it('skips the audit entry when there is no workspace to show it to', () => { - reportUnrecordedDurableProvenance({ surface: 'knowledge', cause: 'durable-provenance-unknown' }) - - expect(mockRecordAudit).not.toHaveBeenCalled() - expect(mockLogger.error).toHaveBeenCalled() - }) - - it('separates non-exact write telemetry from permissive reads and copies only safe fields', () => { - const report = { - surface: 'knowledge' as const, - status: 'unknown' as const, - cause: 'source-provenance-unknown' as const, - recordCount: 2, - workspaceId: 'workspace-1', - resourceId: 'document-1', - content: 'private document content', - entries: [{ secretName: 'PRIVATE_TOKEN', ciphertext: 'private encrypted value' }], - } - reportDurableSecretProvenanceWrite(report) - - expect(mockPersistenceLogger.error).toHaveBeenCalledWith( - 'Writing non-exact durable secret provenance', - { - surface: 'knowledge', - status: 'unknown', - cause: 'source-provenance-unknown', - recordCount: 2, - workspaceId: 'workspace-1', - resourceId: 'document-1', - } - ) - expect(mockLogger.error).not.toHaveBeenCalled() - expect(mockRecordAudit).not.toHaveBeenCalled() - expect(isDurableSecretProvenanceEnforced('knowledge')).toBe(false) - }) - - it('reports existing refusals without resolving or changing enforcement', () => { - configure('workspace-file') - reportDurableSecretProvenanceRefusal({ - surface: 'workspace-file', - cause: 'workspace-file-opaque-secret-content', - }) - - expect(mockPersistenceLogger.error).toHaveBeenCalledWith( - 'Refusing unavailable durable secret provenance', - { surface: 'workspace-file', cause: 'workspace-file-opaque-secret-content' } - ) - expect(mockLogger.error).not.toHaveBeenCalled() - expect(mockRecordAudit).not.toHaveBeenCalled() - expect(isDurableSecretProvenanceEnforced('workspace-file')).toBe(true) - expect(isDurableSecretProvenanceEnforced('memory')).toBe(false) - }) -}) diff --git a/apps/sim/lib/execution/durable-secret-provenance-enforcement.ts b/apps/sim/lib/execution/durable-secret-provenance-enforcement.ts deleted file mode 100644 index c8b9c382055..00000000000 --- a/apps/sim/lib/execution/durable-secret-provenance-enforcement.ts +++ /dev/null @@ -1,237 +0,0 @@ -import { AuditAction, AuditResourceType, recordAudit } from '@sim/audit' -import { createLogger } from '@sim/logger' -import { env } from '@/lib/core/config/env' - -const logger = createLogger('DurableSecretProvenanceEnforcement') -const persistenceLogger = createLogger('DurableSecretProvenancePersistence') - -/** - * Durable stores that can hand a run a value whose secret provenance was never recorded. - * - * Named by the call site rather than derived, so the surface survives refactors and stays - * greppable — the same convention the projection-refusal `site` strings use. - * - * One policy governs all of them: an **absence** — nobody recorded what these bytes carry — may be - * read, with an audit entry naming the surface, because a value nobody recorded says exactly what - * an untracked one does. A **taint** — a writer that knew secrets were present and could not map - * them to this output — is refused, and no policy relaxes it. - * - * Only `workspace-file` stores the difference, and that is deliberate rather than drift. On the - * other surfaces every non-exact sidecar comes from one condition, an incomplete incoming bundle - * or registry, which is always an absence; two stored statuses say everything there is to say. - * Files are the only surface that derives one stored object from another — an archive extracted - * into children, a generated asset, a transcoded output — so they are the only one that can refuse - * on purpose, and the only one with two claims to keep apart. Adding a third status elsewhere would - * encode a distinction that surface cannot make; removing it here would collapse one that matters. - */ -export const DURABLE_SECRET_PROVENANCE_SURFACES = [ - 'memory', - 'table-row', - 'knowledge', - 'workspace-file', -] as const - -export type DurableSecretProvenanceSurface = (typeof DURABLE_SECRET_PROVENANCE_SURFACES)[number] - -/** Reads the configured surfaces once; an unrecognized name is reported rather than assumed. */ -function resolveEnforcedSurfaces(): ReadonlySet { - const configured = env.DURABLE_SECRET_PROVENANCE_ENFORCED_SURFACES?.trim() - if (!configured) return new Set() - - const requested = configured - .split(',') - .map((entry) => entry.trim().toLowerCase()) - .filter((entry) => entry.length > 0) - if (requested.includes('all')) return new Set(DURABLE_SECRET_PROVENANCE_SURFACES) - - const enforced = new Set() - const unrecognized: string[] = [] - for (const entry of requested) { - const surface = DURABLE_SECRET_PROVENANCE_SURFACES.find((candidate) => candidate === entry) - if (surface) enforced.add(surface) - else unrecognized.push(entry) - } - if (unrecognized.length > 0) { - logger.error('Ignoring unrecognized durable secret provenance surfaces', { - unrecognized, - supported: [...DURABLE_SECRET_PROVENANCE_SURFACES], - }) - } - return enforced -} - -let enforcedSurfaces: ReadonlySet | undefined - -/** - * True when unrecorded provenance from this surface must fail the run rather than warn. - * - * Nothing is enforced by default. `unknown` provenance means "nobody recorded what secrets this - * value carries", which is the same thing a pre-tracking legacy row says — and legacy rows are read - * as carrying none. Enforcing one and not the other made an *aware* writer that momentarily could - * not vouch strictly worse than an unaware one: the row it wrote latched every run that later read - * it, and each latched run wrote more such rows, so a workspace could not recover without a data - * repair. - * - * Warning instead keeps that state visible and measurable while the writers that produce it are - * fixed. The sidecar still records `unknown` faithfully, so a surface can be closed back up once - * its writers stop losing provenance — and the rows that will start failing are countable from the - * sidecar table before the switch is thrown. This is a deliberate posture: an unenforced surface - * can under-redact a value whose provenance was lost. - * - * Set `DURABLE_SECRET_PROVENANCE_ENFORCED_SURFACES` to `all`, or to a comma-separated subset of - * {@link DURABLE_SECRET_PROVENANCE_SURFACES}, to close a surface. - * - * Workspace files were held out of this list while their fail-closed posture was its own decision. - * That decision arrived as broken state: because a file that was never tracked already reads as - * exact-empty, refusing only the file whose provenance could not be recorded made a writer that - * momentarily could not vouch permanently worse off than one that never tried — with no way back, - * since nothing rewrites a file's provenance but another content write. Live files across several - * workspaces sat unreadable behind it, by every tool route at once, carrying exactly as much - * information about their contents as the untracked files beside them: none. - */ -export function isDurableSecretProvenanceEnforced( - surface: DurableSecretProvenanceSurface -): boolean { - enforcedSurfaces ??= resolveEnforcedSurfaces() - return enforcedSurfaces.has(surface) -} - -/** - * What a surface could not vouch for. - * - * A closed union rather than a free-form string, for the reason the resolved-secret registry's - * reason set is one: a surface stays open on the strength of these lines trending to zero, and a - * cause that a call site can spell freely cannot be aggregated or alerted on. - */ -export type UnrecordedDurableProvenanceCause = - | 'durable-provenance-unknown' - | 'row-sidecar-not-exact' - | 'stored-memory-provenance-unknown' - -export interface UnrecordedDurableProvenanceReport { - surface: DurableSecretProvenanceSurface - cause: UnrecordedDurableProvenanceCause - /** How many records in this one read were unrecorded, when the caller reads a page at a time. */ - affectedCount?: number - workspaceId?: string - /** - * Whose access authorized the read. Null where the surface cannot name one — an audit row with - * no actor still carries the workspace, surface, and cause, which is what the trail is for. - */ - actorUserId?: string | null -} - -/** - * Records that a read proceeded on provenance nobody wrote down. - * - * Error, not warn, for the same reason the originating-fault reasons use it: error is the only - * level that survives every default the logger falls back to — production, test, and a self-hosted - * chart that sets no `LOG_LEVEL`. A surface stays open on the strength of this line being visible - * and trending to zero, so a level that a deployment can silently filter would leave the posture - * unmeasured. It is deliberately noisy on an affected workspace; that is the signal. - */ -export function reportUnrecordedDurableProvenance(report: UnrecordedDurableProvenanceReport): void { - logger.error('Proceeding on unrecorded durable secret provenance', { - surface: report.surface, - cause: report.cause, - enforced: false, - ...(report.affectedCount !== undefined ? { affectedCount: report.affectedCount } : {}), - ...(report.workspaceId ? { workspaceId: report.workspaceId } : {}), - }) - - /** - * The log line is for us; this is for the people who own the secrets. - * - * Recorded here rather than where provenance was lost, because losing it costs nothing on its - * own — a write nobody could vouch for is just data at rest. The exposure is this moment: a - * value crossing into a run that will project it to a model with no way to recognise a secret - * inside it and redact it. That is what a reader needs told, and it is why the entry names the - * surface and the count rather than a secret, which is precisely what was not recorded. - * - * Fire-and-forget by construction — `recordAudit` never throws — so the trail can never be the - * reason a run fails. Skipped without a workspace: the entry would name no one it concerns. - */ - if (!report.workspaceId) return - recordAudit({ - workspaceId: report.workspaceId, - actorId: report.actorUserId ?? null, - action: AuditAction.SECRET_PROVENANCE_UNRECORDED, - resourceType: AuditResourceType.SECRET_PROVENANCE, - resourceId: report.surface, - description: - 'A run read data whose secret provenance was never recorded, so any secret it carries could not be redacted before reaching a model.', - metadata: { - surface: report.surface, - cause: report.cause, - ...(report.affectedCount !== undefined ? { affectedCount: report.affectedCount } : {}), - }, - }) -} - -export type DurableSecretProvenanceWriteCause = - | 'source-provenance-unknown' - | 'invalid-provenance-entries' - | 'source-hash-unavailable' - | 'workspace-file-write-unknown' - | 'workspace-file-write-unrecorded' - -export interface DurableSecretProvenanceWriteReport { - surface: DurableSecretProvenanceSurface - status: 'unknown' | 'unrecorded' - cause: DurableSecretProvenanceWriteCause - recordCount?: number - workspaceId?: string - resourceId?: string -} - -export type DurableSecretProvenanceRefusalCause = - | 'knowledge-document-source-unavailable' - | 'knowledge-result-provenance-unavailable' - | 'knowledge-chunk-source-unavailable' - | 'knowledge-workspace-file-source-unavailable' - | 'workspace-file-provenance-unavailable' - | 'workspace-file-opaque-secret-content' - | 'workspace-file-registry-unavailable' - | 'workspace-file-unrecorded-enforced' - -export interface DurableSecretProvenanceRefusalReport { - surface: DurableSecretProvenanceSurface - cause: DurableSecretProvenanceRefusalCause - workspaceId?: string - resourceId?: string -} - -/** - * Reports a non-exact sidecar write without claiming its enclosing transaction committed. - * Kept separate from permissive-read telemetry so writer defects and affected reads can be counted - * independently. Callers report only tracked writes; ordinary legacy records are not a fault. - */ -export function reportDurableSecretProvenanceWrite( - report: DurableSecretProvenanceWriteReport -): void { - persistenceLogger.error('Writing non-exact durable secret provenance', { - surface: report.surface, - status: report.status, - cause: report.cause, - ...(report.recordCount !== undefined ? { recordCount: report.recordCount } : {}), - ...(report.workspaceId ? { workspaceId: report.workspaceId } : {}), - ...(report.resourceId ? { resourceId: report.resourceId } : {}), - }) -} - -/** Reports an existing refusal decision without changing the surface's compatibility policy. */ -export function reportDurableSecretProvenanceRefusal( - report: DurableSecretProvenanceRefusalReport -): void { - persistenceLogger.error('Refusing unavailable durable secret provenance', { - surface: report.surface, - cause: report.cause, - ...(report.workspaceId ? { workspaceId: report.workspaceId } : {}), - ...(report.resourceId ? { resourceId: report.resourceId } : {}), - }) -} - -/** Test seam: forces the next read to re-resolve the env-configured surfaces. */ -export function resetDurableSecretProvenanceEnforcementCache(): void { - enforcedSurfaces = undefined -} diff --git a/apps/sim/lib/execution/durable-secret-provenance-telemetry.test.ts b/apps/sim/lib/execution/durable-secret-provenance-telemetry.test.ts new file mode 100644 index 00000000000..bd319fa3db2 --- /dev/null +++ b/apps/sim/lib/execution/durable-secret-provenance-telemetry.test.ts @@ -0,0 +1,58 @@ +/** @vitest-environment node */ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const { logger } = vi.hoisted(() => ({ logger: { error: vi.fn() } })) +vi.mock('@sim/logger', () => ({ createLogger: () => logger })) + +import { + reportDurableSecretProvenanceRefusal, + reportDurableSecretProvenanceWrite, +} from '@/lib/execution/durable-secret-provenance-telemetry' + +describe('durable secret provenance telemetry', () => { + beforeEach(() => vi.clearAllMocks()) + it('reports non-exact writes without including private content or entries', () => { + const report = { + surface: 'knowledge' as const, + status: 'unknown' as const, + cause: 'source-provenance-unknown' as const, + recordCount: 2, + workspaceId: 'workspace-1', + resourceId: 'document-1', + content: 'private content', + entries: [{ encryptedValue: 'private ciphertext' }], + } + reportDurableSecretProvenanceWrite(report) + expect(logger.error).toHaveBeenCalledExactlyOnceWith( + 'Writing non-exact durable secret provenance', + { + surface: 'knowledge', + status: 'unknown', + cause: 'source-provenance-unknown', + recordCount: 2, + workspaceId: 'workspace-1', + resourceId: 'document-1', + } + ) + }) + it('reports a refusal without including private content or entries', () => { + const report = { + surface: 'workspace-file' as const, + cause: 'workspace-file-opaque-secret-content' as const, + workspaceId: 'workspace-1', + resourceId: 'file-1', + content: 'private content', + entries: [{ encryptedValue: 'private ciphertext' }], + } + reportDurableSecretProvenanceRefusal(report) + expect(logger.error).toHaveBeenCalledExactlyOnceWith( + 'Refusing unavailable durable secret provenance', + { + surface: 'workspace-file', + cause: 'workspace-file-opaque-secret-content', + workspaceId: 'workspace-1', + resourceId: 'file-1', + } + ) + }) +}) diff --git a/apps/sim/lib/execution/durable-secret-provenance-telemetry.ts b/apps/sim/lib/execution/durable-secret-provenance-telemetry.ts new file mode 100644 index 00000000000..bf77d2be246 --- /dev/null +++ b/apps/sim/lib/execution/durable-secret-provenance-telemetry.ts @@ -0,0 +1,67 @@ +import { createLogger } from '@sim/logger' + +const persistenceLogger = createLogger('DurableSecretProvenancePersistence') + +export type DurableSecretProvenanceSurface = 'memory' | 'table-row' | 'knowledge' | 'workspace-file' + +export type DurableSecretProvenanceWriteCause = + | 'source-provenance-unknown' + | 'invalid-provenance-entries' + | 'source-hash-unavailable' + | 'workspace-file-write-unknown' + | 'workspace-file-write-unrecorded' + +export interface DurableSecretProvenanceWriteReport { + surface: DurableSecretProvenanceSurface + status: 'unknown' | 'unrecorded' + cause: DurableSecretProvenanceWriteCause + recordCount?: number + workspaceId?: string + resourceId?: string +} + +export type DurableSecretProvenanceRefusalCause = + | 'knowledge-document-source-unavailable' + | 'knowledge-result-provenance-unavailable' + | 'knowledge-chunk-source-unavailable' + | 'knowledge-workspace-file-source-unavailable' + | 'workspace-file-provenance-unavailable' + | 'workspace-file-opaque-secret-content' + | 'workspace-file-registry-unavailable' + | 'workspace-file-unrecorded-enforced' + +export interface DurableSecretProvenanceRefusalReport { + surface: DurableSecretProvenanceSurface + cause: DurableSecretProvenanceRefusalCause + workspaceId?: string + resourceId?: string +} + +/** + * Reports a non-exact sidecar write without claiming its enclosing transaction committed. + * Writer defects and refused reads are reported separately. Ordinary legacy records are not faults. + */ +export function reportDurableSecretProvenanceWrite( + report: DurableSecretProvenanceWriteReport +): void { + persistenceLogger.error('Writing non-exact durable secret provenance', { + surface: report.surface, + status: report.status, + cause: report.cause, + ...(report.recordCount !== undefined ? { recordCount: report.recordCount } : {}), + ...(report.workspaceId ? { workspaceId: report.workspaceId } : {}), + ...(report.resourceId ? { resourceId: report.resourceId } : {}), + }) +} + +/** Reports a durable read refusal without exposing content or secret values. */ +export function reportDurableSecretProvenanceRefusal( + report: DurableSecretProvenanceRefusalReport +): void { + persistenceLogger.error('Refusing unavailable durable secret provenance', { + surface: report.surface, + cause: report.cause, + ...(report.workspaceId ? { workspaceId: report.workspaceId } : {}), + ...(report.resourceId ? { resourceId: report.resourceId } : {}), + }) +} diff --git a/apps/sim/lib/execution/durable-secret-provenance.test.ts b/apps/sim/lib/execution/durable-secret-provenance.test.ts index 3180c24db5c..26f48ed5f08 100644 --- a/apps/sim/lib/execution/durable-secret-provenance.test.ts +++ b/apps/sim/lib/execution/durable-secret-provenance.test.ts @@ -2,18 +2,6 @@ * @vitest-environment node */ import { beforeEach, describe, expect, it, vi } from 'vitest' - -const { mockIsEnforced, mockReport } = vi.hoisted(() => ({ - mockIsEnforced: vi.fn(() => false), - mockReport: vi.fn(), -})) - -vi.mock('@/lib/execution/durable-secret-provenance-enforcement', () => ({ - DURABLE_SECRET_PROVENANCE_SURFACES: ['memory', 'table-row', 'knowledge'], - isDurableSecretProvenanceEnforced: mockIsEnforced, - reportUnrecordedDurableProvenance: mockReport, -})) - import { durableSecretProvenanceFromPrivateBundle, filterDurableSecretProvenanceBySourceValues, @@ -275,37 +263,12 @@ describe('importing unrecorded durable provenance', () => { beforeEach(() => { vi.clearAllMocks() - mockIsEnforced.mockReturnValue(false) }) - it('warns and leaves the registry able to vouch when the surface is not enforced', async () => { + it('refuses unknown provenance at the shared import boundary', async () => { const registry = new ResolvedSecretTraceRegistry() - await expect( - importDurableSecretProvenance(registry, UNKNOWN, undefined, 'memory') - ).resolves.toBe(true) - expect(registry.isPermanentlyIncomplete()).toBe(false) - expect(mockReport).toHaveBeenCalledWith({ - surface: 'memory', - cause: 'durable-provenance-unknown', - }) - }) - - it('latches the registry once that surface is closed', async () => { - mockIsEnforced.mockReturnValue(true) - const registry = new ResolvedSecretTraceRegistry() - - await expect( - importDurableSecretProvenance(registry, UNKNOWN, undefined, 'memory') - ).resolves.toBe(false) - expect(registry.isPermanentlyIncomplete()).toBe(true) - expect(mockReport).not.toHaveBeenCalled() - }) - - it('latches for a caller that has not declared a surface', async () => { - const registry = new ResolvedSecretTraceRegistry() - - await expect(importDurableSecretProvenance(registry, UNKNOWN)).resolves.toBe(false) + await expect(importDurableSecretProvenance(registry, UNKNOWN, undefined)).resolves.toBe(false) expect(registry.isPermanentlyIncomplete()).toBe(true) }) @@ -313,10 +276,7 @@ describe('importing unrecorded durable provenance', () => { const registry = new ResolvedSecretTraceRegistry() const malformed = { status: 'exact', entries: [{ encryptedValue: '' }] } as never - await expect( - importDurableSecretProvenance(registry, malformed, undefined, 'memory') - ).resolves.toBe(false) + await expect(importDurableSecretProvenance(registry, malformed, undefined)).resolves.toBe(false) expect(registry.isPermanentlyIncomplete()).toBe(true) - expect(mockReport).not.toHaveBeenCalled() }) }) diff --git a/apps/sim/lib/execution/durable-secret-provenance.ts b/apps/sim/lib/execution/durable-secret-provenance.ts index 6fbdc24feed..2e7900dfd0c 100644 --- a/apps/sim/lib/execution/durable-secret-provenance.ts +++ b/apps/sim/lib/execution/durable-secret-provenance.ts @@ -1,10 +1,5 @@ import { createHash } from 'node:crypto' import type { DurableSecretProvenanceEntry } from '@sim/db/schema' -import { - type DurableSecretProvenanceSurface, - isDurableSecretProvenanceEnforced, - reportUnrecordedDurableProvenance, -} from '@/lib/execution/durable-secret-provenance-enforcement' import { isPrivateSecretProvenanceBundleV1, type PrivateSecretProvenanceBundleV1, @@ -205,35 +200,13 @@ export function filterDurableSecretProvenanceBySourceValues( return entries ? { status: 'exact', entries } : { status: 'unknown' } } -/** - * Imports durable entries into a model-bound registry, preserving source-scope anonymity. - * - * `surface` selects the enforcement policy for provenance nobody recorded. Omitting it enforces, - * which is the right default for a caller that has not been reviewed against - * {@link isDurableSecretProvenanceEnforced} yet. - */ +/** Imports exact durable entries into a model-bound registry, preserving source-scope anonymity. */ export async function importDurableSecretProvenance( registry: ResolvedSecretTraceRegistry, provenance: DurableSecretProvenance, - value?: unknown, - surface?: DurableSecretProvenanceSurface, - /** - * Set by a caller that reports the whole read itself. - * - * This function sees one record and knows no workspace, so its report can only ever be a log - * line, one per record. A caller reading a page can say the same thing once, with the workspace - * and the count — which is the entry that reaches the people who own the secrets. Both reporting - * would double-count the same event at two different granularities. - */ - options: { reportUnrecorded?: boolean } = {} + value?: unknown ): Promise { if (provenance.status === 'unknown') { - if (surface && !isDurableSecretProvenanceEnforced(surface)) { - if (options.reportUnrecorded !== false) { - reportUnrecordedDurableProvenance({ surface, cause: 'durable-provenance-unknown' }) - } - return true - } registry.markIncomplete('durable-provenance-unknown') return false } diff --git a/apps/sim/lib/function-execution/execute-request.ts b/apps/sim/lib/function-execution/execute-request.ts index de66f8dbfe9..af34bdf672e 100644 --- a/apps/sim/lib/function-execution/execute-request.ts +++ b/apps/sim/lib/function-execution/execute-request.ts @@ -1052,7 +1052,6 @@ async function importRuntimeFileContributors( workspaceId: context.workspaceId, identity, registry: context.runtimeFileSecretTraceRegistry, - actorUserId: context.fileAccessUserId, }) if (!imported) throw new Error('File secret provenance is unavailable for Function execution') } diff --git a/apps/sim/lib/internal/file/operations.provenance.test.ts b/apps/sim/lib/internal/file/operations.provenance.test.ts index e82ebb64904..c3741e17441 100644 --- a/apps/sim/lib/internal/file/operations.provenance.test.ts +++ b/apps/sim/lib/internal/file/operations.provenance.test.ts @@ -13,7 +13,6 @@ import { import { beforeEach, describe, expect, it, vi } from 'vitest' const { - mockEnforced, mockAssertActiveWorkspaceAccess, mockFetchWorkspaceFileBuffer, mockLoadActiveWorkspaceContext, @@ -23,7 +22,6 @@ const { mockResolveWorkspaceFileReference, mockUpdateWorkspaceFileContent, } = vi.hoisted(() => ({ - mockEnforced: vi.fn(() => false), mockAssertActiveWorkspaceAccess: vi.fn(), mockFetchWorkspaceFileBuffer: vi.fn(), mockLoadActiveWorkspaceContext: vi.fn(), @@ -164,9 +162,7 @@ vi.mock('@/app/api/files/authorization', () => ({ verifyFileAccess: vi.fn(), })) -vi.mock('@/lib/execution/durable-secret-provenance-enforcement', () => ({ - isDurableSecretProvenanceEnforced: mockEnforced, - reportUnrecordedDurableProvenance: vi.fn(), +vi.mock('@/lib/execution/durable-secret-provenance-telemetry', () => ({ reportDurableSecretProvenanceWrite: vi.fn(), reportDurableSecretProvenanceRefusal: vi.fn(), })) @@ -259,7 +255,6 @@ describe('appended file provenance', () => { beforeEach(() => { vi.clearAllMocks() resetDbChainMock() - mockEnforced.mockReturnValue(false) dbChainMockFns.returning.mockResolvedValue([{ id: 'file-1' }]) mockResolveEffectiveWorkspacePermission.mockResolvedValue('write') mockLoadActiveWorkspaceContext.mockResolvedValue({ @@ -366,35 +361,31 @@ describe('appended file provenance', () => { expect(persisted.contentUpdatedAt).toEqual(NEXT_CONTENT_UPDATED_AT) expect(persisted.entries).toHaveLength(expectedStatus === 'exact' && secret ? 1 : 0) expect(dbChainMockFns.set).toHaveBeenCalledWith({ secretProvenanceVersion: 1 }) - for (const enforced of [false, true]) { - mockEnforced.mockReturnValue(enforced) - queueTableRows(workspaceFiles, [ - joinedRow(persisted.status, persisted.entries, persisted.contentUpdatedAt), - ]) - const registry = new ResolvedSecretTraceRegistry([], SCOPE) - const permitted = await importWorkspaceFileSecretProvenanceForModelView({ - workspaceId: 'workspace-1', - identity: IDENTITY, - registry, - view: 'complete', - value: `before:${content}`, + + queueTableRows(workspaceFiles, [ + joinedRow(persisted.status, persisted.entries, persisted.contentUpdatedAt), + ]) + const registry = new ResolvedSecretTraceRegistry([], SCOPE) + const permitted = await importWorkspaceFileSecretProvenanceForModelView({ + workspaceId: 'workspace-1', + identity: IDENTITY, + registry, + view: 'complete', + value: `before:${content}`, + }) + expect(permitted).toBe(expectedStatus === 'exact') + if (permitted) { + expect(projectResolvedSecretModelContent(`before:${content}`, registry)).toEqual({ + safe: true, + value: secret ? 'before:{{TOKEN}}' : `before:${content}`, }) - expect(permitted).toBe( - expectedStatus === 'exact' || (expectedStatus === 'unrecorded' && !enforced) - ) - if (permitted) { - expect(projectResolvedSecretModelContent(`before:${content}`, registry)).toEqual({ - safe: true, - value: secret ? 'before:{{TOKEN}}' : `before:${content}`, - }) - } - queueTableRows(workspaceFiles, [ - joinedRow(persisted.status, persisted.entries, persisted.contentUpdatedAt), - ]) - expect(await isOpaqueWorkspaceFileEgressSafe('workspace-1', IDENTITY)).toBe( - (expectedStatus === 'exact' && !secret) || (expectedStatus === 'unrecorded' && !enforced) - ) } + queueTableRows(workspaceFiles, [ + joinedRow(persisted.status, persisted.entries, persisted.contentUpdatedAt), + ]) + expect(await isOpaqueWorkspaceFileEgressSafe('workspace-1', IDENTITY)).toBe( + expectedStatus === 'exact' && !secret + ) } ) }) @@ -419,20 +410,14 @@ describe('execution-file content provenance', () => { }) it.each([ - { status: 'exact', version: 1, stale: false, enforced: false, complete: true }, - { status: 'exact', version: 1, stale: false, enforced: true, complete: true }, - { status: 'unrecorded', version: 1, stale: false, enforced: false, complete: true }, - { status: 'unrecorded', version: 1, stale: false, enforced: true, complete: false }, - { status: 'unknown', version: 1, stale: false, enforced: false, complete: false }, - { status: 'unknown', version: 1, stale: false, enforced: true, complete: false }, - { status: 'unknown', version: null, stale: false, enforced: false, complete: true }, - { status: 'unknown', version: null, stale: false, enforced: true, complete: true }, - { status: 'exact', version: 1, stale: true, enforced: false, complete: false }, - { status: 'exact', version: 1, stale: true, enforced: true, complete: false }, + { status: 'exact', version: 1, stale: false, complete: true }, + { status: 'unrecorded', version: 1, stale: false, complete: false }, + { status: 'unknown', version: 1, stale: false, complete: false }, + { status: 'unknown', version: null, stale: false, complete: true }, + { status: 'exact', version: 1, stale: true, complete: false }, ])( - 'reads $status version=$version stale=$stale with enforcement=$enforced', - async ({ status, version, stale, enforced, complete }) => { - mockEnforced.mockReturnValue(enforced) + 'reads $status version=$version stale=$stale', + async ({ status, version, stale, complete }) => { queueTableRows(workspaceFiles, [ { ...joinedRow(status), @@ -450,7 +435,6 @@ describe('execution-file content provenance', () => { ) it('retains exact secret-bearing execution lineage for downstream text projections', async () => { - mockEnforced.mockReturnValue(true) queueTableRows(workspaceFiles, [ joinedRow('exact', [ { diff --git a/apps/sim/lib/internal/file/operations.ts b/apps/sim/lib/internal/file/operations.ts index 5e84bdcac74..ece36d9e054 100644 --- a/apps/sim/lib/internal/file/operations.ts +++ b/apps/sim/lib/internal/file/operations.ts @@ -46,7 +46,6 @@ import type { } from '@/lib/uploads/contexts/workspace/workspace-file-manager' import { getBoundWorkspaceFileSecretProvenance, - mayReadUnrecordedWorkspaceFile, mergeWorkspaceFileSecretProvenance, type WorkspaceFileSecretProvenance, type WorkspaceFileSecretProvenanceIdentity, @@ -620,7 +619,6 @@ export async function getFileContentProvenance( } const provenance = await readFileSourceSecretProvenance(principal, workspaceId, source.identity) signal?.throwIfAborted() - if (provenance.status === 'unrecorded' && mayReadUnrecordedWorkspaceFile(workspaceId)) continue if (provenance.status !== 'exact' || (source.opaque && provenance.entries.length > 0)) { accumulator.markIncomplete('workspace-file-provenance-unknown') continue diff --git a/apps/sim/lib/internal/memory/provenance.ts b/apps/sim/lib/internal/memory/provenance.ts index 788186ecef4..fa8722c4ccc 100644 --- a/apps/sim/lib/internal/memory/provenance.ts +++ b/apps/sim/lib/internal/memory/provenance.ts @@ -67,9 +67,7 @@ export async function createMemoryToolResponse( const registry = new ResolvedSecretTraceRegistry([], scope) for (const item of provenance) { - await importDurableSecretProvenance(registry, item.provenance, item.data, 'memory', { - reportUnrecorded: false, - }) + await importDurableSecretProvenance(registry, item.provenance, item.data) } const envelope = serializePrivateToolMetadataResponseEnvelope( body, diff --git a/apps/sim/lib/knowledge/__integration__/external-file-provenance.integration.ts b/apps/sim/lib/knowledge/__integration__/external-file-provenance.integration.ts index df6b7ca6b22..e4bd466e9e9 100644 --- a/apps/sim/lib/knowledge/__integration__/external-file-provenance.integration.ts +++ b/apps/sim/lib/knowledge/__integration__/external-file-provenance.integration.ts @@ -11,7 +11,6 @@ import { eq, inArray } from 'drizzle-orm' import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest' const fixtureStorage = vi.hoisted(() => { - process.env.DURABLE_SECRET_PROVENANCE_ENFORCED_SURFACES = 'workspace-file' return { root: '' } }) vi.mock('@/lib/uploads/core/setup.server', () => ({ @@ -24,7 +23,6 @@ import { fileParseBodySchema } from '@/lib/api/contracts/storage-transfer' import { encryptSecret } from '@/lib/core/security/encryption' import * as inputValidation from '@/lib/core/security/input-validation.server' import { isUserFileWithMetadata } from '@/lib/core/utils/user-file' -import { isDurableSecretProvenanceEnforced } from '@/lib/execution/durable-secret-provenance-enforcement' import { executeFileParserOperation } from '@/lib/internal/file/parser' import { createKnowledgeAclFixtureIds, @@ -108,7 +106,6 @@ async function identityFor(file: UserFile): Promise { fixtureStorage.root = mkdtempSync(path.join(tmpdir(), 'sim-external-file-provenance-')) - expect(isDurableSecretProvenanceEnforced('workspace-file')).toBe(true) }) beforeEach(() => { fetchSpy.mockReset() diff --git a/apps/sim/lib/knowledge/__integration__/upload-read-provenance.integration.ts b/apps/sim/lib/knowledge/__integration__/upload-read-provenance.integration.ts index 54db3536ac7..a602cf272f6 100644 --- a/apps/sim/lib/knowledge/__integration__/upload-read-provenance.integration.ts +++ b/apps/sim/lib/knowledge/__integration__/upload-read-provenance.integration.ts @@ -136,7 +136,6 @@ describe('chat upload reads racing with save_upload', () => { expect( await importWorkspaceFileSecretProvenanceForModelView({ workspaceId: ids.workspaceId, - actorUserId: ids.aliceId, identity: envelope.file, value: envelope.value, view: 'opaque', @@ -202,7 +201,6 @@ describe('chat upload reads racing with save_upload', () => { expect( await importWorkspaceFileSecretProvenanceForModelView({ workspaceId: ids.workspaceId, - actorUserId: ids.aliceId, identity: read.file, value: read.value, view: 'opaque', diff --git a/apps/sim/lib/knowledge/api/secret-provenance.ts b/apps/sim/lib/knowledge/api/secret-provenance.ts index 4ca15750404..29935ee84d5 100644 --- a/apps/sim/lib/knowledge/api/secret-provenance.ts +++ b/apps/sim/lib/knowledge/api/secret-provenance.ts @@ -243,8 +243,6 @@ export async function finalizeKnowledgePersistedResponse(options: { registry, documents: options.documents, chunks: options.chunks, - ...(options.workspaceId ? { workspaceId: options.workspaceId } : {}), - actorUserId: options.userId, }) return finalizeKnowledgeRegistryResponse({ headers: options.headers, diff --git a/apps/sim/lib/knowledge/application/add-workspace-files.ts b/apps/sim/lib/knowledge/application/add-workspace-files.ts index 0e458afd603..b4822bf8a2f 100644 --- a/apps/sim/lib/knowledge/application/add-workspace-files.ts +++ b/apps/sim/lib/knowledge/application/add-workspace-files.ts @@ -5,7 +5,7 @@ import { checkAttributedUsageLimits } from '@/lib/billing/core/billing-attributi import { authorizeWorkspaceOperation } from '@/lib/core/application' import { asOrchestrationError, OrchestrationError } from '@/lib/core/orchestration/types' import { generateRequestId } from '@/lib/core/utils/request' -import { reportDurableSecretProvenanceRefusal } from '@/lib/execution/durable-secret-provenance-enforcement' +import { reportDurableSecretProvenanceRefusal } from '@/lib/execution/durable-secret-provenance-telemetry' import { PROVENANCE_MAX_ENTRIES } from '@/lib/execution/provenance-limits' import { knowledgeDelegationPolicy } from '@/lib/knowledge/application/authorization' import { defineAuthorizedKnowledgeUseCase } from '@/lib/knowledge/application/authorized-knowledge-use-case' diff --git a/apps/sim/lib/knowledge/application/chunks.ts b/apps/sim/lib/knowledge/application/chunks.ts index 6e5822856aa..fd2f5774014 100644 --- a/apps/sim/lib/knowledge/application/chunks.ts +++ b/apps/sim/lib/knowledge/application/chunks.ts @@ -7,7 +7,7 @@ import { createDurableSecretProvenanceRegistry, type DurableSecretProvenance, } from '@/lib/execution/durable-secret-provenance' -import { reportDurableSecretProvenanceRefusal } from '@/lib/execution/durable-secret-provenance-enforcement' +import { reportDurableSecretProvenanceRefusal } from '@/lib/execution/durable-secret-provenance-telemetry' import { defineAuthorizedKnowledgeUseCase } from '@/lib/knowledge/application/authorized-knowledge-use-case' import { resolveKnowledgeAttributedUserId } from '@/lib/knowledge/application/billing' import { KnowledgeDocumentNotReadyError } from '@/lib/knowledge/application/chunk-errors' diff --git a/apps/sim/lib/knowledge/application/read-indexed-document.ts b/apps/sim/lib/knowledge/application/read-indexed-document.ts index 37acf6579b2..d5f2106597a 100644 --- a/apps/sim/lib/knowledge/application/read-indexed-document.ts +++ b/apps/sim/lib/knowledge/application/read-indexed-document.ts @@ -1,4 +1,3 @@ -import { resolvePrincipalSubjectUserId } from '@sim/auth/principal' import { db } from '@sim/db' import { document, embedding } from '@sim/db/schema' import { and, eq, isNull, lte, sql } from 'drizzle-orm' @@ -161,13 +160,9 @@ export const readIndexedKnowledgeDocument = defineAuthorizedKnowledgeUseCase({ connectorType: doc.connectorType, processingStatus: doc.processingStatus, } - const provenanceContext = { - registry: input.resultSecretRegistry, - actorUserId: resolvePrincipalSubjectUserId(principal) ?? undefined, - } if ( !(await importKnowledgePersistedResponseSecretProvenance({ - ...provenanceContext, + registry: input.resultSecretRegistry, documents: [{ id: doc.id, source: createKnowledgeDocumentSourceValue(doc), value }], })) ) { @@ -221,7 +216,7 @@ export const readIndexedKnowledgeDocument = defineAuthorizedKnowledgeUseCase({ const chunks = page.chunks.map(({ id, chunkIndex, content }) => ({ id, chunkIndex, content })) if ( !(await importKnowledgePersistedResponseSecretProvenance({ - ...provenanceContext, + registry: input.resultSecretRegistry, chunks: chunks.map((chunk) => ({ ...chunk, documentId, value: chunk })), })) ) { diff --git a/apps/sim/lib/knowledge/application/read-search-document.ts b/apps/sim/lib/knowledge/application/read-search-document.ts index dbc52ccc0b5..94aef40c21f 100644 --- a/apps/sim/lib/knowledge/application/read-search-document.ts +++ b/apps/sim/lib/knowledge/application/read-search-document.ts @@ -115,12 +115,10 @@ export const readSearchDocument = defineAuthorizedKnowledgeUseCase({ const metadata = provenance.documentMetadata[context.documentId] if ( metadata && - !(await importDurableSecretProvenance( - input.resultSecretRegistry, - metadata.provenance, - { documentName: metadata.filename, sourceUrl: metadata.sourceUrl }, - 'knowledge' - )) + !(await importDurableSecretProvenance(input.resultSecretRegistry, metadata.provenance, { + documentName: metadata.filename, + sourceUrl: metadata.sourceUrl, + })) ) { throw new Error('Knowledge document provenance is unavailable') } diff --git a/apps/sim/lib/knowledge/application/search.ts b/apps/sim/lib/knowledge/application/search.ts index 1b7c2adcf40..5b186cc1f22 100644 --- a/apps/sim/lib/knowledge/application/search.ts +++ b/apps/sim/lib/knowledge/application/search.ts @@ -13,11 +13,7 @@ import { resourceScopeFromOwner, resourceScopeKey } from '@/lib/core/resource-sc import { PlatformEvents } from '@/lib/core/telemetry' import { generateRequestId } from '@/lib/core/utils/request' import { importDurableSecretProvenance } from '@/lib/execution/durable-secret-provenance' -import { - isDurableSecretProvenanceEnforced, - reportDurableSecretProvenanceRefusal, - reportUnrecordedDurableProvenance, -} from '@/lib/execution/durable-secret-provenance-enforcement' +import { reportDurableSecretProvenanceRefusal } from '@/lib/execution/durable-secret-provenance-telemetry' import { requireOrganizationSearchAvailable } from '@/lib/knowledge/access/availability' import { createKnowledgeAccessProvider } from '@/lib/knowledge/access/scope' import type { KnowledgeAccessProvider } from '@/lib/knowledge/access/types' @@ -550,8 +546,6 @@ const searchKnowledgeUseCase = defineAuthorizedKnowledgeUseCase({ status: rerankerStatus, candidateCount, resultCount: rows.length, - unrecordedChunkCount: provenanceSnapshot?.unrecordedCount ?? 0, - enforced: isDurableSecretProvenanceEnforced('knowledge'), workspaceId: context.workspaceId, }) } else if (useReranker) { @@ -670,8 +664,6 @@ const searchKnowledgeUseCase = defineAuthorizedKnowledgeUseCase({ } }) if (registry && provenanceSnapshot) { - const knowledgeEnforced = isDurableSecretProvenanceEnforced('knowledge') - let unrecordedCount = provenanceSnapshot.unrecordedCount for (const [documentId, document] of Object.entries(provenanceSnapshot.documentMetadata)) { const renderedMetadata = results .filter((result) => result.documentId === documentId) @@ -681,36 +673,14 @@ const searchKnowledgeUseCase = defineAuthorizedKnowledgeUseCase({ metadata: result.metadata, })) if (renderedMetadata.length === 0) continue - if (document.provenance.status === 'unknown' && !knowledgeEnforced) unrecordedCount += 1 if ( !(await measureSearchStage('metadata_provenance', () => - importDurableSecretProvenance( - registry, - document.provenance, - renderedMetadata, - 'knowledge', - { reportUnrecorded: false } - ) + importDurableSecretProvenance(registry, document.provenance, renderedMetadata) )) ) { registry.markIncomplete('knowledge-result-provenance-unavailable') } } - /** - * One entry for the whole search — chunks and rendered metadata are one read. Skipped when - * the registry latched: a latched read never reaches a model, and this entry exists to say a - * fail-open read went ahead unvouched. - */ - if (unrecordedCount > 0 && !registry.isPermanentlyIncomplete()) { - reportUnrecordedDurableProvenance({ - surface: 'knowledge', - cause: 'durable-provenance-unknown', - affectedCount: unrecordedCount, - workspaceId: context.workspaceId, - - actorUserId: userId, - }) - } } annotateSearchDiagnostics({ resultCount: results.length }) const cost = baseCost diff --git a/apps/sim/lib/knowledge/secret-provenance.test.ts b/apps/sim/lib/knowledge/secret-provenance.test.ts index 4236fe27e66..2b5b06de981 100644 --- a/apps/sim/lib/knowledge/secret-provenance.test.ts +++ b/apps/sim/lib/knowledge/secret-provenance.test.ts @@ -19,22 +19,17 @@ import { } from '@/lib/knowledge/secret-provenance' import { ResolvedSecretTraceRegistry } from '@/executor/utils/resolved-secret-trace-registry' -const { mockDecryptSecret, mockIsEnforced, mockReport, mockReportWrite, mockReportRefusal } = - vi.hoisted(() => ({ - mockDecryptSecret: vi.fn(), - mockIsEnforced: vi.fn(() => false), - mockReport: vi.fn(), - mockReportWrite: vi.fn(), - mockReportRefusal: vi.fn(), - })) +const { mockDecryptSecret, mockReportWrite, mockReportRefusal } = vi.hoisted(() => ({ + mockDecryptSecret: vi.fn(), + mockReportWrite: vi.fn(), + mockReportRefusal: vi.fn(), +})) vi.mock('@/lib/core/security/encryption', () => ({ decryptSecret: mockDecryptSecret, })) -vi.mock('@/lib/execution/durable-secret-provenance-enforcement', () => ({ - isDurableSecretProvenanceEnforced: mockIsEnforced, - reportUnrecordedDurableProvenance: mockReport, +vi.mock('@/lib/execution/durable-secret-provenance-telemetry', () => ({ reportDurableSecretProvenanceWrite: mockReportWrite, reportDurableSecretProvenanceRefusal: mockReportRefusal, })) @@ -59,7 +54,6 @@ describe('knowledge durable secret provenance', () => { resetDbChainMock() queueTableRows(document, [DOCUMENT_ROW]) mockDecryptSecret.mockResolvedValue({ decrypted: 'tracked-secret' }) - mockIsEnforced.mockReturnValue(false) }) it('uses the same explicit source shape for joined rows and persisted writes', () => { @@ -189,7 +183,7 @@ describe('knowledge durable secret provenance', () => { }) }) -describe('knowledge unrecorded-read reporting', () => { +describe('knowledge durable provenance enforcement', () => { const SCOPE = { userId: 'user-1', workspaceId: 'workspace-1' } const UNRECORDED_DOCUMENT_ROW = { id: 'doc-1', @@ -213,10 +207,9 @@ describe('knowledge unrecorded-read reporting', () => { beforeEach(() => { vi.clearAllMocks() resetDbChainMock() - mockIsEnforced.mockReturnValue(false) }) - it('reports one aggregated entry per read, naming workspace, actor, and count', async () => { + it('refuses unknown document provenance in a mixed persisted response', async () => { queueTableRows(document, [UNRECORDED_DOCUMENT_ROW]) queueTableRows(embedding, [UNRECORDED_CHUNK_ROW]) const registry = new ResolvedSecretTraceRegistry([], SCOPE) @@ -226,24 +219,13 @@ describe('knowledge unrecorded-read reporting', () => { registry, documents: [{ id: 'doc-1', source: DOCUMENT_SOURCE, value: {} }], chunks: [{ id: 'chunk-1', documentId: 'doc-1', content: 'chunk text', value: {} }], - workspaceId: 'workspace-1', - actorUserId: 'user-1', }) - ).resolves.toBe(true) + ).resolves.toBe(false) - expect(registry.isPermanentlyIncomplete()).toBe(false) - expect(mockReport).toHaveBeenCalledTimes(1) - expect(mockReport).toHaveBeenCalledWith({ - surface: 'knowledge', - cause: 'durable-provenance-unknown', - affectedCount: 2, - workspaceId: 'workspace-1', - actorUserId: 'user-1', - }) + expect(registry.isPermanentlyIncomplete()).toBe(true) }) - /** A fault return fails the read closed, so no unvouched record reached anything to report. */ - it('reports nothing when the read fails closed on a missing row', async () => { + it('refuses missing persisted rows', async () => { queueTableRows(document, []) const registry = new ResolvedSecretTraceRegistry([], SCOPE) @@ -251,16 +233,11 @@ describe('knowledge unrecorded-read reporting', () => { importKnowledgePersistedResponseSecretProvenance({ registry, documents: [{ id: 'doc-1', source: DOCUMENT_SOURCE, value: {} }], - workspaceId: 'workspace-1', - actorUserId: 'user-1', }) ).resolves.toBe(false) - - expect(mockReport).not.toHaveBeenCalled() }) - it('latches without reporting once the surface is enforced', async () => { - mockIsEnforced.mockReturnValue(true) + it('refuses unknown document provenance', async () => { queueTableRows(document, [UNRECORDED_DOCUMENT_ROW]) const registry = new ResolvedSecretTraceRegistry([], SCOPE) @@ -268,17 +245,13 @@ describe('knowledge unrecorded-read reporting', () => { importKnowledgePersistedResponseSecretProvenance({ registry, documents: [{ id: 'doc-1', source: DOCUMENT_SOURCE, value: {} }], - workspaceId: 'workspace-1', - actorUserId: 'user-1', }) ).resolves.toBe(false) expect(registry.isPermanentlyIncomplete()).toBe(true) - expect(mockReport).not.toHaveBeenCalled() }) - /** The search read spans chunks and rendered metadata, so its caller owns the one report. */ - it('returns the unrecorded count from a search import instead of reporting it', async () => { + it('refuses unrecorded chunks during a search import', async () => { queueTableRows(embedding, [{ ...UNRECORDED_CHUNK_ROW, documentId: DOCUMENT_ROW.id }]) queueTableRows(document, [DOCUMENT_ROW]) const registry = new ResolvedSecretTraceRegistry([], SCOPE) @@ -288,8 +261,7 @@ describe('knowledge unrecorded-read reporting', () => { results: [{ id: 'chunk-1', documentId: DOCUMENT_ROW.id, content: 'chunk text' }], }) - expect(snapshot.imported).toBe(true) - expect(snapshot.unrecordedCount).toBe(1) - expect(mockReport).not.toHaveBeenCalled() + expect(snapshot.imported).toBe(false) + expect(registry.isPermanentlyIncomplete()).toBe(true) }) }) diff --git a/apps/sim/lib/knowledge/secret-provenance.ts b/apps/sim/lib/knowledge/secret-provenance.ts index f2fff65491d..3f73183cb46 100644 --- a/apps/sim/lib/knowledge/secret-provenance.ts +++ b/apps/sim/lib/knowledge/secret-provenance.ts @@ -19,11 +19,9 @@ import { normalizeDurableSecretProvenanceEntries, } from '@/lib/execution/durable-secret-provenance' import { - isDurableSecretProvenanceEnforced, reportDurableSecretProvenanceRefusal, reportDurableSecretProvenanceWrite, - reportUnrecordedDurableProvenance, -} from '@/lib/execution/durable-secret-provenance-enforcement' +} from '@/lib/execution/durable-secret-provenance-telemetry' import { ResolvedSecretTraceRegistry, type ResolvedSecretTraceScopeV1, @@ -435,7 +433,7 @@ export async function loadKnowledgeDocumentSecretRegistry( tracked: row.secretProvenanceVersion === 1 || currentSourceFileProvenance !== undefined, } const registry = new ResolvedSecretTraceRegistry([], scope) - if (!(await importDurableSecretProvenance(registry, provenance, undefined, 'knowledge'))) { + if (!(await importDurableSecretProvenance(registry, provenance, undefined))) { throw new Error('Knowledge document secret provenance is unavailable') } return { registry, provenance, tracked: true } @@ -475,21 +473,9 @@ export async function importKnowledgePersistedResponseSecretProvenance(options: content: string value: unknown }[] - /** Names the workspace in the aggregated unrecorded-read audit entry; legacy KBs have none. */ - workspaceId?: string - /** Whose access authorized the read, for the same entry. */ - actorUserId?: string }): Promise { const documents = options.documents ?? [] const chunks = options.chunks ?? [] - /** - * Counted here and reported once at the end of the proceed path, the shape the memory and table - * surfaces use: the per-record import knows no workspace, so its report never produced the - * workspace-visible audit entry, and it logged once per record. A fault return skips the report — - * that read fails closed, so no unvouched record reached anything. - */ - const knowledgeEnforced = isDurableSecretProvenanceEnforced('knowledge') - let unrecordedCount = 0 const documentIds = [...new Set(documents.map((item) => item.id))] const chunkIds = [...new Set(chunks.map((item) => item.id))] const [documentRows, chunkRows] = await Promise.all([ @@ -531,12 +517,7 @@ export async function importKnowledgePersistedResponseSecretProvenance(options: readBoundKnowledgeDocumentSecretProvenance({ ...row, source }), source ) - if (provenance.status === 'unknown' && !knowledgeEnforced) unrecordedCount += 1 - if ( - !(await importDurableSecretProvenance(options.registry, provenance, item.value, 'knowledge', { - reportUnrecorded: false, - })) - ) { + if (!(await importDurableSecretProvenance(options.registry, provenance, item.value))) { return false } } @@ -548,25 +529,11 @@ export async function importKnowledgePersistedResponseSecretProvenance(options: return false } const provenance = readBoundKnowledgeEmbeddingSecretProvenance(row) - if (provenance.status === 'unknown' && !knowledgeEnforced) unrecordedCount += 1 - if ( - !(await importDurableSecretProvenance(options.registry, provenance, item.value, 'knowledge', { - reportUnrecorded: false, - })) - ) { + if (!(await importDurableSecretProvenance(options.registry, provenance, item.value))) { return false } } - if (unrecordedCount > 0) { - reportUnrecordedDurableProvenance({ - surface: 'knowledge', - cause: 'durable-provenance-unknown', - affectedCount: unrecordedCount, - ...(options.workspaceId ? { workspaceId: options.workspaceId } : {}), - actorUserId: options.actorUserId ?? null, - }) - } return !options.registry.isPermanentlyIncomplete() } @@ -576,13 +543,6 @@ export async function importKnowledgeSearchResultSecretProvenance(options: { results: readonly { id: string; documentId: string; content: string }[] }): Promise<{ imported: boolean - /** - * Chunks whose stored provenance was unrecorded and whose import proceeded fail-open. The caller - * folds this into one read-level audit report — it owns the workspace and the metadata imports - * that share the same read, and it reports nothing when the registry latched, since a latched - * read never reaches a model. - */ - unrecordedCount: number documentMetadata: Record< string, { @@ -600,7 +560,7 @@ export async function importKnowledgeSearchResultSecretProvenance(options: { > }> { if (options.results.length === 0) { - return { imported: true, unrecordedCount: 0, documentMetadata: {} } + return { imported: true, documentMetadata: {} } } const embeddingIds = [...new Set(options.results.map((result) => result.id))] const documentIds = [...new Set(options.results.map((result) => result.documentId))] @@ -617,35 +577,24 @@ export async function importKnowledgeSearchResultSecretProvenance(options: { ]) const chunkById = new Map(chunks.map((row) => [row.id, row])) if (chunkById.size !== embeddingIds.length) { - return { imported: false, unrecordedCount: 0, documentMetadata: {} } + return { imported: false, documentMetadata: {} } } - const knowledgeEnforced = isDurableSecretProvenanceEnforced('knowledge') - let unrecordedCount = 0 for (const result of options.results) { const row = chunkById.get(result.id) if (!row || row.documentId !== result.documentId || row.content !== result.content) { - return { imported: false, unrecordedCount: 0, documentMetadata: {} } + return { imported: false, documentMetadata: {} } } const provenance = readBoundKnowledgeEmbeddingSecretProvenance(row) - if (provenance.status === 'unknown' && !knowledgeEnforced) unrecordedCount += 1 - if ( - !(await importDurableSecretProvenance( - options.registry, - provenance, - result.content, - 'knowledge', - { reportUnrecorded: false } - )) - ) { - return { imported: false, unrecordedCount: 0, documentMetadata: {} } + if (!(await importDurableSecretProvenance(options.registry, provenance, result.content))) { + return { imported: false, documentMetadata: {} } } } const documentById = new Map(documents.map((row) => [row.id, row])) if (documentById.size !== documentIds.length) { - return { imported: false, unrecordedCount: 0, documentMetadata: {} } + return { imported: false, documentMetadata: {} } } if (options.results.some((result) => !documentById.has(result.documentId))) { - return { imported: false, unrecordedCount: 0, documentMetadata: {} } + return { imported: false, documentMetadata: {} } } const documentMetadata: Record< string, @@ -684,7 +633,6 @@ export async function importKnowledgeSearchResultSecretProvenance(options: { } return { imported: !options.registry.isPermanentlyIncomplete(), - unrecordedCount, documentMetadata, } } diff --git a/apps/sim/lib/memory/application/use-cases.test.ts b/apps/sim/lib/memory/application/use-cases.test.ts index f30c95699ba..585db69d2bb 100644 --- a/apps/sim/lib/memory/application/use-cases.test.ts +++ b/apps/sim/lib/memory/application/use-cases.test.ts @@ -10,7 +10,6 @@ import type { BillingAttributionSnapshot } from '@/lib/billing/core/billing-attr const mocks = vi.hoisted(() => ({ loadWorkspace: vi.fn(), resolvePermission: vi.fn(), - reportUnrecorded: vi.fn(), readBoundProvenance: vi.fn(), })) @@ -30,11 +29,6 @@ vi.mock('@/lib/billing/core/billing-attribution', () => ({ assertBillingAttributionSnapshot: (value: unknown) => value, })) -vi.mock('@/lib/execution/durable-secret-provenance-enforcement', () => ({ - isDurableSecretProvenanceEnforced: () => false, - reportUnrecordedDurableProvenance: mocks.reportUnrecorded, -})) - vi.mock('@/lib/memory/secret-provenance', () => ({ readBoundMemorySecretProvenance: mocks.readBoundProvenance, replaceMemorySecretProvenanceInTx: vi.fn(), @@ -130,13 +124,6 @@ describe('Memory application use cases', () => { userId: BILLING_OWNER_ID, workspaceId: WORKSPACE_ID, }) - expect(mocks.reportUnrecorded).toHaveBeenCalledWith({ - surface: 'memory', - cause: 'durable-provenance-unknown', - affectedCount: 1, - workspaceId: WORKSPACE_ID, - actorUserId: BILLING_OWNER_ID, - }) }) it('rejects billing attribution outside the authorized canonical workspace', async () => { @@ -170,6 +157,5 @@ describe('Memory application use cases', () => { resolveBillingAttribution.mock.invocationCallOrder[0] ) expect(mocks.readBoundProvenance).not.toHaveBeenCalled() - expect(mocks.reportUnrecorded).not.toHaveBeenCalled() }) }) diff --git a/apps/sim/lib/memory/application/use-cases.ts b/apps/sim/lib/memory/application/use-cases.ts index f7d57f561d9..c9e94e4f3e8 100644 --- a/apps/sim/lib/memory/application/use-cases.ts +++ b/apps/sim/lib/memory/application/use-cases.ts @@ -16,10 +16,6 @@ import { type DurableSecretProvenance, mergeDurableSecretProvenance, } from '@/lib/execution/durable-secret-provenance' -import { - isDurableSecretProvenanceEnforced, - reportUnrecordedDurableProvenance, -} from '@/lib/execution/durable-secret-provenance-enforcement' import { memoryDelegationPolicy } from '@/lib/memory/application/authorization' import { memoryOperations } from '@/lib/memory/application/operations' import { lockMemoryConversationInTx } from '@/lib/memory/locks' @@ -99,7 +95,6 @@ function memoryMessageError(data: unknown): string | null { async function loadReadProvenance( records: MemoryRecord[], - scope: MemoryLegacyProvenanceScope, signal?: AbortSignal ): Promise { if (records.length === 0) return [] @@ -113,8 +108,6 @@ async function loadReadProvenance( const result: MemoryReadProvenance[] = [] const ids = [...recordsById.keys()] - const enforced = isDurableSecretProvenanceEnforced('memory') - let unrecordedCount = 0 for (let index = 0; index < ids.length; index += PRIVATE_MEMORY_QUERY_CHUNK_SIZE) { signal?.throwIfAborted() @@ -135,22 +128,11 @@ async function loadReadProvenance( status: sidecar?.status ?? null, entries: sidecar?.entries, }) - if (provenance.status === 'unknown' && !enforced) unrecordedCount += 1 result.push({ data: record.data, provenance }) } } } - if (unrecordedCount > 0) { - reportUnrecordedDurableProvenance({ - surface: 'memory', - cause: 'durable-provenance-unknown', - affectedCount: unrecordedCount, - workspaceId: scope.workspaceId, - actorUserId: scope.userId, - }) - } - return result } @@ -173,7 +155,7 @@ async function readResultProvenance( input.resolveBillingAttribution )) return { - readProvenance: await loadReadProvenance(records, provenanceScope, input.signal), + readProvenance: await loadReadProvenance(records, input.signal), provenanceScope, } } diff --git a/apps/sim/lib/memory/message-provenance.postgres.test.ts b/apps/sim/lib/memory/message-provenance.postgres.test.ts index e675e8dd343..3fa1fc2b69b 100644 --- a/apps/sim/lib/memory/message-provenance.postgres.test.ts +++ b/apps/sim/lib/memory/message-provenance.postgres.test.ts @@ -10,9 +10,8 @@ import { drizzle, type PostgresJsDatabase } from 'drizzle-orm/postgres-js' import postgres from 'postgres' import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest' -const { database, mockIsEnforced } = vi.hoisted(() => ({ +const { database } = vi.hoisted(() => ({ database: { current: undefined as PostgresJsDatabase | undefined }, - mockIsEnforced: vi.fn(), })) vi.unmock('drizzle-orm') @@ -32,10 +31,6 @@ vi.mock('@sim/db', () => ({ vi.mock('@/lib/core/security/encryption', () => ({ decryptSecret: async (value: string) => ({ decrypted: value.replace('cipher-', 'secret-') }), })) -vi.mock('@/lib/execution/durable-secret-provenance-enforcement', () => ({ - isDurableSecretProvenanceEnforced: mockIsEnforced, - reportUnrecordedDurableProvenance: vi.fn(), -})) vi.mock('@/lib/logs/execution/pii-redaction', () => ({ redactObjectStrings: async (value: unknown) => value, })) @@ -189,11 +184,10 @@ describe.skipIf(!databaseUrl)('memory provenance in PostgreSQL', () => { } }) - describe.each([false, true])('enforcement %s', (enforced) => { + describe('enforced memory provenance', () => { it('keeps a large one-secret conversation exact across tool and native writes and model reads', async () => { if (!connection) throw new Error('PostgreSQL test database is not initialized') - mockIsEnforced.mockReturnValue(enforced) - const key = `large-conversation-${enforced}` + const key = `large-conversation-strict` const messages = Array.from({ length: 17_000 }, (_, index) => ({ role: 'user', content: `secret-SHARED message-${index}`, @@ -246,9 +240,8 @@ describe.skipIf(!databaseUrl)('memory provenance in PostgreSQL', () => { 'preserves both first appends and secret bindings for %s', async (mode) => { if (!connection) throw new Error('PostgreSQL test database is not initialized') - mockIsEnforced.mockReturnValue(enforced) for (let index = 0; index < 8; index++) { - const key = `${enforced}-${mode}-${index}` + const key = `strict-${mode}-${index}` await Promise.all([ mode === 'native-native' ? nativeAppend(key, 'A') : toolAppend(key, 'A'), mode === 'tool-tool' ? toolAppend(key, 'B') : nativeAppend(key, 'B'), diff --git a/apps/sim/lib/memory/message-provenance.test.ts b/apps/sim/lib/memory/message-provenance.test.ts index af5d95e202a..554d9356bc0 100644 --- a/apps/sim/lib/memory/message-provenance.test.ts +++ b/apps/sim/lib/memory/message-provenance.test.ts @@ -9,18 +9,12 @@ import { beforeEach, describe, expect, it, vi } from 'vitest' const mocks = vi.hoisted(() => ({ decrypt: vi.fn(), - isEnforced: vi.fn(), - report: vi.fn(), logger: { error: vi.fn(), warn: vi.fn(), info: vi.fn(), debug: vi.fn() }, loadWorkspace: vi.fn(), })) vi.mock('@sim/logger', () => ({ createLogger: () => mocks.logger })) vi.mock('@/lib/core/security/encryption', () => ({ decryptSecret: mocks.decrypt })) -vi.mock('@/lib/execution/durable-secret-provenance-enforcement', () => ({ - isDurableSecretProvenanceEnforced: mocks.isEnforced, - reportUnrecordedDurableProvenance: mocks.report, -})) vi.mock('@/lib/logs/execution/pii-redaction', () => ({ redactObjectStrings: vi.fn(async (value: unknown) => value), })) @@ -125,11 +119,10 @@ function principal(): WorkflowExecutionDelegatedPrincipal { } } -describe.each([false, true])('memory message provenance with enforcement %s', (enforced) => { +describe('memory message provenance', () => { beforeEach(() => { vi.clearAllMocks() resetDbChainMock() - mocks.isEnforced.mockReturnValue(enforced) mocks.decrypt.mockResolvedValue({ decrypted: SECRET }) mocks.loadWorkspace.mockResolvedValue({ workspaceId: SCOPE.workspaceId, @@ -283,7 +276,6 @@ describe.each([false, true])('memory message provenance with enforcement %s', (e const context = executionContext() expect((await new Memory().fetchMemoryMessages(context, INPUTS))[0].content).toBe('{{TOKEN}}') expect(context.resolvedSecretTraceRegistry?.isComplete()).toBe(true) - expect(mocks.report).not.toHaveBeenCalled() expect(mocks.logger.error).toHaveBeenCalledExactlyOnceWith( 'Validated historical memory secret provenance', { @@ -400,8 +392,7 @@ describe.each([false, true])('memory message provenance with enforcement %s', (e await importDurableSecretProvenance( readerRegistry, selector.select(messages, false), - messages, - 'memory' + messages ) ).toBe(true) expect(readerRegistry.exportProvenance().entries).toHaveLength(1) @@ -454,7 +445,6 @@ describe.each([false, true])('memory message provenance with enforcement %s', (e expect(result[0].content).toBe('{{TOKEN}}') expect(execution.resolvedSecretTraceRegistry?.isComplete()).toBe(true) expect(mocks.decrypt).toHaveBeenCalledWith('corrupt-ciphertext', { logFailure: false }) - expect(mocks.report).not.toHaveBeenCalled() expect(mocks.logger.error).toHaveBeenCalledWith( 'Historical memory secret provenance could not be recovered', { diff --git a/apps/sim/lib/memory/secret-provenance.ts b/apps/sim/lib/memory/secret-provenance.ts index cc6fd560cf8..22d200ca4bf 100644 --- a/apps/sim/lib/memory/secret-provenance.ts +++ b/apps/sim/lib/memory/secret-provenance.ts @@ -120,7 +120,7 @@ export async function bindMemorySecretProvenanceToMessages( if (provenance.status === 'unknown' || provenance.entries.length === 0) return provenance const registry = new ResolvedSecretTraceRegistry() - if (!(await importDurableSecretProvenance(registry, provenance, messages, 'memory'))) { + if (!(await importDurableSecretProvenance(registry, provenance, messages))) { return { status: 'unknown' } } const entries = new Map() diff --git a/apps/sim/lib/table/rows/secret-provenance.postgres.test.ts b/apps/sim/lib/table/rows/secret-provenance.postgres.test.ts index d0a43d6558e..2fda878794d 100644 --- a/apps/sim/lib/table/rows/secret-provenance.postgres.test.ts +++ b/apps/sim/lib/table/rows/secret-provenance.postgres.test.ts @@ -6,7 +6,7 @@ * From apps/sim, run: * `TABLE_PROVENANCE_TEST_DATABASE_URL=postgresql://user@127.0.0.1:5432/postgres bun run test lib/table/rows/secret-provenance.postgres.test.ts` * CI needs a local PostgreSQL service and this variable; the default unit suite separately - * checks flag policy, stale-snapshot reporting, and write-event attribution without a database. + * checks enforcement, stale-snapshot reporting, and write-event attribution without a database. */ import { userTableRows } from '@sim/db/schema' import { loggingSessionMock } from '@sim/testing' @@ -39,15 +39,11 @@ import { cancelWorkflowGroupRuns } from '@/lib/table/workflow-columns' import { executeWorkflow } from '@/lib/workflows/executor/execute-workflow' import type { ExecutionCallbacks } from '@/executor/execution/types' -const { database, mockIsEnforced, mockReport, mockError, mockExecuteWorkflowCore } = vi.hoisted( - () => ({ - database: { current: undefined as PostgresJsDatabase | undefined }, - mockIsEnforced: vi.fn(() => false), - mockReport: vi.fn(), - mockError: vi.fn(), - mockExecuteWorkflowCore: vi.fn(), - }) -) +const { database, mockError, mockExecuteWorkflowCore } = vi.hoisted(() => ({ + database: { current: undefined as PostgresJsDatabase | undefined }, + mockError: vi.fn(), + mockExecuteWorkflowCore: vi.fn(), +})) vi.unmock('@sim/db/schema') vi.unmock('drizzle-orm') @@ -88,10 +84,6 @@ vi.mock('@/lib/workflows/executor/execution-core', () => ({ vi.mock('@/lib/workflows/executor/pause-persistence', () => ({ handlePostExecutionPauseState: vi.fn(), })) -vi.mock('@/lib/execution/durable-secret-provenance-enforcement', () => ({ - isDurableSecretProvenanceEnforced: mockIsEnforced, - reportUnrecordedDurableProvenance: mockReport, -})) const databaseUrl = process.env.TABLE_PROVENANCE_TEST_DATABASE_URL if (databaseUrl && !['localhost', '127.0.0.1', '[::1]'].includes(new URL(databaseUrl).hostname)) { @@ -238,7 +230,6 @@ describe.skipIf(!databaseUrl)('table provenance in PostgreSQL', () => { beforeEach(async () => { vi.clearAllMocks() - mockIsEnforced.mockReturnValue(false) if (!connection) throw new Error('PostgreSQL test database is not initialized') await connection.unsafe( 'TRUNCATE table_row_executions, user_table_rows, user_table_row_secret_provenance, user_table_definitions' @@ -280,7 +271,6 @@ describe.skipIf(!databaseUrl)('table provenance in PostgreSQL', () => { } it('keeps the original single-row value and secret lineage across a concurrent stamped write', async () => { - mockIsEnforced.mockReturnValue(true) await insertRow({ status: 'exact', entries: [ @@ -304,7 +294,6 @@ describe.skipIf(!databaseUrl)('table provenance in PostgreSQL', () => { }) it('captures only projected, returned rows and ignores an unknown pagination witness', async () => { - mockIsEnforced.mockReturnValue(true) await insertRow({ status: 'exact', entries: [secretEntry] }) await insertRow({ id: 'row-2', status: 'unknown' }) const reader = new TableRowProvenanceReader(scope) @@ -327,7 +316,6 @@ describe.skipIf(!databaseUrl)('table provenance in PostgreSQL', () => { }) it('captures insert provenance before a later writer changes the returned row', async () => { - mockIsEnforced.mockReturnValue(true) const reader = new TableRowProvenanceReader(scope) const row = await insertOrderedRow({ tableId: 'table-1', @@ -346,7 +334,6 @@ describe.skipIf(!databaseUrl)('table provenance in PostgreSQL', () => { }) it('returns the actual merged row and captures its lineage before the write transaction commits', async () => { - mockIsEnforced.mockReturnValue(true) await insertRow({ status: 'exact' }) if (!database.current) throw new Error('Database unavailable') const transact = database.current.transaction.bind(database.current) @@ -490,7 +477,6 @@ describe.skipIf(!databaseUrl)('table provenance in PostgreSQL', () => { ) expect(stored.entries).toEqual(baseStatus === 'exact' ? [boundEntry] : []) expect(fireTableTrigger).toHaveBeenCalledOnce() - mockIsEnforced.mockReturnValue(true) const provenance = await loadTableRowSecretProvenance( [{ id: 'row-1', updatedAt: written!.updatedAt }], scope @@ -724,43 +710,38 @@ describe.skipIf(!databaseUrl)('table provenance in PostgreSQL', () => { unrecorded: false, }, { name: 'exact-empty', fixture: { status: 'exact' }, unrecorded: false }, - ])('classifies $name explicitly under both flag settings', async ({ fixture, unrecorded }) => { - await insertRow(fixture) - for (const enforced of [false, true]) { - mockIsEnforced.mockReturnValue(enforced) - mockReport.mockClear() + ])( + 'classifies $name explicitly without a compatibility bypass', + async ({ fixture, unrecorded }) => { + await insertRow(fixture) + await expect( getTableSnapshotModelMountSafety({ tableId: 'table-1', workspaceId: 'workspace-1', rowsVersion: 7, }) - ).resolves.toBe(enforced && unrecorded ? 'unsafe-provenance' : 'safe') - expect(mockReport).toHaveBeenCalledTimes(unrecorded && !enforced ? 1 : 0) + ).resolves.toBe(unrecorded ? 'unsafe-provenance' : 'safe') } - }) + ) it.each([ { name: 'known secret entries', entries: [secretEntry] }, { name: 'malformed array', entries: [null] }, { name: 'malformed object', entries: {} }, - ])( - 'keeps $name unsafe with the flag off and does not report a proceeded read', - async ({ entries }) => { - await insertRow({ status: 'exact', entries }) - await insertRow({ id: 'unrecorded-row', status: 'unknown' }) - await expect( - getTableSnapshotModelMountSafety({ - tableId: 'table-1', - workspaceId: 'workspace-1', - rowsVersion: 7, - }) - ).resolves.toBe('unsafe-provenance') - expect(mockReport).not.toHaveBeenCalled() - } - ) + ])('keeps $name unsafe unsafe without a compatibility bypass', async ({ entries }) => { + await insertRow({ status: 'exact', entries }) + await insertRow({ id: 'unrecorded-row', status: 'unknown' }) + await expect( + getTableSnapshotModelMountSafety({ + tableId: 'table-1', + workspaceId: 'workspace-1', + rowsVersion: 7, + }) + ).resolves.toBe('unsafe-provenance') + }) - it('returns one count for a stable allowed snapshot containing several unrecorded rows', async () => { + it('refuses a stable snapshot containing several unrecorded rows', async () => { await insertRow({ id: 'missing' }) await insertRow({ id: 'unknown', status: 'unknown' }) await expect( @@ -769,13 +750,7 @@ describe.skipIf(!databaseUrl)('table provenance in PostgreSQL', () => { workspaceId: 'workspace-1', rowsVersion: 7, }) - ).resolves.toBe('safe') - expect(mockReport).toHaveBeenCalledExactlyOnceWith({ - surface: 'table-row', - cause: 'row-sidecar-not-exact', - affectedCount: 2, - workspaceId: 'workspace-1', - }) + ).resolves.toBe('unsafe-provenance') }) it('preserves legacy compatibility and records every SQL-derived unknown by cause', async () => { @@ -909,7 +884,6 @@ describe.skipIf(!databaseUrl)('table provenance in PostgreSQL', () => { ).resolves.toEqual({ version: 1, complete: true, entries: expectedEntries, scope }) } expect(mockError).not.toHaveBeenCalled() - expect(mockReport).not.toHaveBeenCalled() }) it('preserves wide bindings through derived SQL and removes only the deleted column', async () => { diff --git a/apps/sim/lib/table/rows/secret-provenance.test.ts b/apps/sim/lib/table/rows/secret-provenance.test.ts index 36dc1b8f1dd..147f73c97c8 100644 --- a/apps/sim/lib/table/rows/secret-provenance.test.ts +++ b/apps/sim/lib/table/rows/secret-provenance.test.ts @@ -6,9 +6,7 @@ import { dbChainMock, dbChainMockFns, queueTableRows, resetDbChainMock } from '@ import { eq } from 'drizzle-orm' import { beforeEach, describe, expect, it, vi } from 'vitest' -const { mockIsEnforced, mockReport, mockError } = vi.hoisted(() => ({ - mockIsEnforced: vi.fn(() => false), - mockReport: vi.fn(), +const { mockError } = vi.hoisted(() => ({ mockError: vi.fn(), })) @@ -16,12 +14,6 @@ vi.mock('@sim/logger', () => ({ createLogger: () => ({ error: mockError, warn: vi.fn() }), })) -vi.mock('@/lib/execution/durable-secret-provenance-enforcement', () => ({ - DURABLE_SECRET_PROVENANCE_SURFACES: ['memory', 'table-row', 'knowledge'], - isDurableSecretProvenanceEnforced: mockIsEnforced, - reportUnrecordedDurableProvenance: mockReport, -})) - import { PROVENANCE_MAX_SERIALIZED_BYTES } from '@/lib/execution/provenance-limits' import type { DbTransaction } from '@/lib/table/planner' import { @@ -65,7 +57,6 @@ describe('table row secret provenance', () => { beforeEach(() => { vi.clearAllMocks() resetDbChainMock() - mockIsEnforced.mockReturnValue(false) }) it('checks a version-pinned table with one aggregate rather than loading its rows', async () => { @@ -99,7 +90,6 @@ describe('table row secret provenance', () => { ).resolves.toBe('unsafe-provenance') expect(dbChainMockFns.limit).toHaveBeenCalledTimes(2) - expect(mockReport).not.toHaveBeenCalled() }) it('rejects a snapshot when the table changes during the safety check', async () => { @@ -114,35 +104,21 @@ describe('table row secret provenance', () => { rowsVersion: 7, }) ).resolves.toBe('stale') - expect(mockReport).not.toHaveBeenCalled() }) - it.each([false, true])( - 'applies the table-row enforcement policy to snapshot absences (%s)', - async (enforced) => { - mockIsEnforced.mockReturnValue(enforced) - queueTableRows(userTableDefinitions, [{ rowsVersion: 7 }]) - queueTableRows(userTableDefinitions, [{ rowsVersion: 7 }]) - queueTableRows(userTableRows, [{ unsafeCount: '0', unrecordedCount: '3' }]) - - await expect( - getTableSnapshotModelMountSafety({ - tableId: 'table-1', - workspaceId: 'workspace-1', - rowsVersion: 7, - }) - ).resolves.toBe(enforced ? 'unsafe-provenance' : 'safe') + it('refuses snapshot provenance absences', async () => { + queueTableRows(userTableDefinitions, [{ rowsVersion: 7 }]) + queueTableRows(userTableDefinitions, [{ rowsVersion: 7 }]) + queueTableRows(userTableRows, [{ unsafeCount: '0', unrecordedCount: '3' }]) - if (enforced) expect(mockReport).not.toHaveBeenCalled() - else - expect(mockReport).toHaveBeenCalledExactlyOnceWith({ - surface: 'table-row', - cause: 'row-sidecar-not-exact', - affectedCount: 3, - workspaceId: 'workspace-1', - }) - } - ) + await expect( + getTableSnapshotModelMountSafety({ + tableId: 'table-1', + workspaceId: 'workspace-1', + rowsVersion: 7, + }) + ).resolves.toBe('unsafe-provenance') + }) it('keeps untouched legacy rows readable with exact-empty provenance', async () => { queueTableRows(userTableRows, [ @@ -371,8 +347,7 @@ describe('table row secret provenance', () => { }) }) - it('fails closed for stale tracked rows once the table-row surface is enforced', async () => { - mockIsEnforced.mockReturnValue(true) + it('refuses stale tracked rows', async () => { queueTableRows(userTableRows, [ { id: 'tracked-row', @@ -398,11 +373,7 @@ describe('table row secret provenance', () => { }) }) - /** - * The shape that broke production: one unrecorded row in a page voided the whole read, and a - * page is what a `query_rows` block hands downstream, so every later model boundary refused. - */ - it('keeps a page readable when one row is unrecorded, without dropping its siblings', async () => { + it('refuses a page containing an unknown tracked row', async () => { queueTableRows(userTableRows, [ { id: 'unknown-row', @@ -442,17 +413,10 @@ describe('table row secret provenance', () => { ) ).resolves.toEqual({ version: 1, - complete: true, - entries: [{ encryptedValue: 'encrypted-local', name: 'LOCAL_SECRET' }], + complete: false, + entries: [], scope: { userId: 'user-1', workspaceId: 'workspace-1' }, }) - expect(mockReport).toHaveBeenCalledWith({ - surface: 'table-row', - cause: 'row-sidecar-not-exact', - affectedCount: 1, - workspaceId: 'workspace-1', - actorUserId: 'user-1', - }) }) it('rejects contradictory duplicate row crossings before reading provenance', async () => { diff --git a/apps/sim/lib/table/rows/secret-provenance.ts b/apps/sim/lib/table/rows/secret-provenance.ts index dbd50ee65bc..7e4a952e304 100644 --- a/apps/sim/lib/table/rows/secret-provenance.ts +++ b/apps/sim/lib/table/rows/secret-provenance.ts @@ -7,10 +7,6 @@ import { } from '@sim/db/schema' import { createLogger } from '@sim/logger' import { and, asc, eq, gt, inArray, type SQL, sql } from 'drizzle-orm' -import { - isDurableSecretProvenanceEnforced, - reportUnrecordedDurableProvenance, -} from '@/lib/execution/durable-secret-provenance-enforcement' import { SecretProvenanceBudget } from '@/lib/execution/provenance-budget' import { PROVENANCE_MAX_ENTRIES, @@ -833,16 +829,7 @@ export async function getTableSnapshotModelMountSafety(options: { } if (!counts || Number(counts.unsafeCount) > 0) return 'unsafe-provenance' - const unrecordedCount = Number(counts.unrecordedCount) - if (unrecordedCount > 0) { - if (isDurableSecretProvenanceEnforced('table-row')) return 'unsafe-provenance' - reportUnrecordedDurableProvenance({ - surface: 'table-row', - cause: 'row-sidecar-not-exact', - affectedCount: unrecordedCount, - workspaceId: options.workspaceId, - }) - } + if (Number(counts.unrecordedCount) > 0) return 'unsafe-provenance' return 'safe' } @@ -939,7 +926,6 @@ export async function loadTableRowSecretProvenance( const currentById = new Map(currentRows.map((row) => [row.id, row])) const aggregator = createStoredEntryAggregator(scope) - let unrecordedRowCount = 0 for (const rowId of rowIds) { const current = currentById.get(rowId) const crossing = crossingById.get(rowId) @@ -953,16 +939,7 @@ export async function loadTableRowSecretProvenance( current.sidecarStatus !== 'exact' || !current.sidecarIsCurrent ) { - /** - * One such row would otherwise void the whole page, and a page is what a `query_rows` block - * hands downstream — so a single row nobody recorded provenance for latched every run that - * read the table. Unenforced, the row contributes nothing, exactly like the legacy row above. - */ - if (isDurableSecretProvenanceEnforced('table-row')) { - return incomplete('row-sidecar-not-exact') - } - unrecordedRowCount += 1 - continue + return incomplete('row-sidecar-not-exact') } const parsed = normalizeStoredEntries(current.sidecarEntries) if (!parsed) return incomplete('row-sidecar-malformed') @@ -972,16 +949,6 @@ export async function loadTableRowSecretProvenance( } } - if (unrecordedRowCount > 0) { - reportUnrecordedDurableProvenance({ - surface: 'table-row', - cause: 'row-sidecar-not-exact', - affectedCount: unrecordedRowCount, - ...(scope.workspaceId ? { workspaceId: scope.workspaceId } : {}), - actorUserId: scope.userId, - }) - } - const entries = aggregator.build() if (!entries) return incomplete('row-provenance-budget-exceeded') const provenance: ResolvedSecretTraceProvenanceV1 = { diff --git a/apps/sim/lib/uploads/contexts/workspace/workspace-file-secret-provenance.test.ts b/apps/sim/lib/uploads/contexts/workspace/workspace-file-secret-provenance.test.ts index b6bdbd17523..f3459324a8a 100644 --- a/apps/sim/lib/uploads/contexts/workspace/workspace-file-secret-provenance.test.ts +++ b/apps/sim/lib/uploads/contexts/workspace/workspace-file-secret-provenance.test.ts @@ -5,17 +5,12 @@ import { workspaceFileSecretProvenance, workspaceFiles } from '@sim/db/schema' import { dbChainMock, dbChainMockFns, queueTableRows, resetDbChainMock } from '@sim/testing' import { beforeEach, describe, expect, it, vi } from 'vitest' -const { mockIsEnforced, mockReport, mockReportWrite, mockReportRefusal } = vi.hoisted(() => ({ - mockIsEnforced: vi.fn(() => false), - mockReport: vi.fn(), +const { mockReportWrite, mockReportRefusal } = vi.hoisted(() => ({ mockReportWrite: vi.fn(), mockReportRefusal: vi.fn(), })) -vi.mock('@/lib/execution/durable-secret-provenance-enforcement', () => ({ - DURABLE_SECRET_PROVENANCE_SURFACES: ['memory', 'table-row', 'knowledge', 'workspace-file'], - isDurableSecretProvenanceEnforced: mockIsEnforced, - reportUnrecordedDurableProvenance: mockReport, +vi.mock('@/lib/execution/durable-secret-provenance-telemetry', () => ({ reportDurableSecretProvenanceWrite: mockReportWrite, reportDurableSecretProvenanceRefusal: mockReportRefusal, })) @@ -77,27 +72,25 @@ describe('execution file sidecars at model boundaries', () => { status, entries, } - for (const enforced of [false, true]) { - mockIsEnforced.mockReturnValue(enforced) - queueTableRows(workspaceFiles, [row]) - expect(await isModelSafeWorkspaceFileKey(key, { workspaceId: 'workspace-1' })).toBe(safe) - queueTableRows(workspaceFiles, [row]) - expect( - await filterModelSafeWorkspaceFileAttachments([{ id: 'invented-id', key }], { - workspaceId: 'workspace-1', - }) - ).toEqual(safe ? [{ id: 'invented-id', key }] : []) - queueTableRows(workspaceFiles, [row]) - const bound = await getBoundWorkspaceFileSecretProvenance('workspace-1', { - fileId: 'canonical-id', - key, - context: 'execution', - contentUpdatedAt: CONTENT_UPDATED_AT, + + queueTableRows(workspaceFiles, [row]) + expect(await isModelSafeWorkspaceFileKey(key, { workspaceId: 'workspace-1' })).toBe(safe) + queueTableRows(workspaceFiles, [row]) + expect( + await filterModelSafeWorkspaceFileAttachments([{ id: 'invented-id', key }], { + workspaceId: 'workspace-1', }) - expect(bound.status).toBe( - version === null || (status === 'exact' && !stale) ? 'exact' : 'unknown' - ) - } + ).toEqual(safe ? [{ id: 'invented-id', key }] : []) + queueTableRows(workspaceFiles, [row]) + const bound = await getBoundWorkspaceFileSecretProvenance('workspace-1', { + fileId: 'canonical-id', + key, + context: 'execution', + contentUpdatedAt: CONTENT_UPDATED_AT, + }) + expect(bound.status).toBe( + version === null || (status === 'exact' && !stale) ? 'exact' : 'unknown' + ) } ) }) @@ -106,7 +99,6 @@ describe('workspace file secret provenance', () => { beforeEach(() => { vi.clearAllMocks() resetDbChainMock() - mockIsEnforced.mockReturnValue(false) dbChainMockFns.returning.mockResolvedValue([{ id: 'tracked-file' }]) }) @@ -501,7 +493,6 @@ describe('workspace file secret provenance', () => { * stored `unknown` above is dropped: a writer refused those bytes on purpose, which is a * different claim from nobody having recorded them, and no policy relaxes it. */ - { id: 'unrecorded-id', key: 'unrecorded-key' }, { id: 'pre-marker-sidecar-id', key: 'pre-marker-sidecar-key' }, { id: 'legacy-id', key: 'legacy-key' }, { id: 'inline-file' }, @@ -848,7 +839,7 @@ describe('workspace file secret provenance', () => { * Unrecorded says exactly what an untracked file says, and that one has always mounted. There is * nothing to import either way, so the mount proceeds and the workspace is told. */ - it('mounts an unrecorded file without importing provenance for it', async () => { + it('refuses to mount an unrecorded tracked file', async () => { const registry = { importProvenance: vi.fn(), isPermanentlyIncomplete: vi.fn().mockReturnValue(false), @@ -869,9 +860,8 @@ describe('workspace file secret provenance', () => { identity: { fileId: 'file-1', key: 'file-key', context: 'workspace' }, registry, }) - ).resolves.toBe(true) + ).resolves.toBe(false) expect(registry.importProvenance).not.toHaveBeenCalled() - expect(mockReport).toHaveBeenCalledWith(expect.objectContaining({ surface: 'workspace-file' })) }) it('rejects tracked mounted-file provenance when no complete runtime registry can import it', async () => { @@ -1608,79 +1598,7 @@ describe('workspace file secret provenance', () => { ) }) - /** - * The whole reason this surface was brought under the policy: a file that was never tracked - * already reads as safe two branches earlier, and an unrecorded one says exactly the same thing - * about its contents. Refusing only the second left files permanently unreadable for having - * tried to record provenance and failed. - */ - it('reads an unrecorded file, as it already reads an untracked one', async () => { - queueTableRows(workspaceFiles, [ - { - id: 'unrecorded-id', - key: 'unrecorded-key', - workspaceId: 'workspace-1', - context: 'workspace', - fileContentUpdatedAt: CONTENT_UPDATED_AT, - secretProvenanceVersion: 1, - provenanceContentUpdatedAt: CONTENT_UPDATED_AT, - status: 'unrecorded', - entries: [], - }, - ]) - - await expect(isModelSafeWorkspaceFileKey('unrecorded-key')).resolves.toBe(true) - expect(mockReport).toHaveBeenCalledWith( - expect.objectContaining({ surface: 'workspace-file', cause: 'durable-provenance-unknown' }) - ) - }) - - /** The audit row names who read past the absence when the caller can say; null otherwise. */ - it('carries the actor into the unrecorded-read report when the caller supplies one', async () => { - queueTableRows(workspaceFiles, [ - { - id: 'unrecorded-id', - key: 'unrecorded-key', - workspaceId: 'workspace-1', - context: 'workspace', - fileContentUpdatedAt: CONTENT_UPDATED_AT, - secretProvenanceVersion: 1, - provenanceContentUpdatedAt: CONTENT_UPDATED_AT, - status: 'unrecorded', - entries: [], - }, - ]) - - await expect( - isModelSafeWorkspaceFileKey('unrecorded-key', { actorUserId: 'user-1' }) - ).resolves.toBe(true) - expect(mockReport).toHaveBeenCalledWith(expect.objectContaining({ actorUserId: 'user-1' })) - - mockReport.mockClear() - queueTableRows(workspaceFiles, [ - { - id: 'unrecorded-id', - key: 'unrecorded-key', - workspaceId: 'workspace-1', - context: 'workspace', - fileContentUpdatedAt: CONTENT_UPDATED_AT, - secretProvenanceVersion: 1, - provenanceContentUpdatedAt: CONTENT_UPDATED_AT, - status: 'unrecorded', - entries: [], - }, - ]) - await expect(isModelSafeWorkspaceFileKey('unrecorded-key')).resolves.toBe(true) - expect(mockReport).toHaveBeenCalledWith(expect.objectContaining({ actorUserId: null })) - }) - - /** - * The row has to be a recorded absence, not a refusal. A stored `unknown` is refused whatever the - * flag says, so asserting against one would pass with enforcement off and prove nothing about the - * switch this whole posture rests on. - */ - it('refuses an unrecorded file again once the surface is closed', async () => { - mockIsEnforced.mockReturnValue(true) + it('refuses an unrecorded tracked file', async () => { queueTableRows(workspaceFiles, [ { id: 'unrecorded-id', @@ -1696,7 +1614,6 @@ describe('workspace file secret provenance', () => { ]) await expect(isModelSafeWorkspaceFileKey('unrecorded-key')).resolves.toBe(false) - expect(mockReport).not.toHaveBeenCalled() expect(mockReportRefusal).toHaveBeenCalledWith({ surface: 'workspace-file', cause: 'workspace-file-unrecorded-enforced', @@ -1769,7 +1686,7 @@ describe('createWorkspaceFileSecretProvenanceFromRegistry write decision', () => /** * A registry latched with nothing resolved is an absence, not a taint: no plaintext exists in - * the context to be in the bytes, so the file must stay readable under the unrecorded policy. + * the context to be in the bytes, so the writer records an absence instead of known taint. * Stamping taint here made one failed workflow run hard-refuse every file its chat later wrote. */ it('classifies a latched registry holding no active entries as unrecorded', async () => { diff --git a/apps/sim/lib/uploads/contexts/workspace/workspace-file-secret-provenance.ts b/apps/sim/lib/uploads/contexts/workspace/workspace-file-secret-provenance.ts index d3afdaef602..2ca4e4299f4 100644 --- a/apps/sim/lib/uploads/contexts/workspace/workspace-file-secret-provenance.ts +++ b/apps/sim/lib/uploads/contexts/workspace/workspace-file-secret-provenance.ts @@ -13,11 +13,9 @@ import { isPrivateSecretProvenanceScopeCompatible, } from '@/lib/execution/durable-secret-provenance' import { - isDurableSecretProvenanceEnforced, reportDurableSecretProvenanceRefusal, reportDurableSecretProvenanceWrite, - reportUnrecordedDurableProvenance, -} from '@/lib/execution/durable-secret-provenance-enforcement' +} from '@/lib/execution/durable-secret-provenance-telemetry' import { PROVENANCE_MAX_ENTRIES, PROVENANCE_MAX_SERIALIZED_BYTES, @@ -37,16 +35,9 @@ export const MODEL_UNSAFE_WORKSPACE_FILE_ERROR_MESSAGE = 'File cannot be sent to a model because its secret provenance is unavailable' /** - * What can be said about the secrets a file's bytes carry. - * - * The sidecar's `status` column stores all three values, and reader and storage still mean - * different things by two of them. Stored `'unrecorded'` is the writer saying nobody vouched for - * these bytes; stored `'unknown'` is a writer that knew secrets were in scope and could not map - * them. This union records what a *reader* can conclude, so a missing row, moved version, or stale - * or malformed sidecar also lands on `'unknown'` — "the writer refused" and "there is nothing - * usable to read" share a conclusion but not a stored value. Only `'unrecorded'` is an absence a - * policy may relax; it is the name the shared vocabulary already uses - * (`reportUnrecordedDurableProvenance`, the `secret_provenance.unrecorded` audit action). + * Exact sidecars identify known secret contributions. Unrecorded sidecars represent missing + * writer provenance; unknown sidecars represent taint or an invalid binding. Both refuse reads + * for tracked files, while null tracking markers retain legacy compatibility. */ export type WorkspaceFileSecretProvenance = | { status: 'exact'; entries: readonly WorkspaceFileSecretProvenanceEntry[] } @@ -272,11 +263,7 @@ export async function createWorkspaceFileSecretProvenanceFromRegistry( representations: readonly WorkspaceFileSecretProvenanceRepresentation[] = [], representationsComplete = true ): Promise { - /** - * No registry means no recorder ran, so nothing was written down about these bytes. That is an - * absence, and it is the one thing this surface's policy may relax — distinct from the taint - * every `safe: false` below produces, which a caller persists as `unknown` and no policy relaxes. - */ + /** Missing recording context is persisted separately from known taint; both refuse later reads. */ if (!registry) return { safe: true, provenance: { status: 'unrecorded' } } const sourceProvenance = registry.exportCommittedProvenanceForValue(sourceValue) const persistedProvenance = Object.is(sourceValue, persistedValue) @@ -791,10 +778,8 @@ export async function getBoundWorkspaceFileSecretProvenance( row.provenanceContentUpdatedAt?.getTime() === row.fileContentUpdatedAt.getTime() if (!bindingIsCurrent || !isValidStoredEntries(row.entries)) return { status: 'unknown' } /** - * The one shape the surface's policy may relax: a sidecar bound to this exact content recording - * that nobody vouched for it — the same statement the untracked file above makes. A stored - * `unknown` is the opposite claim, written by a writer that refused these bytes on purpose, and - * stays refused. + * Preserve the writer's absence/taint distinction for diagnostics. Readers refuse both; + * legacy compatibility is determined only by the tracking marker above. */ if (row.status === 'unrecorded') return { status: 'unrecorded' } if (row.status !== 'exact') return { status: 'unknown' } @@ -851,12 +836,6 @@ export async function getBoundWorkspaceFileSecretProvenanceByMetadata( result.set(row.id, { status: 'unknown' }) continue } - /** - * Same answer the single-file reader gives the same row. Collapsing a recorded absence into - * `unknown` here would leave two classifiers describing one policy differently, and the - * caller that eventually distinguishes them would get a different verdict depending on which - * one it happened to call. - */ if (row.status === 'unrecorded') { result.set(row.id, { status: 'unrecorded' }) continue @@ -877,43 +856,14 @@ export async function getBoundWorkspaceFileSecretProvenanceByMetadata( return result } -/** - * Whether a file nobody could vouch for may still be read. - * - * A file that was never tracked already returns exact-empty a branch earlier, and it makes exactly - * the same statement as this one: nobody recorded which secrets these bytes carry. Refusing only - * the second left a writer that momentarily could not vouch permanently worse off than one that - * never tried, with no way back — nothing rewrites a file's provenance but another content write, - * so the file simply stopped working, everywhere, for good. - * - * So it reads, and the workspace is told. Deliberately narrower than "not exact": a stale binding - * describes content that has since changed and a malformed sidecar is a fault, and neither is the - * absence this covers. Closing the surface again is a matter of naming it in - * `DURABLE_SECRET_PROVENANCE_ENFORCED_SURFACES`. - */ -export function mayReadUnrecordedWorkspaceFile( - workspaceId: string | undefined, - count = 1, - actorUserId?: string -): boolean { - if (isDurableSecretProvenanceEnforced('workspace-file')) { - reportDurableSecretProvenanceRefusal({ - surface: 'workspace-file', - cause: 'workspace-file-unrecorded-enforced', - workspaceId, - }) - return false - } - if (count > 0) { - reportUnrecordedDurableProvenance({ - surface: 'workspace-file', - cause: 'durable-provenance-unknown', - ...(count > 1 ? { affectedCount: count } : {}), - ...(workspaceId ? { workspaceId } : {}), - actorUserId: actorUserId ?? null, - }) - } - return true +/** Refuses tracked content whose writer could not record its provenance. */ +function refuseUnrecordedWorkspaceFile(workspaceId: string | undefined): false { + reportDurableSecretProvenanceRefusal({ + surface: 'workspace-file', + cause: 'workspace-file-unrecorded-enforced', + workspaceId, + }) + return false } /** Reports the canonical file identity without exposing its storage key or contents. */ @@ -946,8 +896,6 @@ export async function importWorkspaceFileSecretProvenanceForModelView(args: { registry?: ResolvedSecretTraceRegistry view: 'complete' | 'derived' | 'opaque' value?: unknown - /** Whose access authorized the read, for the unrecorded-read audit entry; null when unnameable. */ - actorUserId?: string }): Promise { const provenance = await getBoundWorkspaceFileSecretProvenance(args.workspaceId, args.identity) if (provenance.status === 'unknown') { @@ -958,7 +906,7 @@ export async function importWorkspaceFileSecretProvenanceForModelView(args: { ) } if (provenance.status === 'unrecorded') { - return mayReadUnrecordedWorkspaceFile(args.workspaceId, 1, args.actorUserId) + return refuseUnrecordedWorkspaceFile(args.workspaceId) } if (provenance.entries.length === 0) return true if (args.view === 'opaque' || !args.registry) { @@ -999,7 +947,7 @@ export async function isOpaqueWorkspaceFileEgressSafe( identity.fileId ) } - if (provenance.status === 'unrecorded') return mayReadUnrecordedWorkspaceFile(workspaceId) + if (provenance.status === 'unrecorded') return refuseUnrecordedWorkspaceFile(workspaceId) return ( provenance.entries.length === 0 || refuseWorkspaceFileProvenance( @@ -1019,8 +967,6 @@ export async function importWorkspaceFileSecretProvenanceForRuntime(args: { workspaceId: string identity: WorkspaceFileSecretProvenanceIdentity registry?: ResolvedSecretTraceRegistry - /** Whose access authorized the read, for the unrecorded-read audit entry; null when unnameable. */ - actorUserId?: string }): Promise { const provenance = await getBoundWorkspaceFileSecretProvenance(args.workspaceId, args.identity) if (provenance.status === 'unknown') { @@ -1031,7 +977,7 @@ export async function importWorkspaceFileSecretProvenanceForRuntime(args: { ) } if (provenance.status === 'unrecorded') { - return mayReadUnrecordedWorkspaceFile(args.workspaceId, 1, args.actorUserId) + return refuseUnrecordedWorkspaceFile(args.workspaceId) } if (provenance.entries.length === 0) return true if (!args.registry) { @@ -1059,7 +1005,7 @@ export async function filterModelSafeWorkspaceFileAttachments< TAttachment extends WorkspaceFileAttachmentIdentity, >( attachments: readonly TAttachment[], - options: { workspaceId?: string; actorUserId?: string } = {} + options: { workspaceId?: string } = {} ): Promise { if (attachments.length === 0) return [] if (attachments.length > PROVENANCE_MAX_ENTRIES) { @@ -1098,24 +1044,19 @@ export async function filterModelSafeWorkspaceFileAttachments< return false } unrecorded += 1 - return !isDurableSecretProvenanceEnforced('workspace-file') + return false }) if (refused > 0) { refuseWorkspaceFileProvenance('workspace-file-provenance-unavailable', options.workspaceId) } /** One report for the whole set of attachments, which is one read, rather than one per file. */ if (unrecorded > 0) { - mayReadUnrecordedWorkspaceFile(options.workspaceId, unrecorded, options.actorUserId) + refuseUnrecordedWorkspaceFile(options.workspaceId) } return kept } -/** - * Classifies one row without deciding it. `unrecorded` is kept apart from `unsafe` because only the - * first is the same statement an untracked file makes, and only the first is the surface's policy - * to relax — a stale binding describes content that has since changed, and a malformed sidecar is a - * fault no policy relaxes. - */ +/** Keeps missing writer provenance distinct from taint for refusal diagnostics. */ type ModelSafeWorkspaceFileClassification = 'safe' | 'unrecorded' | 'unsafe' function classifyModelSafeWorkspaceFileRow( @@ -1174,7 +1115,7 @@ async function loadModelSafeWorkspaceFileRows( */ export async function isModelSafeWorkspaceFileKey( key: string, - options: { workspaceId?: string; actorUserId?: string } = {} + options: { workspaceId?: string } = {} ): Promise { return areModelSafeWorkspaceFileKeys([key], options) } @@ -1186,7 +1127,7 @@ export async function isModelSafeWorkspaceFileKey( */ export async function areModelSafeWorkspaceFileKeys( keys: readonly string[], - options: { workspaceId?: string; actorUserId?: string } = {} + options: { workspaceId?: string } = {} ): Promise { const uniqueKeys = [...new Set(keys.filter((key) => key.length > 0))] if (uniqueKeys.length === 0) return true @@ -1215,8 +1156,5 @@ export async function areModelSafeWorkspaceFileKeys( if (classification === 'unrecorded') unrecorded += 1 } /** One report for the batch, not one per key: a caller checking many keys is one read. */ - return ( - unrecorded === 0 || - mayReadUnrecordedWorkspaceFile(options.workspaceId, unrecorded, options.actorUserId) - ) + return unrecorded === 0 || refuseUnrecordedWorkspaceFile(options.workspaceId) } From ba8e6fbe2aefd52938ab6ad932a1fa1bf6932ee0 Mon Sep 17 00:00:00 2001 From: Vikhyath Mondreti Date: Thu, 17 Sep 2026 12:33:30 -0700 Subject: [PATCH 2/2] fix(provenance): mark legacy attachment replay fixtures --- .../sim/executor/handlers/agent/agent-handler.test.ts | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/apps/sim/executor/handlers/agent/agent-handler.test.ts b/apps/sim/executor/handlers/agent/agent-handler.test.ts index b5b61ec6c1d..85d14bd6e27 100644 --- a/apps/sim/executor/handlers/agent/agent-handler.test.ts +++ b/apps/sim/executor/handlers/agent/agent-handler.test.ts @@ -394,7 +394,10 @@ describe('AgentBlockHandler', () => { expect(inputs).toEqual(original) queueTableRows(schemaMock.memory, [ - { data: [...stored, { role: 'assistant', content: 'First answer' }] }, + { + secretProvenanceVersion: null, + data: [...stored, { role: 'assistant', content: 'First answer' }], + }, ]) mockGetProviderFromModel.mockReturnValue('anthropic') const nextContext = { ...mockContext, executionId: 'exec-2' } @@ -428,6 +431,7 @@ describe('AgentBlockHandler', () => { mockGetProviderFromModel.mockReturnValue('openai') queueTableRows(schemaMock.memory, [ { + secretProvenanceVersion: null, data: [ { role: 'user', content: 'Analyze this file', executionId: 'exec-1', files: [file] }, ], @@ -448,7 +452,9 @@ describe('AgentBlockHandler', () => { it('saves a new attachment appended to an existing conversation', async () => { mockGetProviderFromModel.mockReturnValue('openai') - queueTableRows(schemaMock.memory, [{ data: [{ role: 'assistant', content: 'Hello' }] }]) + queueTableRows(schemaMock.memory, [ + { secretProvenanceVersion: null, data: [{ role: 'assistant', content: 'Hello' }] }, + ]) await handler.execute({ ...mockContext, executionId: 'exec-2' }, mockBlock, { model: 'gpt-4o', memoryType: 'conversation', @@ -468,6 +474,7 @@ describe('AgentBlockHandler', () => { const hydrate = vi.spyOn(userFileBase64, 'hydrateUserFilesWithBase64') queueTableRows(schemaMock.memory, [ { + secretProvenanceVersion: null, data: [ { role: 'user', content: 'Old file', files: [file] }, { role: 'assistant', content: 'Recent answer' },