improvement(provenance): enforce tracked durable reads

apps/docs/content/docs/platform/self-hosting/environment-variables.mdx

@@ -234,9 +234,10 @@ width. Check what you pick there, or upgrade Ollama.
 | `COPILOT_API_KEY` | API key for Chat. Without it the Sim Chat block, scheduled prompt jobs, and Inbox cannot run |
 | `NEXT_PUBLIC_CHAT_DISABLED` | Set to `true` to hide the Chat module: the workspace lands on your first workflow, with no chats list, scheduled tasks, or editor Chat panel. Chat is shown when unset; `npx sim-setup` sets it for you if you skip the chat key |
 | `PII_URL` | Base URL of the Presidio service backing PII detection and redaction. The Helm chart wires it to its own `pii` Service when `pii.enabled`; on Compose point it at the PII service on your network. The default `http://localhost:5001` exists only in local development, and leaving it makes redaction fail |
-| `DURABLE_SECRET_PROVENANCE_ENFORCED_SURFACES` | Durable stores where a value whose secret provenance was never recorded fails the run instead of logging a warning. `all`, or a comma-separated subset of `memory`, `table-row`, `knowledge`, `workspace-file`. Unset (nothing enforced) by default |
 | `ADMIN_API_KEY` | Admin API key for GitOps operations and organization provisioning |
 
+Tracked memory, table rows, knowledge content, and workspace files require valid secret provenance before entering a model or a trusted runtime. Records with a null provenance tracking marker retain legacy compatibility.
+
 ## Enterprise Features
 
 Enterprise features are unlocked by configuration rather than billing on self-hosted deployments. One switch turns on the full set; per-feature flags below it override the switch either way.

apps/sim/app/api/v2/knowledge/search/route.provenance.test.ts

@@ -107,10 +107,6 @@ import {
 } from '@sim/testing'
 import { NextRequest } from 'next/server'
 import { env } from '@/lib/core/config/env'
-import {
-  isDurableSecretProvenanceEnforced,
-  resetDurableSecretProvenanceEnforcementCache,
-} from '@/lib/execution/durable-secret-provenance-enforcement'
 import { createKnowledgeDocumentSourceValue } from '@/lib/knowledge/secret-provenance'
 import { POST } from '@/app/api/v2/knowledge/search/route'
 import { ResolvedSecretTraceRegistry } from '@/executor/utils/resolved-secret-trace-registry'
@@ -211,12 +207,6 @@ function providerPayload() {
   return JSON.parse(provider.fetch.mock.calls[0][1].body)
 }
 
-function enforceKnowledge(enforced: boolean) {
-  env.DURABLE_SECRET_PROVENANCE_ENFORCED_SURFACES = enforced ? 'all' : ''
-  resetDurableSecretProvenanceEnforcementCache()
-  expect(isDurableSecretProvenanceEnforced('knowledge')).toBe(enforced)
-}
-
 async function requestSearch(overrides: Partial<typeof requestInput> = {}) {
   return POST(
     new NextRequest('http://localhost/api/v2/knowledge/search', {
@@ -230,7 +220,6 @@ async function requestSearch(overrides: Partial<typeof requestInput> = {}) {
 beforeEach(() => {
   vi.clearAllMocks()
   resetDbChainMock()
-  enforceKnowledge(true)
   env.COHERE_API_KEY = 'synthetic-cohere-key'
   provider.decrypt.mockResolvedValue({ decrypted: SECRET })
   provider.fetch.mockResolvedValue(
@@ -264,26 +253,22 @@ beforeEach(() => {
 
 /** The route, use case, sidecar binding/import, registry, projection and provider request builder are real. */
 describe('Knowledge search provenance through the V2 route and reranker HTTP boundary', () => {
-  it.each([false, true])(
-    'redacts current known-secret chunks with enforcement=%s',
-    async (enforced) => {
-      enforceKnowledge(enforced)
-      seedSidecar('exact')
-      const response = await requestSearch()
-      const body = await response.json()
-      expect(response.status).toBe(200)
-      expect(body.data.rerankerStatus).toBe('applied')
-      expect(providerPayload().documents).toEqual([CONTENT.replace(SECRET, '{{TOKEN}}')])
-      expect(body.data.results[0].content).toBe(CONTENT)
-      expect(provider.decrypt).toHaveBeenCalledWith('synthetic-encrypted-token')
-      expect(mocks.generateEmbedding).toHaveBeenCalledWith(
-        requestInput.query,
-        expect.anything(),
-        'workspace-1',
-        undefined
-      )
-    }
-  )
+  it('redacts current known-secret chunks', async () => {
+    seedSidecar('exact')
+    const response = await requestSearch()
+    const body = await response.json()
+    expect(response.status).toBe(200)
+    expect(body.data.rerankerStatus).toBe('applied')
+    expect(providerPayload().documents).toEqual([CONTENT.replace(SECRET, '{{TOKEN}}')])
+    expect(body.data.results[0].content).toBe(CONTENT)
+    expect(provider.decrypt).toHaveBeenCalledWith('synthetic-encrypted-token')
+    expect(mocks.generateEmbedding).toHaveBeenCalledWith(
+      requestInput.query,
+      expect.anything(),
+      'workspace-1',
+      undefined
+    )
+  })
 
   it('does not assign a billing owner secret name to a workspace-key caller', async () => {
     seedSidecar('exact')
@@ -314,7 +299,7 @@ describe('Knowledge search provenance through the V2 route and reranker HTTP bou
   })
 
   it.each(['unknown', 'missing', 'stale', 'malformed'] as const)(
-    'refuses %s tracked provenance before provider HTTP when enforcement is enabled',
+    'refuses %s tracked provenance before provider HTTP',
     async (status) => {
       seedSidecar(status)
       const response = await requestSearch()
@@ -326,27 +311,12 @@ describe('Knowledge search provenance through the V2 route and reranker HTTP bou
     }
   )
 
-  it.each(['unknown', 'missing', 'stale', 'malformed'] as const)(
-    'preserves existing flag-off compatibility for %s sidecars',
-    async (status) => {
-      enforceKnowledge(false)
-      seedSidecar(status)
-      const response = await requestSearch()
-      expect(response.status).toBe(200)
-      expect(providerPayload().documents).toEqual([CONTENT])
-    }
-  )
-
-  it.each([false, true])(
-    'keeps pre-tracking NULL rows readable with enforcement=%s',
-    async (enforced) => {
-      enforceKnowledge(enforced)
-      seedSidecar('legacy')
-      const response = await requestSearch()
-      expect(response.status).toBe(200)
-      expect(providerPayload().documents).toEqual([CONTENT])
-    }
-  )
+  it('keeps pre-tracking NULL rows readable', async () => {
+    seedSidecar('legacy')
+    const response = await requestSearch()
+    expect(response.status).toBe(200)
+    expect(providerPayload().documents).toEqual([CONTENT])
+  })
 
   it('does not subject a raw public read without reranking to durable-model enforcement', async () => {
     seedSidecar('unknown')

apps/sim/executor/handlers/agent/agent-handler.test.ts

@@ -394,7 +394,10 @@ describe('AgentBlockHandler', () => {
         expect(inputs).toEqual(original)
 
         queueTableRows(schemaMock.memory, [
-          { data: [...stored, { role: 'assistant', content: 'First answer' }] },
+          {
+            secretProvenanceVersion: null,
+            data: [...stored, { role: 'assistant', content: 'First answer' }],
+          },
         ])
         mockGetProviderFromModel.mockReturnValue('anthropic')
         const nextContext = { ...mockContext, executionId: 'exec-2' }
@@ -428,6 +431,7 @@ describe('AgentBlockHandler', () => {
       mockGetProviderFromModel.mockReturnValue('openai')
       queueTableRows(schemaMock.memory, [
         {
+          secretProvenanceVersion: null,
           data: [
             { role: 'user', content: 'Analyze this file', executionId: 'exec-1', files: [file] },
           ],
@@ -448,7 +452,9 @@ describe('AgentBlockHandler', () => {
 
     it('saves a new attachment appended to an existing conversation', async () => {
       mockGetProviderFromModel.mockReturnValue('openai')
-      queueTableRows(schemaMock.memory, [{ data: [{ role: 'assistant', content: 'Hello' }] }])
+      queueTableRows(schemaMock.memory, [
+        { secretProvenanceVersion: null, data: [{ role: 'assistant', content: 'Hello' }] },
+      ])
       await handler.execute({ ...mockContext, executionId: 'exec-2' }, mockBlock, {
         model: 'gpt-4o',
         memoryType: 'conversation',
@@ -468,6 +474,7 @@ describe('AgentBlockHandler', () => {
       const hydrate = vi.spyOn(userFileBase64, 'hydrateUserFilesWithBase64')
       queueTableRows(schemaMock.memory, [
         {
+          secretProvenanceVersion: null,
           data: [
             { role: 'user', content: 'Old file', files: [file] },
             { role: 'assistant', content: 'Recent answer' },

apps/sim/executor/handlers/agent/memory.test.ts

@@ -1,18 +1,9 @@
 import { loggerMock, queueTableRows, resetDbChainMock, schemaMock } from '@sim/testing'
 import { beforeEach, describe, expect, it, vi } from 'vitest'
 
-const { mockDecryptSecret, mockRedactObjectStrings, mockIsEnforced, mockReportUnrecorded } =
-  vi.hoisted(() => ({
-    mockDecryptSecret: vi.fn(),
-    mockRedactObjectStrings: vi.fn(async (value: unknown) => value),
-    mockIsEnforced: vi.fn(() => false),
-    mockReportUnrecorded: vi.fn(),
-  }))
-
-vi.mock('@/lib/execution/durable-secret-provenance-enforcement', () => ({
-  DURABLE_SECRET_PROVENANCE_SURFACES: ['memory', 'table-row', 'knowledge'],
-  isDurableSecretProvenanceEnforced: mockIsEnforced,
-  reportUnrecordedDurableProvenance: mockReportUnrecorded,
+const { mockDecryptSecret, mockRedactObjectStrings } = vi.hoisted(() => ({
+  mockDecryptSecret: vi.fn(),
+  mockRedactObjectStrings: vi.fn(async (value: unknown) => value),
 }))
 
 vi.mock('@/lib/core/security/encryption', () => ({
@@ -56,7 +47,6 @@ describe('Memory', () => {
   beforeEach(() => {
     vi.clearAllMocks()
     resetDbChainMock()
-    mockIsEnforced.mockReturnValue(false)
     mockDecryptSecret.mockImplementation(async (encryptedValue: string) => ({
       decrypted: `decrypted:${encryptedValue}`,
     }))
@@ -313,6 +303,7 @@ describe('Memory', () => {
       async ({ provider, render }) => {
         queueTableRows(schemaMock.memory, [
           {
+            secretProvenanceVersion: null,
             data: [
               {
                 role: 'user',
@@ -359,7 +350,7 @@ describe('Memory', () => {
         context: 'execution',
       }
       queueTableRows(schemaMock.memory, [
-        { data: [{ role: 'user', content: 'File', files: [file] }] },
+        { secretProvenanceVersion: null, data: [{ role: 'user', content: 'File', files: [file] }] },
       ])
       const context = {
         workspaceId: 'workspace-1',
@@ -388,6 +379,7 @@ describe('Memory', () => {
     it('bounds historical file loading even when the messages contain no text', async () => {
       queueTableRows(schemaMock.memory, [
         {
+          secretProvenanceVersion: null,
           data: Array.from({ length: MEMORY.MAX_REPLAY_FILE_REFERENCES + 1 }, () => ({
             role: 'user',
             content: '',
@@ -406,6 +398,7 @@ describe('Memory', () => {
     it('does not carry inline-only or malformed file objects into a later turn', async () => {
       queueTableRows(schemaMock.memory, [
         {
+          secretProvenanceVersion: null,
           data: [
             {
               role: 'user',
@@ -695,33 +688,7 @@ describe('Memory', () => {
       expect(mockDecryptSecret).not.toHaveBeenCalled()
     })
 
-    /** Trace 2's shape: a stored memory a previous run could not vouch for. */
-    it('reads a memory with unrecorded provenance while the surface stays open', async () => {
-      const registry = new ResolvedSecretTraceRegistry([], {
-        userId: 'user-1',
-        workspaceId: 'workspace-1',
-      })
-      vi.spyOn(memoryService as any, 'fetchMemory').mockResolvedValueOnce({
-        messages: [{ role: 'user', content: 'how do i see my tickets?' }],
-        provenance: { status: 'unknown' },
-      })
-
-      const messages = await memoryService.fetchMemoryMessages(
-        createContext(registry) as never,
-        inputs
-      )
-
-      expect(messages).toEqual([{ role: 'user', content: 'how do i see my tickets?' }])
-      expect(registry.isPermanentlyIncomplete()).toBe(false)
-      expect(mockReportUnrecorded).toHaveBeenCalledWith({
-        surface: 'memory',
-        cause: 'stored-memory-provenance-unknown',
-        workspaceId: 'workspace-1',
-      })
-    })
-
-    it('refuses that same memory once the memory surface is closed', async () => {
-      mockIsEnforced.mockReturnValue(true)
+    it('refuses tracked memory with unknown provenance', async () => {
       const registry = new ResolvedSecretTraceRegistry([], {
         userId: 'user-1',
         workspaceId: 'workspace-1',
@@ -734,7 +701,6 @@ describe('Memory', () => {
       await expect(
         memoryService.fetchMemoryMessages(createContext(registry) as never, inputs)
       ).rejects.toThrow()
-      expect(mockReportUnrecorded).not.toHaveBeenCalled()
     })
   })
 

apps/sim/executor/handlers/agent/memory.ts

@@ -11,10 +11,6 @@ import {
   importDurableSecretProvenance,
   mergeDurableSecretProvenance,
 } from '@/lib/execution/durable-secret-provenance'
-import {
-  isDurableSecretProvenanceEnforced,
-  reportUnrecordedDurableProvenance,
-} from '@/lib/execution/durable-secret-provenance-enforcement'
 import { mergeFileKeys } from '@/lib/execution/payloads/access-keys'
 import { redactObjectStrings } from '@/lib/logs/execution/pii-redaction'
 import { lockMemoryConversationInTx } from '@/lib/memory/locks'
@@ -101,21 +97,16 @@ export class Memory {
       const staged = new ResolvedSecretTraceRegistry([], scope, { staged: true })
       staged.mergeToolCallRegistry(ctx.resolvedSecretTraceRegistry)
       includeRecovered =
-        (await importDurableSecretProvenance(
-          staged,
-          selection.select(messages, true),
-          messages,
-          'memory'
-        )) && staged.getModelEgressSnapshot().complete
+        (await importDurableSecretProvenance(staged, selection.select(messages, true), messages)) &&
+        staged.getModelEgressSnapshot().complete
       if (includeRecovered) {
         for (const message of messages) {
           const stagedMessage = new ResolvedSecretTraceRegistry([], scope, { staged: true })
           if (
             !(await importDurableSecretProvenance(
               stagedMessage,
               selection.select([message], true),
-              message,
-              'memory'
+              message
             )) ||
             !stagedMessage.getModelEgressSnapshot().complete
           ) {
@@ -146,31 +137,15 @@ export class Memory {
     const selectProvenance = (values: readonly unknown[]) =>
       selection.select(values, includeRecovered)
     const selectedProvenance = selectProvenance(messages)
-    /**
-     * Unrecorded provenance is checked through the same policy the shared import uses, so stored
-     * memory written by a run that could not vouch does not permanently refuse every later turn.
-     */
-    let refuseStoredProvenance: boolean
-    if (selectedProvenance.status === 'unknown') {
-      refuseStoredProvenance = isDurableSecretProvenanceEnforced('memory')
-      if (!refuseStoredProvenance) {
-        reportUnrecordedDurableProvenance({
-          surface: 'memory',
-          cause: 'stored-memory-provenance-unknown',
-          ...(ctx.workspaceId ? { workspaceId: ctx.workspaceId } : {}),
-        })
-      }
-    } else {
-      refuseStoredProvenance =
-        (selectedProvenance.entries.length > 0 && !ctx.resolvedSecretTraceRegistry) ||
-        (ctx.resolvedSecretTraceRegistry !== undefined &&
-          !(await importDurableSecretProvenance(
-            ctx.resolvedSecretTraceRegistry,
-            selectedProvenance,
-            messages,
-            'memory'
-          )))
-    }
+    const refuseStoredProvenance =
+      selectedProvenance.status === 'unknown' ||
+      (selectedProvenance.entries.length > 0 && !ctx.resolvedSecretTraceRegistry) ||
+      (ctx.resolvedSecretTraceRegistry !== undefined &&
+        !(await importDurableSecretProvenance(
+          ctx.resolvedSecretTraceRegistry,
+          selectedProvenance,
+          messages
+        )))
     if (refuseStoredProvenance) {
       refuseResolvedSecretProjection({
         site: 'memory.storedProvenanceImport',
@@ -187,14 +162,7 @@ export class Memory {
           [],
           ctx.resolvedSecretTraceRegistry?.exportProvenance().scope
         )
-        if (
-          !(await importDurableSecretProvenance(
-            modelRegistry,
-            messageProvenance,
-            message,
-            'memory'
-          ))
-        ) {
+        if (!(await importDurableSecretProvenance(modelRegistry, messageProvenance, message))) {
           refuseResolvedSecretProjection({
             site: 'memory.messageProvenanceImport',
             message: MEMORY_CONTENT_REFUSAL,

apps/sim/lib/copilot/tools/handlers/vfs.ts

@@ -130,7 +130,6 @@ async function canReturnWorkspaceFileValue(
         registry: context.resolvedSecretTraceRegistry,
         view: provenanceView,
         value,
-        actorUserId: context.userId,
       }))
     ) {
       return false

apps/sim/lib/core/config/env.ts

@@ -189,7 +189,6 @@ export const env = createEnv({
     BILLING_CONCURRENCY_LIMIT_ENTERPRISE:   z.string().optional(),                  // In-flight executions per Enterprise billing account (metadata-overridable)
     BILLING_ENABLED:                       z.boolean().optional(),                 // Enable billing enforcement and usage tracking
     TRIGGER_EU_REGION:                     z.boolean().optional(),                 // Route Trigger.dev runs to eu-central-1 instead of the default us-east-1 (fallback for the trigger-eu-region flag when AppConfig is not the source of truth)
-    DURABLE_SECRET_PROVENANCE_ENFORCED_SURFACES: z.string().optional(),            // Durable surfaces where unrecorded secret provenance fails the run instead of logging a warning: "all", or a comma-separated subset of memory,table-row,knowledge,workspace-file (default: none enforced)
 
     // Table feature limits (per plan). Apply when billing is disabled (free tier defaults) or for billed plans.
     FREE_TABLES_LIMIT:                     z.number().optional(),                  // Max user tables per workspace on free tier (default: 5)