improvement(knowledge): decide mirrored-permission freshness from the document row

Every ACL writer now records `document.acl_valid_until`: admin mode a fixed window
after its verification evidence, members mode the earliest expiry among the
observations its ACL was materialized from, and NULL where no source evidence
applies. Behind the `knowledge-acl-expiry` flag the read predicate compares that
column instead of re-deriving freshness per candidate through the connector's
members and observations, which costs a lookup per document examined.

The stored expiry is never later than the evidence behind it, so the flagged
predicate can hide a document the per-candidate proof would still admit and can
never admit one it would refuse. Script migration 0020 backfills existing rows.

Author: waleedlatif1

apps/sim/lib/core/config/env.ts

@@ -630,6 +630,7 @@ export const env = createEnv({
     CREDENTIAL_GROUPS:                    z.boolean().optional(),                 // Enable enterprise Credential Groups globally
     KNOWLEDGE_MEMBER_ACCESS:              z.boolean().optional(),                 // Enable per-member knowledge connectors and hybrid-by-default retrieval globally
     KNOWLEDGE_TIN_KEYWORD:                z.boolean().optional(),                 // Rank large-scope keyword retrieval through the Tin text index where it exists
+    KNOWLEDGE_ACL_EXPIRY:                 z.boolean().optional(),                 // Read mirrored-permission freshness from document.acl_valid_until instead of per-candidate evidence
 
     // Organizations - for self-hosted deployments
     ORGANIZATIONS_ENABLED:                 z.boolean().optional(),                 // Enable organizations on self-hosted (bypasses plan requirements)

apps/sim/lib/core/config/feature-flags.test.ts

@@ -177,6 +177,19 @@ describe('isFeatureEnabled', () => {
     })
   })
 
+  describe('knowledge-acl-expiry flag', () => {
+    it('is a global switch', async () => {
+      expect(await isFeatureEnabled('knowledge-acl-expiry')).toBe(false)
+      envRef.KNOWLEDGE_ACL_EXPIRY = true
+      expect(await isFeatureEnabled('knowledge-acl-expiry')).toBe(true)
+    })
+
+    it('follows an AppConfig global rule', async () => {
+      withAppConfig({ 'knowledge-acl-expiry': { enabled: true } })
+      expect(await isFeatureEnabled('knowledge-acl-expiry')).toBe(true)
+    })
+  })
+
   describe('knowledge-member-access flag', () => {
     it('uses a global fallback switch off AppConfig', async () => {
       expect(await isFeatureEnabled('knowledge-member-access')).toBe(false)

apps/sim/lib/core/config/feature-flags.ts

@@ -93,6 +93,15 @@ const FEATURE_FLAGS = {
       'KNOWLEDGE_MEMBER_ACCESS.',
     fallback: 'KNOWLEDGE_MEMBER_ACCESS',
   },
+  'knowledge-acl-expiry': {
+    description:
+      'Read mirrored-permission freshness from document.acl_valid_until, written by every ACL ' +
+      'writer, instead of re-deriving it per candidate from verification and observation ' +
+      'timestamps. Strictly no wider than the per-candidate proof: a stored expiry is never ' +
+      'later than the evidence behind it. Requires the 0020 backfill. Off-AppConfig falls back ' +
+      'to KNOWLEDGE_ACL_EXPIRY.',
+    fallback: 'KNOWLEDGE_ACL_EXPIRY',
+  },
   'knowledge-tin-keyword': {
     description:
       'Rank keyword retrieval for members whose permitted set is too large to enumerate through ' +

apps/sim/lib/knowledge/__integration__/migration-fixture.ts

@@ -56,7 +56,7 @@ export async function createEnterpriseSearchMigrationFixture(databaseUrl: string
     CREATE TABLE document (
       id text PRIMARY KEY, external_id text, connector_id text, knowledge_base_id text,
       tag1 text, tag2 text, tag3 text, tag4 text, tag5 text, tag6 text, tag7 text,
-      acl text[] NOT NULL DEFAULT '{ws}', storage_key text,
+      acl text[] NOT NULL DEFAULT '{ws}', storage_key text, acl_valid_until timestamp,
       user_excluded boolean NOT NULL DEFAULT false, archived_at timestamp
     );
     CREATE INDEX doc_connector_id_idx ON document(connector_id);

apps/sim/lib/knowledge/access/freshness.ts

@@ -1,2 +1,14 @@
+import { type SQL, sql } from 'drizzle-orm'
+
 /** Source authorization evidence expires even when its sync scheduler has stopped. */
 export const SOURCE_ACL_MAX_AGE_MS = 24 * 60 * 60 * 1000
+
+/** How long evidence recorded at `recordedAt` stays current, as a SQL expression. */
+export function aclValidUntilFrom(recordedAt: SQL): SQL {
+  return sql`(${recordedAt}) + (${SOURCE_ACL_MAX_AGE_MS} * interval '1 millisecond')`
+}
+
+/** Evidence older than this is no longer current. */
+export function aclFreshnessCutoff(): SQL {
+  return sql`statement_timestamp() - (${SOURCE_ACL_MAX_AGE_MS} * interval '1 millisecond')`
+}

apps/sim/lib/knowledge/access/predicate.postgres.test.ts

@@ -21,6 +21,7 @@ vi.mock('@/connectors/registry.server', () => ({ CONNECTOR_REGISTRY: {} }))
 const { drizzle } = await import('drizzle-orm/postgres-js')
 const schema = await import('@sim/db/schema')
 const { persistDocumentAcls } = await import('@/lib/knowledge/connectors/sync-persistence')
+const { materializeDocumentAcls } = await import('@/lib/knowledge/connectors/member-observations')
 const { mergeMirroredAcls, hideUnlistedDocuments } = await import(
   '@/lib/knowledge/connectors/mirrored-acls'
 )
@@ -87,7 +88,8 @@ describe.runIf(Boolean(databaseUrl))('knowledge ACLs in PostgreSQL', () => {
     join = false,
     githubInstallationGrants?: GitHubInstallationReadGrant[],
     userId = 'reader',
-    confluenceSiteGrants?: ConfluenceSiteReadGrant[]
+    confluenceSiteGrants?: ConfluenceSiteReadGrant[],
+    storedAclFreshness = false
   ): Promise<boolean> {
     const query = new PgDialect().sqlToQuery(
       knowledgeAccessCondition({
@@ -96,6 +98,7 @@ describe.runIf(Boolean(databaseUrl))('knowledge ACLs in PostgreSQL', () => {
         tokens,
         githubInstallationGrants,
         confluenceSiteGrants,
+        storedAclFreshness,
       })
     )
     const values = query.params.map((value: unknown) => {
@@ -495,6 +498,75 @@ describe.runIf(Boolean(databaseUrl))('knowledge ACLs in PostgreSQL', () => {
     expect(await readable(['ws'], 'upload')).toBe(true)
   })
 
+  /**
+   * The stored expiry every ACL writer records must decide mirrored freshness exactly as the
+   * per-candidate proof does, except where it is deliberately stricter: a members-mode ACL expires
+   * with the earliest observation it names, so one stale observer hides the document from readers
+   * whose own observation is still current, until the sweep drops it and re-materializes.
+   */
+  it('decides mirrored freshness from the row, never admitting what per-candidate evidence refuses', async () => {
+    const proof = (tokens: string[], id: string) => readable(tokens, id)
+    const stored = (tokens: string[], id: string) =>
+      readable(tokens, id, false, undefined, 'reader', undefined, true)
+
+    await putDocument('admin-fresh', ['u:alice@corp.com'])
+    await connection.unsafe(
+      "UPDATE document SET acl_valid_until = acl_verified_at + interval '24 hours' WHERE id = 'admin-fresh'"
+    )
+    await putDocument('admin-stale', ['u:alice@corp.com'])
+    await connection.unsafe(`UPDATE document SET acl_verified_at = statement_timestamp() - interval '25 hours',
+      acl_valid_until = statement_timestamp() - interval '1 hour' WHERE id = 'admin-stale'`)
+    await putDocument('admin-unbackfilled', ['u:alice@corp.com'])
+    await connection.unsafe(
+      "UPDATE document SET acl_valid_until = NULL WHERE id = 'admin-unbackfilled'"
+    )
+
+    await connection.unsafe(
+      `INSERT INTO knowledge_connector_member(id, workspace_id, connector_id, subject_token, status) VALUES
+      ('alice', 'workspace', 'members', $1, 'active'), ('bob', 'workspace', 'members', $2, 'active')`,
+      [alice, bob]
+    )
+    const putMembers = async (id: string, seen: string[]) => {
+      await connection.unsafe(
+        "INSERT INTO document(id, connector_id, acl) VALUES ($1, 'members', string_to_array($2, ','))",
+        [id, [alice, bob].join(',')]
+      )
+      await connection.unsafe(
+        `INSERT INTO knowledge_document_observation VALUES
+         ($1, 'alice', statement_timestamp() - ($2)::interval),
+         ($1, 'bob', statement_timestamp() - ($3)::interval)`,
+        [id, seen[0], seen[1]]
+      )
+      /** The real writer decides both the ACL and its expiry, so this cannot drift from it. */
+      await materializeDocumentAcls('members', [id], drizzle(connection, { schema }))
+    }
+    await putMembers('members-fresh', ['1 hour', '2 hours'])
+    await putMembers('members-stale', ['25 hours', '26 hours'])
+    await putMembers('members-mixed', ['1 hour', '26 hours'])
+
+    for (const [id, tokens, expected] of [
+      ['admin-fresh', ['u:alice@corp.com'], true],
+      ['admin-stale', ['u:alice@corp.com'], false],
+      ['members-fresh', [alice], true],
+      ['members-stale', [alice], false],
+    ] as const) {
+      expect(await proof([...tokens], id)).toBe(expected)
+      expect(await stored([...tokens], id)).toBe(expected)
+    }
+    /** Stricter, never wider: the fresh observer waits for the sweep. */
+    expect(await proof([alice], 'members-mixed')).toBe(true)
+    expect(await stored([alice], 'members-mixed')).toBe(false)
+    expect(await proof([bob], 'members-mixed')).toBe(false)
+    expect(await stored([bob], 'members-mixed')).toBe(false)
+    /** A row the backfill has not reached yet is unreadable rather than assumed current. */
+    expect(await proof(['u:alice@corp.com'], 'admin-unbackfilled')).toBe(true)
+    expect(await stored(['u:alice@corp.com'], 'admin-unbackfilled')).toBe(false)
+    /** Uploads and workspace-mode documents carry no expiry and are unaffected. */
+    await connection.unsafe("INSERT INTO document(id) VALUES ('upload')")
+    expect(await proof(['ws'], 'upload')).toBe(true)
+    expect(await stored(['ws'], 'upload')).toBe(true)
+  })
+
   it('does not let one member refresh another member’s stale observation', async () => {
     await connection.unsafe(
       "INSERT INTO document(id, connector_id, acl) VALUES ('shared', 'members', string_to_array($1, ','))",
@@ -545,6 +617,17 @@ describe.runIf(Boolean(databaseUrl))('knowledge ACLs in PostgreSQL', () => {
       "SELECT jsonb_typeof(acl_requirements) AS shape, acl_requirements FROM document WHERE id = 'persisted'"
     )
     expect(stored).toEqual({ shape: 'array', acl_requirements: [[space], [page]] })
+    /** The writer records when its evidence expires, so a reader never re-derives it. */
+    const [expiry] = await connection.unsafe(
+      `SELECT acl_valid_until = acl_verified_at + interval '24 hours' AS matches_evidence
+       FROM document WHERE id = 'persisted'`
+    )
+    expect(expiry).toEqual({ matches_evidence: true })
+    await persistDocumentAcls('admin', new Map([['page', { acl: [], requirements: [] }]]), executor)
+    const [revoked] = await connection.unsafe(
+      "SELECT acl_valid_until FROM document WHERE id = 'persisted'"
+    )
+    expect(revoked).toEqual({ acl_valid_until: null })
   })
 
   it.each([

apps/sim/lib/knowledge/access/predicate.ts

@@ -12,7 +12,7 @@ import {
 } from '@sim/db/schema'
 import { type SQL, sql } from 'drizzle-orm'
 import { EXTERNAL_GROUP_STALE_AFTER_MS } from '@/lib/knowledge/access/external-groups'
-import { SOURCE_ACL_MAX_AGE_MS } from '@/lib/knowledge/access/freshness'
+import { aclFreshnessCutoff } from '@/lib/knowledge/access/freshness'
 import { confluenceReaderGroupCondition } from '@/lib/knowledge/access/group-membership'
 import type { KnowledgeAccessScope, SystemAccessScope } from '@/lib/knowledge/access/types'
 import { documentConnectorIsActive } from '@/lib/knowledge/documents/connector-lifecycle'
@@ -220,7 +220,21 @@ function storedKnowledgeAccessCondition(
   if (scope.kind === 'system') return documentConnectorIsActive()
   if (scope.tokens.length === 0) return sql`false`
   const tokens = textArrayLiteral(scope.tokens)
-  const cutoff = sql`statement_timestamp() - (${SOURCE_ACL_MAX_AGE_MS} * interval '1 millisecond')`
+  const cutoff = aclFreshnessCutoff()
+  /**
+   * Mirrored permissions are current while the row says so. Every writer of an ACL records when
+   * its evidence expires, so a candidate is checked with one comparison instead of a lookup per
+   * document into its connector's members and observations. The stored expiry is never later than
+   * the evidence it was written from: a members-mode document expires with the earliest
+   * observation its ACL names, so this can hide a document the per-candidate proof would still
+   * admit, and can never admit one it would refuse.
+   */
+  const mirroredIsCurrent = (mode: 'admin' | 'members'): SQL =>
+    scope.storedAclFreshness
+      ? sql`${knowledgeConnector.accessMode} = ${mode} AND ${document.aclValidUntil} > statement_timestamp()`
+      : mode === 'admin'
+        ? sql`${knowledgeConnector.accessMode} = 'admin' AND ${document.aclVerifiedAt} > ${cutoff}`
+        : sql`${knowledgeConnector.accessMode} = 'members' AND ${memberObservationCondition(tokens, cutoff)}`
   return sql`(
     ${aclOverlap(tokens)}
     AND NOT EXISTS (
@@ -240,8 +254,8 @@ function storedKnowledgeAccessCondition(
           AND (
             (${knowledgeConnector.accessMode} = 'workspace' AND ${document.acl} = ARRAY['ws']::text[])
             OR (${document.acl} <> ARRAY['ws']::text[] AND (
-            (${knowledgeConnector.accessMode} = 'admin' AND ${document.aclVerifiedAt} > ${cutoff})
-            OR (${knowledgeConnector.accessMode} = 'members' AND ${memberObservationCondition(tokens, cutoff)})
+            (${mirroredIsCurrent('admin')})
+            OR (${mirroredIsCurrent('members')})
             ))
           )
       )

apps/sim/lib/knowledge/access/scope.ts

@@ -16,6 +16,7 @@ import {
 import { createLogger } from '@sim/logger'
 import { getErrorMessage } from '@sim/utils/errors'
 import { and, eq, gte, inArray, isNull, or, sql } from 'drizzle-orm'
+import { isFeatureEnabled } from '@/lib/core/config/feature-flags'
 import { OrchestrationError } from '@/lib/core/orchestration/types'
 import { type ResourceScope, resourceScopeFromOwner } from '@/lib/core/resource-scope'
 import { resourceScopeCondition } from '@/lib/core/resource-scope.server'
@@ -355,25 +356,46 @@ async function resolveKnowledgeIdentity(
   if (subject?.kind !== 'sim_user') {
     if (context.organizationId)
       throw new OrchestrationError('forbidden', 'Organization search requires a user subject')
-    return { access: WORKSPACE_ACCESS_SCOPE, githubReaders: [], confluenceReaders: [] }
+    const storedAclFreshness = await storedAclFreshnessEnabled()
+    return {
+      /** The shared constant while the rollout has not reached this request. */
+      access: storedAclFreshness.storedAclFreshness
+        ? { ...WORKSPACE_ACCESS_SCOPE, ...storedAclFreshness }
+        : WORKSPACE_ACCESS_SCOPE,
+      githubReaders: [],
+      confluenceReaders: [],
+    }
   }
   return resolveUserKnowledgeIdentity(subject.userId, context)
 }
 
 async function resolveUserKnowledgeIdentity(userId: string, context: KnowledgeAccessScopeContext) {
   resourceScopeFromOwner(context)
-  const {
-    tokens,
-    githubReaders = [],
-    confluenceReaders = [],
-  } = await loadUserAccess(userId, context)
+  const [{ tokens, githubReaders = [], confluenceReaders = [] }, storedAclFreshness] =
+    await Promise.all([loadUserAccess(userId, context), storedAclFreshnessEnabled()])
   return {
-    access: { kind: 'user' as const, userId, tokens },
+    access: { kind: 'user' as const, userId, tokens, ...storedAclFreshness },
     githubReaders,
     confluenceReaders,
   }
 }
 
+/**
+ * Whether this request reads mirrored-permission freshness from the document row. Resolved once
+ * per scope so the predicate stays synchronous; a configuration read that fails keeps the
+ * per-candidate proof, which is correct and only slower.
+ */
+async function storedAclFreshnessEnabled(): Promise<{ storedAclFreshness?: true }> {
+  try {
+    return (await isFeatureEnabled('knowledge-acl-expiry')) ? { storedAclFreshness: true } : {}
+  } catch (error) {
+    logger.warn('Feature flag read failed; deriving ACL freshness per candidate', {
+      error: getErrorMessage(error),
+    })
+    return {}
+  }
+}
+
 /**
  * The scope of a person identified only by user id — the shape session-backed
  * routes outside the application layer have in hand. Never call this with a
@@ -384,7 +406,11 @@ export async function resolveUserKnowledgeAccessScope(
   userId: string,
   workspaceId: string | undefined
 ): Promise<KnowledgeAccessScope> {
-  return { kind: 'user', userId, tokens: (await loadUserAccess(userId, { workspaceId })).tokens }
+  const [access, storedAclFreshness] = await Promise.all([
+    loadUserAccess(userId, { workspaceId }),
+    storedAclFreshnessEnabled(),
+  ])
+  return { kind: 'user', userId, tokens: access.tokens, ...storedAclFreshness }
 }
 
 /** Memoises {@link resolveKnowledgeAccessScope} for one operation; a failed lookup is retried on the next call. */

apps/sim/lib/knowledge/access/types.ts

@@ -30,6 +30,8 @@ export const WORKSPACE_ACCESS_TOKENS = [PUBLIC_ACCESS_TOKEN, WORKSPACE_ACCESS_TO
 export interface WorkspaceAccessScope {
   kind: 'workspace'
   tokens: typeof WORKSPACE_ACCESS_TOKENS
+  /** See {@link UserAccessScope.storedAclFreshness}. */
+  storedAclFreshness?: true
 }
 
 export interface UserAccessScope {
@@ -45,6 +47,14 @@ export interface UserAccessScope {
   githubInstallationGrants?: readonly GitHubInstallationReadGrant[]
   /** Current reader access to the immutable Confluence site behind a central crawl. */
   confluenceSiteGrants?: readonly ConfluenceSiteReadGrant[]
+  /**
+   * Whether mirrored-permission freshness is read from `document.acl_valid_until` instead of
+   * being re-derived per candidate from observation and verification timestamps. Resolved once
+   * per scope from the `knowledge-acl-expiry` rollout flag, so a predicate stays synchronous and
+   * every surface that carries a scope agrees on one answer for the whole request. Absent while
+   * the flag is off, so a scope is unchanged until the rollout reaches it.
+   */
+  storedAclFreshness?: true
 }
 
 export interface ConfluenceSiteReadGrant {

apps/sim/lib/knowledge/connectors/member-observations.ts

@@ -21,6 +21,7 @@ import {
   sql,
 } from 'drizzle-orm'
 import type { DbOrTx } from '@/lib/db/types'
+import { aclValidUntilFrom } from '@/lib/knowledge/access/freshness'
 import { textArrayLiteral } from '@/lib/knowledge/access/predicate'
 import {
   MEMBER_OBSERVATION_STALE_AFTER_HOURS,
@@ -66,6 +67,25 @@ function observedAcl() {
   ), '{}'::text[])`
 }
 
+/**
+ * When a members-mode document's ACL stops being current evidence: the earliest expiry among the
+ * active observations it was materialized from, so the row fails closed as soon as any observer
+ * it names goes stale. The staleness sweep then drops that observation and re-materializes, which
+ * restores the document for the observers that are still fresh.
+ */
+function observedAclValidUntil() {
+  return sql<Date | null>`(
+    SELECT min(${aclValidUntilFrom(
+      sql`GREATEST(${knowledgeDocumentObservation.lastSeenAt}, ${knowledgeConnectorMember.memberSyncedThrough})`
+    )})
+    FROM ${knowledgeDocumentObservation}
+    JOIN ${knowledgeConnectorMember}
+      ON ${knowledgeConnectorMember.id} = ${knowledgeDocumentObservation.memberId}
+     AND ${knowledgeConnectorMember.status} = 'active'
+    WHERE ${knowledgeDocumentObservation.documentId} = ${document.id}
+  )`
+}
+
 function observationQuery() {
   return db
     .select({ one: sql`1` })
@@ -340,12 +360,17 @@ export async function materializeDocumentAcls(
     const batch = ids.slice(offset, offset + MATERIALIZE_BATCH_SIZE)
     const rows = await executor
       .update(document)
-      .set({ acl: observedAcl(), aclRequirements: [], aclVerifiedAt: null })
+      .set({
+        acl: observedAcl(),
+        aclRequirements: [],
+        aclVerifiedAt: null,
+        aclValidUntil: observedAclValidUntil(),
+      })
       .where(
         and(
           inArray(document.id, batch),
           eq(document.connectorId, connectorId),
-          sql`(${document.acl} IS DISTINCT FROM ${observedAcl()} OR ${document.aclRequirements} <> '[]'::jsonb OR ${document.aclVerifiedAt} IS NOT NULL)`
+          sql`(${document.acl} IS DISTINCT FROM ${observedAcl()} OR ${document.aclRequirements} <> '[]'::jsonb OR ${document.aclVerifiedAt} IS NOT NULL OR ${document.aclValidUntil} IS DISTINCT FROM ${observedAclValidUntil()})`
         )
       )
       .returning({ id: document.id })